爬虫相关

基于Tornado的异步爬虫

  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  4. import time
  5. import logging
  6. from datetime import timedelta
  7. from tornado import httpclient, gen, ioloop, queues
  8. import traceback
  9. from bs4 import BeautifulSoup
  12. def logged(class_):
  13.     logging.basicConfig(level=logging.INFO)
  14.     class_.logger = logging.getLogger(class_.__name__)
  15.     return class_
  18. @logged
  19. class AsyncSpider(object):
  20.     """A simple class of asynchronous spider."""
  21.     def __init__(self, urls, concurrency=10, results=None, **kwargs):
  22.         super(AsyncSpider, self).__init__(**kwargs)
  24.         self.concurrency = concurrency
  25.         self._q = queues.Queue()
  26.         self._fetching = set()
  27.         self._fetched = set()
  28.         if results is None:
  29.             self.results = []
  30.         for url in urls:
  31.             self._q.put(url)
  32.         httpclient.AsyncHTTPClient.configure(
  33.             "tornado.curl_httpclient.CurlAsyncHTTPClient"
  34.         )
  36.     def fetch(self, url, **kwargs):
  37.         fetch = getattr(httpclient.AsyncHTTPClient(), 'fetch')
  38.         http_request = httpclient.HTTPRequest(url, **kwargs)
  39.         return fetch(http_request, raise_error=False)
  41.     def handle_html(self, url, html):
  42.         """处理html页面"""
  43.         print(url)
  45.     def handle_response(self, url, response):
  46.         """处理http响应，对于200响应码直接处理html页面，
  47.         否则按照需求处理不同响应码"""
  48.         if response.code == 200:
  49.             self.handle_html(url, response.body)
  51.         elif response.code == 599:    # retry
  52.             self._fetching.remove(url)
  53.             self._q.put(url)
  55.     @gen.coroutine
  56.     def get_page(self, url):
  57.         # yield gen.sleep(10)    # sleep when need
  58.         try:
  59.             response = yield self.fetch(url)
  60.             self.logger.debug('######fetched %s' % url)
  61.         except Exception as e:
  62.             self.logger.debug('Exception: %s %s' % (e, url))
  63.             raise gen.Return(e)
  64.         raise gen.Return(response)    # py3 can just return response
  66.     @gen.coroutine
  67.     def _run(self):
  68.         @gen.coroutine
  69.         def fetch_url():
  70.             current_url = yield self._q.get()
  71.             try:
  72.                 if current_url in self._fetching:
  73.                     return
  75.                 self.logger.debug('fetching****** %s' % current_url)
  76.                 self._fetching.add(current_url)
  78.                 response = yield self.get_page(current_url)
  79.                 self.handle_response(current_url, response)    # handle reponse
  81.                 self._fetched.add(current_url)
  83.             finally:
  84.                 self._q.task_done()
  86.         @gen.coroutine
  87.         def worker():
  88.             while True:
  89.                 yield fetch_url()
  91.         # Start workers, then wait for the work queue to be empty.
  92.         for _ in range(self.concurrency):
  93.             worker()
  95.         yield self._q.join(timeout=timedelta(seconds=300000))
  97.         try:
  98.             assert self._fetching == self._fetched
  99.         except AssertionError:    # some http error not handle
  100.             print(self._fetching-self._fetched)
  101.             print(self._fetched-self._fetching)
  103.     def run(self):
  104.         io_loop = ioloop.IOLoop.current()
  105.         io_loop.run_sync(self._run)
  108. class MySpider(AsyncSpider):
  110.     def fetch(self, url, **kwargs):
  111.         """重写父类fetch方法可以添加cookies，headers，timeout等信息"""
  112.         cookies_str = "PHPSESSID=j1tt66a829idnms56ppb70jri4; pspt=%7B%22id%22%3A%2233153%22%2C%22pswd%22%3A%228835d2c1351d221b4ab016fbf9e8253f%22%2C%22_code%22%3A%22f779dcd011f4e2581c716d1e1b945861%22%7D; key=%E9%87%8D%E5%BA%86%E5%95%84%E6%9C%A8%E9%B8%9F%E7%BD%91%E7%BB%9C%E7%A7%91%E6%8A%80%E6%9C%89%E9%99%90%E5%85%AC%E5%8F%B8; think_language=zh-cn; SERVERID=a66d7d08fa1c8b2e37dbdc6ffff82d9e|1444973193|1444967835; CNZZDATA1254842228=1433864393-1442810831-%7C1444972138"
  113.         headers = {
  114.             'User-Agent': 'mozilla/5.0 (compatible; baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
  115.             'cookie': cookies_str
  116.         }
  117.         return super(MySpider, self).fetch(
  118.             url, headers=headers,
  119.             #proxy_host="127.0.0.1", proxy_port=8787,    # for proxy
  120.         )
  122.     def handle_html(self, url, html):
  123.         print(url)
  124.         #print(BeautifulSoup(html, 'lxml').find('title'))
  127. def main():
  128.     st = time.time()
  129.     urls = []
  130.     n = 1000
  131.     for page in range(1, n):
  132.         urls.append('http://www.jb51.net/article/%s.htm' % page)
  133.     s = MySpider(urls, 10)
  134.     s.run()
  135.     print(time.time()-st)
  136.     print(60.0/(time.time()-st)*1000, 'per minute')
  137.     print(60.0/(time.time()-st)*1000/60.0, 'per second')
  140. if __name__ == '__main__':
  141.     main()

写爬虫会遇到的一些工具函数

  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  4. """
  5. chrome有个功能，对于请求可以直接右键copy as curl，然后在命令行里边用curl
  6. 模拟发送请求。现在需要把此curl字符串处理成requests库可以传入的参数格式，
  7. http://stackoverflow.com/questions/23118249/whats-the-difference-between-request-payload-vs-form-data-as-seen-in-chrome
  8. """
  10. import re
  11. from functools import wraps
  12. import traceback
  13. import requests
  16. def encode_to_dict(encoded_str):
  17.     """ 将encode后的数据拆成dict
  18.     >>> encode_to_dict('name=foo')
  19.     {'name': foo'}
  20.     >>> encode_to_dict('name=foo&val=bar')
  21.     {'name': 'foo', 'val': 'var'}
  22.     """
  24.     pair_list = encoded_str.split('&')
  25.     d = {}
  26.     for pair in pair_list:
  27.         if pair:
  28.             key = pair.split('=')[0]
  29.             val = pair.split('=')[1]
  30.             d[key] = val
  31.     return d
  34. def parse_curl_str(s):
  35.     """convert chrome curl string to url, headers_dict and data"""
  36.     pat = re.compile("'(.*?)'")
  37.     str_list = [i.strip() for i in re.split(pat, s)]   # 拆分curl请求字符串
  39.     url = ''
  40.     headers_dict = {}
  41.     data = ''
  43.     for i in range(0, len(str_list)-1, 2):
  44.         arg = str_list[i]
  45.         string = str_list[i+1]
  47.         if arg.startswith('curl'):
  48.             url = string
  50.         elif arg.startswith('-H'):
  51.             header_key = string.split(':', 1)[0].strip()
  52.             header_val = string.split(':', 1)[1].strip()
  53.             headers_dict[header_key] = header_val
  55.         elif arg.startswith('--data'):
  56.             data = string
  58.     return url, headers_dict, data
  61. def retry(retries=3):
  62.     """一个失败请求重试，或者使用下边这个功能强大的retrying
  63.     pip install retrying
  64.     https://github.com/rholder/retrying
  66.     :param retries: number int of retry times.
  67.     """
  68.     def _retry(func):
  69.         @wraps(func)
  70.         def _wrapper(*args, **kwargs):
  71.             index = 0
  72.             while index < retries:
  73.                 index += 1
  74.                 try:
  75.                     response = func(*args, **kwargs)
  76.                     if response.status_code == 404:
  77.                         print(404)
  78.                         break
  79.                     elif response.status_code != 200:
  80.                         print(response.status_code)
  81.                         continue
  82.                     else:
  83.                         break
  84.                 except Exception as e:
  85.                     traceback.print_exc()
  86.                     response = None
  87.             return response
  88.         return _wrapper
  89.     return _retry
  92. _get = requests.get
  95. @retry(5)
  96. def get(*args, **kwds):
  97.     if 'timeout' not in kwds:
  98.         kwds['timeout'] = 10
  99.     if 'headers' not in kwds:
  100.         headers = {
  101.             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
  102.         }
  103.         kwds['headers'] = headers
  105.     return _get(*args, **kwds)
  107. requests.get = get
  110. def retry_get_html(*args, **kwargs):
  111.     try:
  112.         return get(*args, **kwargs).content
  113.     except AttributeError:
  114.         return ''
  117. def lazy_property(fn):
  118.     attr_name = '_lazy_' + fn.__name__
  120.     @property
  121.     def _lazy_property(self):
  122.         if not hasattr(self, attr_name):
  123.             setattr(self, attr_name, fn(self))
  124.             return getattr(self, attr_name)
  125.     return _lazy_property
  128. def my_ip():
  129.     # url = 'https://api.ipify.org?format=json'
  130.     url = 'http://httpbin.org/ip'
  131.     return requests.get(url).text
  134. def form_data_to_dict(s):
  135.     """form_data_to_dict s是从chrome里边复制得到的form-data表单里的字符串，
  136.     注意*必须*用原始字符串r""
  138.     :param s: form-data string
  139.     """
  140.     arg_list = [line.strip() for line in s.split('\n')]
  141.     d = {}
  142.     for i in arg_list:
  143.         if i:
  144.             k = i.split(':', 1)[0].strip()
  145.             v = ''.join(i.split(':', 1)[1:]).strip()
  146.             d[k] = v
  147.     return d
  149. if __name__ == '__main__':
  150.     import sys
  151.     from pprint import pprint
  152.     try:
  153.         curl_str = sys.argv[1]   # 用三引号括起来作为参数
  154.         url, headers_dict, data = parse_curl_str(curl_str)
  155.         print(url)
  156.         pprint(headers_dict)
  157.         print(data)
  158.     except IndexError:
  159.         exit(0)

如何使用代理

  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  4. # requests proxy demo
  5. import requests
  7. # install lantern first, 这是使用lantern的代理地址
  8. proxies = {
  9.     "http": "http://127.0.0.1:8787",
  10.     "https": "http://127.0.0.1:8787",
  11. }
  13. url = 'http://httpbin.org/ip'
  14. r = requests.get(url, proxies=proxies)
  15. print(r.text)
  18. # requests from version 2.10.0 support socks proxy
  19. # pip install -U requests[socks]
  20. proxies = {'http': "socks5://myproxy:9191"}
  21. requests.get('http://example.org', proxies=proxies)
  23. # tornado proxy demo
  24. # sudo apt-get install libcurl-dev librtmp-dev
  25. # pip install tornado pycurl
  27. from tornado import httpclient, ioloop
  29. config = {
  30.     'proxy_host': 'YOUR_PROXY_HOSTNAME_OR_IP_ADDRESS',
  31.     'proxy_port': 3128
  32. }
  34. httpclient.AsyncHTTPClient.configure(
  35.     "tornado.curl_httpclient.CurlAsyncHTTPClient")
  38. def handle_request(response):
  39.     if response.error:
  40.         print "Error:", response.error
  41.     else:
  42.         print response.body
  43.     ioloop.IOLoop.instance().stop()
  45. http_client = httpclient.AsyncHTTPClient()
  46. http_client.fetch("http://twitter.com/",
  47.     handle_request, **config)
  48. ioloop.IOLoop.instance().start()

使用线程池

  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  3. import concurrent.futures
  4. import bs4
  5. import requests
  8. class ThreadPoolCrawler(object):
  9.     def __init__(self, urls, concurrency=10, **kwargs):
  10.         self.urls = urls
  11.         self.concurrency = concurrency
  12.         self.results = []
  14.     def handle_response(self, url, response):
  15.         pass
  17.     def get(self, *args, **kwargs):
  18.         return requests.get(*args, **kwargs)
  20.     def run(self):
  21.         with concurrent.futures.ThreadPoolExecutor(max_workers=self.concurrency) as executor:
  22.             future_to_url = {
  23.                 executor.submit(self.get, url): url for url in self.urls
  24.             }
  25.             for future in concurrent.futures.as_completed(future_to_url):
  26.                 url = future_to_url[future]
  27.                 try:
  28.                     response = future.result()
  29.                 except Exception as e:
  30.                     import traceback
  31.                     traceback.print_exc()
  32.                 else:
  33.                     self.handle_response(url, response)
  36. class TestCrawler(ThreadPoolCrawler):
  37.     def handle_response(self, url, response):
  38.         soup = bs4.BeautifulSoup(response.text, 'lxml')
  39.         title = soup.find('title')
  40.         self.results.append({url: title})
  43. def main():
  44.     import time
  45.     urls = ['http://localhost:8000'] * 300
  46.     for nums in [2, 5, 10, 15, 20, 50, 70, 100]:
  47.         beg = time.time()
  48.         s = TestCrawler(urls, nums)
  49.         s.run()
  50.         print(nums, time.time()-beg)
  52. if __name__ == '__main__':
  53.     main()

使用tor代理ip

  1. #!/usr/bin/env python
  2. # -*- coding:utf-8 -*-
  4.     # http://ningning.today/2016/03/07/python/python-requests-tor-crawler/
  6. import os
  7. import requests
  8. import requesocks
  10. #url = 'https://api.ipify.org?format=json'
  11. url = 'http://httpbin.org/ip'
  14. def getip_requests(url):
  15.     print "(+) Sending request with plain requests..."
  16.     r = requests.get(url)
  17.     print "(+) IP is: " + r.text.replace("\n", "")
  20. def getip_requesocks(url):
  21.     print "(+) Sending request with requesocks..."
  22.     session = requesocks.session()
  23.     session.proxies = {'http': 'socks5://127.0.0.1:9050',
  24.                        'https': 'socks5://127.0.0.1:9050'}
  25.     r = session.get(url)
  26.     print "(+) IP is: " + r.text.replace("\n", "")
  29. def tor_requests():
  30.     proxies = {
  31.         'http': 'socks5://127.0.0.1:9050',
  32.         'https': 'socks5://127.0.0.1:9050',
  33.     }
  34.     r = requests.get(url, proxies=proxies)
  35.     print r.text
  38. def main():
  39.     print "Running tests..."
  40.     getip_requests(url)
  41.     getip_requesocks(url)
  42.     os.system("""(echo authenticate '"yourpassword"'; echo signal newnym; echo quit) | nc localhost 9051""")
  43.     getip_requesocks(url)
  46. if __name__ == "__main__":
  47.     main()
  48.     #tor_requests()