本节内容将介绍下scrapy引擎具体实现的功能。

引擎是整个scrapy的核心控制和调度scrapy运行.Engine的open_spider方法完成了一些初始化,以及启动调度器获取种子队列以及去重队列.最后调用self._nest_request开始一次爬取过程.

    @defer.inlineCallbacks    def open_spider(self, spider, start_requests=(), close_if_idle=True):        assert self.has_capacity(), "No free spider slot when opening %r" % \            spider.name        logger.info("Spider opened", extra={'spider': spider})        nextcall = CallLaterOnce(self._next_request, spider)  #首先获取next调用。        scheduler = self.scheduler_cls.from_crawler(self.crawler)  #从crawler获取调度器        start_requests = yield self.scraper.spidermw.process_start_requests(start_requests, spider)        slot = Slot(start_requests, close_if_idle, nextcall, scheduler)        self.slot = slot        self.spider = spider        yield scheduler.open(spider)   #启动调度器,        yield self.scraper.open_spider(spider)        self.crawler.stats.open_spider(spider)        yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)        slot.nextcall.schedule()  #调用self._nest_request方法        slot.heartbeat.start(5)

open_spider中slot调用_next_request, 接下来我们看看_next_request ，先是通过_needs_backout(spider)判断是否需要结束爬虫然后返回~然后通过self._next_request_from_scheduler(spider)方法判断是否还有URL需要去爬.

def _next_request(self, spider):        slot = self.slot        if not slot:            return        if self.paused:            return        while not self._needs_backout(spider):   #是否需要返回            if not self._next_request_from_scheduler(spider): #是否还有URL需要爬取                break        if slot.start_requests and not self._needs_backout(spider):            try:                request = next(slot.start_requests)            except StopIteration:                slot.start_requests = None            except Exception:                slot.start_requests = None                logger.error('Error while obtaining start requests',                             exc_info=True, extra={'spider': spider})            else:                self.crawl(request, spider)        if self.spider_is_idle(spider) and slot.close_if_idle:            self._spider_idle(spider)

_next_request又循环通过_next_request_from_scheduler(self,spider)方法从scheduler获取下一个需要爬取的request,然后送到下载器下载页面.

def next_request_from_scheduler(self, spider):        slot = self.slot        request = slot.scheduler.next_request() #从队列获取下一个待爬取的request        if not request:            return        d = self._download(request, spider)  #使用download下载request        d.addBoth(self._handle_downloader_output, request, spider)  #输出下载的response        d.addErrback(lambda f: logger.info('Error while handling downloader output',                                           exc_info=failure_to_exc_info(f),                                           extra={'spider': spider}))        d.addBoth(lambda : slot.removerequest(request))        d.addErrback(lambda f: logger.info('Error while removing request from slot',                                           exc_info=failure_to_exc_info(f),                                           extra={'spider': spider}))        d.addBoth(lambda : slot.nextcall.schedule())        d.addErrback(lambda f: logger.info('Error while scheduling new request',                                           exc_info=failure_to_exc_info(f),                                           extra={'spider': spider}))        return d

接下来将接受下scrapy中下载器download是如何实现的.

ccdef download(self, request, spider):        slot = self.slot        slot.add_request(request)        def _on_success(response):            assert isinstance(response, (Response, Request))            if isinstance(response, Response): #如果返回的是Response对象打印日志                response.request = request # tie request to response received                logkws = self.logformatter.crawled(request, response, spider)                logger.log(*logformatter_adapter(logkws), extra={'spider': spider})                self.signals.send_catch_log(signal=signals.response_received, \                    response=response, request=request, spider=spider)            return response        def _on_complete():            slot.nextcall.schedule()            return _        dwld = self.downloader.fetch(request, spider)  #使用downloader的fetch下载request        dwld.addCallbacks(_on_success) #添加成功回掉方法        dwld.addBoth(_on_complete)        return dwld