Scrapy从脚本运行到分布式爬取的技巧(进阶实践指南)

from scrapy.crawler import CrawlerProcess
from myproject.spiders.my_spider import MySpider
# 方式1：直接定义设置
process = CrawlerProcess({
    'FEEDS': {
        'output.json': {'format': 'json'},
    }
})
process.crawl(MySpider)
process.start()  # 阻塞直到爬取完成
# 方式2：加载项目配置
from scrapy.utils.project import get_project_settings
process = CrawlerProcess(get_project_settings())
process.crawl('followall', domain='scrapy.org')  # 通过名称调用
process.start()

from twisted.internet import reactor
from scrapy.crawler import CrawlerRunner
configure_logging()
runner = CrawlerRunner()
d = runner.crawl(MySpider)
d.addBoth(lambda _: reactor.stop())
reactor.run()  # 需手动管理reactor

process = CrawlerProcess(get_project_settings())
process.crawl(MySpider1)
process.crawl(MySpider2)
process.start()  # 同时启动两个爬虫

@defer.inlineCallbacks
def run_spiders():
    yield runner.crawl(MySpider1)
    yield runner.crawl(MySpider2)
reactor.callWhenRunning(run_spiders)
reactor.run()

curl http://scrapy1:6800/schedule.json \
  -d project=myproject \
  -d spider=spider1 \
  -d part=1

http://example.com/urls-to-crawl/spider1/part1.list
http://example.com/urls-to-crawl/spider1/part2.list