Python爬虫框架scrapy实现的文件下载功能示例

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractor import LinkExtractor
from weidashang.items import matplotlib
class MatSpider(scrapy.Spider):
  name = "mat"
  allowed_domains = ["matplotlib.org"]
  start_urls = ['https://matplotlib.org/examples']
  def parse(self, response):
　　　　　　　#抓取每个脚本文件的访问页面，拿到后下载
    link = LinkExtractor(restrict_css='div.toctree-wrapper.compound li.toctree-l2')
    for link in link.extract_links(response):
      yield scrapy.Request(url=link.url,callback=self.example)
  def example(self,response):
　　　　　　#进入每个脚本的页面，抓取源码文件按钮，并和base_url结合起来形成一个完整的url
    href = response.css('a.reference.external::attr(href)').extract_first()
    url = response.urljoin(href)
    example = matplotlib()
    example['file_urls'] = [url]
    return example

class MyFilePlipeline(FilesPipeline):
  def file_path(self, request, response=None, info=None):
    path = urlparse(request.url).path
    return join(basename(dirname(path)),basename(path))

ITEM_PIPELINES = {
  'weidashang.pipelines.MyFilePlipeline': 1,
}
FILES_STORE = 'examples_src'

class matplotlib(Item):
  file_urls = Field()
  files = Field()

from scrapy.cmdline import execute
execute(['scrapy', 'crawl', 'mat','-o','example.json'])