scrapy实践之翻页爬取的实现

import scrapy
 
from hello_world.items import HelloWorldItem
 
class MirSpider(scrapy.Spider):
  name = "MirSpider"
  start_urls = ["http://mirtarbase.cuhk.edu.cn/php/search.php?opt=species&org=bta&sort=id&order=asc&page=1"]
 
  def parse(self, response):
    domain = 'http://mirtarbase.cuhk.edu.cn'
    for row in response.xpath('//table/tr'):
      item = HelloWorldItem()
      res = []
      for col in (row.xpath('td/text()')):
        res.append(col.extract())
      if res[0] != 'Bos taurus':
        continue
      item['species'] = res[0]
      item['miRNA'] = res[2]
      item['target'] = res[3]
      item['total'] = res[4]
      item['papers'] = res[5]
      yield item
    for url in response.xpath('//a/@href').extract():
      if 'page' in url:
        url = domain + url
        yield scrapy.Request(url, callback = self.parse, dont_filter = False)

from itemadapter import ItemAdapter
 
 
class HelloWorldPipeline:
  def __init__(self):
    self.link_set = set()
 
  def process_item(self, item, spider):
    link = item['miRNA'] + item['target']
    if link in self.link_set:
      raise DropItem(item)
    self.link_set.add(link) 
    return item

ITEM_PIPELINES = {
  'hello_world.pipelines.HelloWorldPipeline': 300,
}