使用Scrapy框架爬取网页并保存到Mysql的实现

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
 
import scrapy
 
 
class HptyItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
 
    球员 = scrapy.Field()
    球队 = scrapy.Field()
    排名 = scrapy.Field()
    场均得分 = scrapy.Field()
    命中率 = scrapy.Field()
    三分命中率 = scrapy.Field()
    罚球命中率 = scrapy.Field()

import scrapy
from ..items import HptyItem
 
 
class SwwSpider(scrapy.Spider):
    name = 'sww'
    allowed_domains = ['https://nba.hupu.com/stats/players']
    start_urls = ['https://nba.hupu.com/stats/players']
 
    def parse(self, response):
        whh = response.xpath('//tbody/tr[not(@class)]')
        for i in whh:
            排名 = i.xpath(
                './td[1]/text()').extract()# 排名
            球员 = i.xpath(
                './td[2]/a/text()').extract()  # 球员
            球队 = i.xpath(
                './td[3]/a/text()').extract()  # 球队
            场均得分 = i.xpath(
                './td[4]/text()').extract()  # 得分
 
            命中率 = i.xpath(
                './td[6]/text()').extract()  # 命中率
            三分命中率 = i.xpath(
                './td[8]/text()').extract()  # 三分命中率
            罚球命中率 = i.xpath(
                './td[10]/text()').extract()  # 罚球命中率
 
            data = HptyItem(球员=球员, 球队=球队, 排名=排名, 场均得分=场均得分, 命中率=命中率, 三分命中率=三分命中率, 罚球命中率=罚球命中率)
            yield data

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
 
# useful for handling different item types with a single interface
from cursor import cursor
from itemadapter import ItemAdapter
import pymysql
 
 
class HptyPipeline:
    def process_item(self, item, spider):
        db = pymysql.connect(host="Localhost", user="root", passwd="root", db="sww", charset="utf8")
        cursor = db.cursor()
        球员 = item["球员"][0]
        球队 = item["球队"][0]
        排名 = item["排名"][0]
        场均得分 = item["场均得分"][0]
        命中率 = item["命中率"]
        三分命中率 = item["三分命中率"][0]
        罚球命中率 = item["罚球命中率"][0]
        # 三分命中率 = item["三分命中率"][0].strip('%')
        # 罚球命中率 = item["罚球命中率"][0].strip('%')
 
        cursor.execute(
            'INSERT INTO nba(球员,球队,排名,场均得分,命中率,三分命中率,罚球命中率) VALUES (%s,%s,%s,%s,%s,%s,%s)',
            (球员, 球队, 排名, 场均得分, 命中率, 三分命中率, 罚球命中率)
        )
        # 对事务操作进行提交
        db.commit()
        # 关闭游标
        cursor.close()
        db.close()
        return item

create database sww;

create table nba (球员 char(20),球队 char(10),排名 char(10),场均得分 char(25),命中率 char(20),三分命中率 char(20),罚球命中率 char(20));