python selenium.webdriver 爬取政策文件的实现

from selenium import webdriver
from urllib.error import HTTPError
from selenium.webdriver.common.by import By   #selenium新版本写法
import warnings
warnings.filterwarnings('ignore')
"""
爬虫国务院文件
传入链接，返还链接内的全部内容，生成字典
"""
def get_info(id,url):
    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    driver = webdriver.Chrome(options=options)
    link = {}
    driver.get(url)
    try:
        link['文章ID'] = id  # 序列ID，从0—现有的文件数
        link['链接'] = url  # 原文链接
        #time.sleep(3)
        link['索引号'] = driver.find_element(By.XPATH,
            '/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[1]/td[2]').text  # 索引号
        link['标题'] = driver.find_element(By.XPATH,
            "/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[3]/td[2]").text  # 标题
        link['发文机关'] = driver.find_element(By.XPATH,
            "/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[2]/td[2]").text  # 发文机关
        link['发文字号'] = driver.find_element(By.XPATH,
            "/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[4]/td[2]").text  # 发文字号
        link['主题分类'] = driver.find_element(By.XPATH,
            "/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[1]/td[4]").text  # 主题分类
        link['成文日期'] = driver.find_element(By.XPATH,
            "/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[2]/td[4]").text  # 成文日期
        link['发布日期'] = driver.find_element(By.XPATH,
            '/html/body/div[4]/div/div[2]/div[1]/table/tbody/tr/td/table/tbody/tr[4]/td[4]')   # 发布日期
        link['文件内容'] = driver.find_element(By.XPATH,"//*[@id='UCAP-CONTENT']").text  # 内容
        with open('汽车/国务院文件/'+link['标题']+'.txt','w',encoding='utf-8') as file:
            file.write(link['文件内容'])
    except HTTPError:
        return None
    driver.quit()
    return link

import pandas as pd
df = pd.DataFrame()
with open('link1.txt','r',encoding='utf-8') as f:
    links = f.readlines()
for id,url in enumerate(links):
    url = url.strip('\n')
    print(url)
    result = get_info(id,url)
    df1 = pd.DataFrame.from_dict(result,'index').T
    df = pd.concat([df,df1],axis=0)
df

df.to_excel('汽车行业政策文本研究.xlsx',index=False)