Python数据提取-lxml模块

html = etree.HTML(text)
ret_list = html.xpath("xpath语法规则字符串")

import requests
from lxml import etree

class Tieba(object):

    def __init__(self,name):
        self.url = "https://tieba.baidu.com/f?ie=utf-8&kw={}".format(name)
        self.headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'
        }

    def get_data(self,url):
        response = requests.get(url,headers=self.headers)
        with open("temp.html","wb") as f:
            f.write(response.content)

        return response.content

    def parse_data(self,data):
        # 创建element对象
        data = data.decode().replace("<!--","").replace("-->","")
        html =etree.HTML(data)

        el_list = html.xpath('//li[@class="j_thread_list clearfix"]/div/div[2]/div[1]/div[1]/a')
        #print(len(el_list))

        data_list = []

        for el in el_list:
            temp = {}
            temp['title'] = el.xpath("./text()")[0]
            temp['link'] = 'http://tieba.baidu.com' + el.xpath("./@href")[0]
            data_list.append(temp)

        # 获取下一页url
        try:
            next_url = 'https:' + html.xpath('//a[contains(text(),"下一页"]/@href')[0]
        except:
            next_url = None

        return data_list,next_url

    def save_data(self,data_list):
        for data in data_list:
            print(data)

    def run(self):
        # url
        # headers
        next_url = self.url

        while True:

            # 发送请求，获取响应
            data = self.get_data(self.url)
            # 从响应中提取数据（数据和翻页用的url)
            data_list,next_url = self.parse_data(data)

            self.save_data(data_list)

            print(next_url)

            # 判断是否终结
            if next_url == None:
                break

if __name__ == '__main__':
    tieba =Tieba("传智播客")
    tieba.run()

from lxml import etree

html_str = """<div<<ul>
        <li class="item-1"><a href="link1.html" rel="external nofollow" >first item</a></li>
        <li class="item-1"><a href="link2.html" rel="external nofollow" >second item</a></li>
        <li class="item-inactive"><a href="link3.html" rel="external nofollow" >third item</a></li>
        <li class="item-1"><a href="link4.html" rel="external nofollow" >fourth item</a></li>
        <li class="item=0"><a href="link5.html" rel="external nofollow" >fifth item</a>
        </ur></div>
"""

html = etree.HTML(html_str)

handeled_html_str = etree.tostring(html).decode()
print(handeled_html_str)