Python爬虫之爬取二手房信息

form lxml import etree
……   ……
tree = etree.HTML(源码)
li_list = tree.xpath( li 标签的路径)

import requests
from lxml import etree
import re

if __name__ == "__main__":
    # UA伪装
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36"
        }

    for pg in range(1, 3): # 翻两页
        # 指定 url
        url = "https://bj.c21.com.cn/ershoufang/pg%s/" % str(pg)

        # 获取网页源码
        page = requests.get(url = url, headers = header).text

        # xpath 解析
        tree = etree.HTML(page)
        li_list = tree.xpath('//ul[@id="availability"]/li') 
        for li in li_list:
            title = li.xpath('div[2]/div/a/text()')[0] # 房子的名称
            # print(title[0]) # 测试
            add = li.xpath('div[2]/div/p//a/text()') # 地址
            add = add[-2: ] + add[0:1] # 地址范围由大到小
            # print(add) # 测试
            div_list = li.xpath('div[2]/div[2]/div')
            # 具体信息
            message_list = ["建筑面积", "房屋户型", "房屋朝向", "所在楼层", "装修情况", "建成时间"]
            for i in range(6):
                div = div_list[i]
                message = div.xpath('span/text()')[0]
                message = re.sub("\s", "", str(message)) # 因为发现获取的文本有很多换行符和空格，所以需要去掉
                message = re.sub("\\n", "", str(message))
                message_list[i] = message_list[i] + "：" + message
            # print(message_list) # 测试
            # 交通情况
            traffic = li.xpath('div[2]/div[4]//text()')
            # print(traffic) # 测试
            # 价格情况
            price = li.xpath('div[2]/div[3]//text()')
            price = price[0] + price[1]
            # print(price) # 测试
            with open("C:\\Users\\ASUS\\Desktop\\CSDN\\数据解析\\xpath\\二手房\\" + "二手房.txt", "a", encoding = "utf-8") as fp:
                fp.write(title + "\n")
                for message in message_list:
                    fp.write(message + "\n")
                if traffic == []:
                    fp.write("交通情况：无介绍" + "\n")
                else:
                    fp.write("交通情况：" + traffic[0] + "\n")
                fp.write("价格：" + price + "\n\n")
            print(title, "下载完成！！！")

    print("over!!!")