Python 使用BeautifulSoup库的方法

import requests
from bs4 import BeautifulSoup

header = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98"}

# 参数1: 解析页面URL
# 参数2: 需要解析的页面定位
# 参数3: 提取标签属性
# 参数4：设置超时时间
# 参数5：设置返回类型(attribute 返回属性字段,text 返回文本字段)
def get_page_attrs(url,regx,attrs,timeout,type):
    respon_page = []
    try:
        respon = requests.get(url=url, headers=header, timeout=timeout)
        if respon.status_code == 200:
            if respon != None:
                soup = BeautifulSoup(respon.text, "html.parser")
                ret = soup.select(regx)
                for item in ret:
                    if type == "attribute":
                        respon_page.append( str(item.attrs[attrs] ))
                    if type == "text":
                        respon_page.append(str(item.get_text()))

            return respon_page
        else:
            return None
    except Exception:
        return None
    return None

# 对页面多次搜索
# 参数1: 需要解析的html文本
# 参数2: 需要解析的页面定位
# 参数3: 提取标签属性
# 参数5：设置返回类型(attribute 返回属性字段,text 返回文本字段)
def search_page(data,regx,attrs,type):
    respon_page = []
    if data != None:
        soup = BeautifulSoup(data, "html.parser")
        ret = soup.select(regx)
        for item in ret:
            if type == "attribute":
                respon_page.append( str(item.attrs[attrs] ))
            if type == "text":
                respon_page.append(str(item.get_text()))
    return respon_page

if __name__ == "__main__":
    # 通过CSS属性定位图片
    ref = get_page_attrs("https://www.cnblogs.com/LyShark/p/15914868.html",
                   "#cnblogs_post_body > p > img",
                   "src",
                   5,
                   "attribute"
                   )
    print(ref)

if __name__ == "__main__":
    # 定位文章列表,两种方式均可
    ref = get_page_attrs("https://www.cnblogs.com/lyshark",
                   "#mainContent > div > div > div.postTitle > a",
                   "href",
                   5,
                   "attribute"
                   )
    print(ref)
    ref = get_page_attrs("https://www.cnblogs.com/lyshark",
                   "div[class='day'] div[class='postCon'] div a",
                   "href",
                   5,
                   "attribute"
                   )
    print(ref)

if __name__ == "__main__":
    # 定位文章文本字段
    ref = get_page_attrs("https://www.cnblogs.com/lyshark",
                   "div[class='day'] div[class='postCon'] div[class='c_b_p_desc']",
                   "",
                   5,
                   "text"
                   )
    for index in ref:
        print(index)

if __name__ == "__main__":
    respon = requests.get(url="https://yiyuan.9939.com/yyk_47122/", headers=header, timeout=5)
    ref = search_page(respon.text,
                      "body > div.hos_top > div > div.info > div.detail.word-break > h1 > a",
                      "",
                      "text"
                      )
    print(ref)
    ref = search_page(respon.text,
                      "body > div.hos_top > div > div.info > div.detail.word-break > div.tel > span",
                      "",
                      "text"
                      )
    print(ref)

find_all(name=None, attrs={}, recursive=True, text=None, limit=None, **kwargs)

import re
import requests
from bs4 import BeautifulSoup
header = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98"}
# 查找文中 所有a标签 且类名是c_b_p_desc_readmore的 并提取出其href字段
# print(bs.find_all('a',class_='c_b_p_desc_readmore')[0]['href'])
# 提取 所有a标签 且id等于blog_nav_admin 类等于menu 并提取出其href字段
# print(bs.find_all('a',id='blog_nav_admin',class_='menu')[0]['href'])
# print(bs.find_all('a',id='blog_nav_admin',class_='menu')[0].attrs['href'])
if __name__ == "__main__":
    url = "https://cassandra.cerias.purdue.edu/CVE_changes/today.html"
    new_cve = []
    ret = requests.get(url=url, headers=header, timeout=5)
    soup = BeautifulSoup(ret.text, 'html.parser')
    for index in soup.find_all('a'):
        href = index.get('href')
        text = index.get_text()
        cve_number = re.findall("[0-9]{1,}-.*",index.get_text())
        print("序号: {:20} 地址: {} CVE-{}".format(text,href,cve_number[0]))

import requests
from bs4 import BeautifulSoup
header = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98"}
if __name__ == "__main__":
    ret = requests.get(url="https://www.cnblogs.com/lyshark", headers=header, timeout=3)
    text = str(ret.content.decode('utf-8'))
    bs = BeautifulSoup(text, "html.parser")
    ret = bs.select('#mainContent > div > div > div.postTitle > a > span')
    for i in ret:
        # 提取出字符串并以列表的形式返回
        string_ = list(i.stripped_strings)
        print(string_)

from bs4 import BeautifulSoup
import requests

head = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'}
ret = requests.get(url="http://www.weather.com.cn/textFC/beijing.shtml", headers=head, timeout=3)
text = str(ret.content.decode('utf-8'))

bs = BeautifulSoup(text,"html.parser")

# 定位到第一个标签上
bs.find_all('div',class_='conMidtab')[1]

# 在conMidtab里面找tr标签并从第3个标签开始保存
tr = bs.find_all('tr')[2:]

for i in tr:
    # 循环找代码中的所有td标签
    td = i.find_all('td')
    # 找所有的td标签,并找出第一个td标签
    city_td = td[0]
    # 获取目标路径下所有的子孙非标签字符串,自动去掉空字符串
    city = list(city_td.stripped_strings)[0]
    # 取出度数的标签
    temp = td[-5]
    temperature = list(temp.stripped_strings)[0]
    print('城市:{}   温度:{}'.format(city,temperature))