Python实现获取网页信息并解析

import requests  
from bs4 import BeautifulSoup  
  
url = "http://www.ruanyifeng.com/blog/2023/05/weekly-issue-253.html"  
response = requests.get(url)  
soup = BeautifulSoup(response.content, "html.parser")  
first_tag = soup.find("h2", string="言论")  
next_sibling = first_tag.find_next_sibling()  
content1 = ""  
while next_sibling.name != "h2":  
    content1 += str(next_sibling.get_text())  
    # content1 += str(next_sibling)  
    content1 += "\n\n"  
    next_sibling = next_sibling.find_next_sibling()  
print(content1)

def find(self, name=None, attrs={}, recursive=True, text=None,  
         **kwargs):  
    """Look in the children of this PageElement and find the first  
    PageElement that matches the given criteria.  
    All find_* methods take a common set of arguments. See the online    documentation for detailed explanations.  
    :param name: A filter on tag name.    :param attrs: A dictionary of filters on attribute values.    :param recursive: If this is True, find() will perform a        recursive search of this PageElement's children. Otherwise,        only the direct children will be considered.    :param limit: Stop looking after finding this many results.    :kwargs: A dictionary of filters on attribute values.    :return: A PageElement.  
    :rtype: bs4.element.PageElement  
    """    
    r = None  
    l = self.find_all(name, attrs, recursive, text, 1, **kwargs)  
    if l:  
        r = l[0]  
    return r

def find_all(self, name=None, attrs={}, recursive=True, text=None,  
             limit=None, **kwargs):  
    """Look in the children of this PageElement and find all  
    PageElements that match the given criteria.  
    All find_* methods take a common set of arguments. See the online    documentation for detailed explanations.  
    :param name: A filter on tag name.    :param attrs: A dictionary of filters on attribute values.    :param recursive: If this is True, find_all() will perform a        recursive search of this PageElement's children. Otherwise,        only the direct children will be considered.    :param limit: Stop looking after finding this many results.    :kwargs: A dictionary of filters on attribute values.    :return: A ResultSet of PageElements.  
    :rtype: bs4.element.ResultSet  
    """    
    generator = self.descendants  
    if not recursive:  
        generator = self.children  
    return self._find_all(name, attrs, text, limit, generator, **kwargs)

soup.find_all('b')
# [<b>这里加粗</b>]

# 导入包
import re
for tag in soup.find_all(re.compile("^b")):
    print(tag.name)
# 结果会找出 body, b等b开头的标签

soup.find_all(["a", "b"])
# 输出： [<b>加粗</b>,
#  <a class="ddd" href="http://xxx" rel="external nofollow" >xxx</a> ]

# 判断一个tag有class属性，但是没有id属性
def has_class_but_no_id(tag):
    return tag.has_attr('class') and not tag.has_attr('id')
# 使用方式
soup.find_all(has_class_but_no_id)

# 查找所有href标签不是https的a标签
def not_https(href):
        return href and not re.compile("https").search(href)
soup.find_all(href=not_https)