Python通过BeautifulSoup抓取网页数据并解析

import asyncio
from crawl4ai import AsyncWebCrawler
from bs4 import BeautifulSoup

async def extract_div_text(html_content):
    """
    从HTML内容中提取特定样式的div文本
    
    参数:
        html_content: 网页的HTML内容
        
    返回:
        提取到的文本内容或未找到的提示信息
    """
    soup = BeautifulSoup(html_content, 'html.parser')
    # 查找目标 div（根据 style 属性匹配）
    target_div = soup.find('div', style=lambda
        value: value and 'cursor: default;font-size: 16px;line-height: 1.8;padding: 0 19px 25px' in value)

    if target_div:
        # 获取 div 内的所有文本，并清理空白
        text = target_div.get_text(separator='\n', strip=True)
        return text
    return "目标 div 未找到"

async def main():
    """
    主函数，执行网页抓取和内容提取
    """
    async with AsyncWebCrawler() as crawler:
        # 抓取目标网页
        result = await crawler.arun("https://www.jjwxc.net/onebook.php?novelid=2490683&chapterid=2")
        
        if hasattr(result, 'html'):
            # 提取目标div中的文本
            extracted_text = await extract_div_text(result.html)
            print(extracted_text)  # 打印全部字符
        else:
            print("未能获取 HTML 内容")

if __name__ == "__main__":
    # 运行异步主函数
    asyncio.run(main())

import asyncio
from crawl4ai import AsyncWebCrawler
from bs4 import BeautifulSoup

async def extract_div_text(html_content):
    soup = BeautifulSoup(html_content, 'html.parser')
    target_div = soup.find('div', style=lambda
        value: value and 'cursor: default;font-size: 16px;line-height: 1.8;padding: 0 19px 25px' in value)
    # ...其余代码...

async def main():
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun("目标URL")
        # ...处理结果...