Python爬虫库requests-html进行HTTP请求HTML解析等高级功能应用

pip install requests-html

from requests_html import HTMLSession
session = HTMLSession()
response = session.get('https://example.com')
print(response.html.text)

# 使用选择器提取标题
titles = response.html.find('h2')
for title in titles:
    print(title.text)

# JavaScript渲染
r = session.get('https://example.com', params={'q': 'python'})
r.html.render()
print(r.html.text)

# 异步JavaScript渲染
r = session.get('https://example.com')
r.html.render(sleep=1, keep_page=True)
print(r.html.text)

# 自定义Headers和Cookies
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
cookies = {'example_cookie': 'value'}
r = session.get('https://example.com', headers=headers, cookies=cookies)
print(r.html.text)

# 抓取动态页面
r = session.get('https://example.com/dynamic-page')
r.html.render()
print(r.html.text)

# 表单提交
payload = {'username': 'user', 'password': 'pass'}
r = session.post('https://example.com/login', data=payload)
print(r.html.text)

# 使用选择器提取链接
links = response.html.find('a')
for link in links:
    print(link.attrs['href'])

# 使用更复杂的选择器和过滤器
articles = response.html.find('article')
for article in articles:
    title = article.find('h2', first=True).text
    author = article.find('.author', first=True).text
    print(f"Title: {title}, Author: {author}")

# 等待页面加载完成
r = session.get('https://example.com/dynamic-content')
r.html.render(wait=2)
print(r.html.text)

# 生成页面截图
r = session.get('https://example.com')
r.html.render(screenshot='screenshot.png')

# 异常处理和错误页面重试
try:
    r = session.get('https://example.com/unstable-page')
    r.html.render(retries=3, wait=2)
    print(r.html.text)
except Exception as e:
    print(f"Error: {e}")

from requests_html import AsyncHTMLSession
async def fetch(url):
    async with AsyncHTMLSession() as session:
        response = await session.get(url)
        return response.html.text
urls = ['https://example.com/page1', 'https://example.com/page2', 'https://example.com/page3']
# 利用asyncio.gather实现并发请求
results = AsyncHTMLSession().run(lambda: [fetch(url) for url in urls])
for result in results:
    print(result)

from requests_html import HTMLSession
session = HTMLSession()
# 利用连接池发送多个请求
responses = session.get(['https://example.com/page1', 'https://example.com/page2', 'https://example.com/page3'])
for response in responses:
    print(response.html.text)

from requests_html import HTMLSession
session = HTMLSession()
# 使用缓存
response = session.get('https://example.com', cached=True)
print(response.html.text)