利用python实现轻松抓取Google搜索数据

import requests

# 替换为你的 API 密钥和搜索引擎 ID
api_key = "YOUR_API_KEY"
search_engine_id = "YOUR_SEARCH_ENGINE_ID"
query = "Python 教程"

url = f"https://www.googleapis.com/customsearch/v1?key={api_key}&cx={search_engine_id}&q={query}"

try:
    response = requests.get(url)
    data = response.json()

    # 检查是否有搜索结果
    if 'items' in data:
        for item in data['items']:
            title = item.get('title')
            link = item.get('link')
            snippet = item.get('snippet')
            print(f"标题: {title}\n链接: {link}\n摘要: {snippet}\n{'-'*50}")
    else:
        print("未找到搜索结果。")

except requests.exceptions.RequestException as e:
    print(f"请求失败: {e}")

import requests
import time
from fake_useragent import UserAgent

# 导入你的代理管理模块
from your_proxy_manager import get_random_proxy

def get_google_search_results(query):
    headers = {
        "User-Agent": UserAgent().random,
        "Accept-Language": "en-US,en;q=0.9",
        "Referer": "https://www.google.com/"
    }
    
    # 获取一个随机住宅代理
    proxy = get_random_proxy()
    proxies = {
        'http': proxy,
        'https': proxy,
    }

    url = f"https://www.google.com/search?q={query}"
    
    try:
        response = requests.get(url, headers=headers, proxies=proxies, timeout=10)
        
        # 检查响应状态码和内容，处理验证码等情况
        if response.status_code == 200:
            # 使用 BeautifulSoup 或 lxml 解析 HTML
            # 提取搜索结果
            print("成功获取数据，开始解析...")
            # 解析逻辑...
            return True
        else:
            print(f"请求失败，状态码: {response.status_code}")
            return False
            
    except Exception as e:
        print(f"请求异常: {e}")
        return False
        
# 运行示例
if __name__ == "__main__":
    search_query = "数据抓取 Python"
    success = get_google_search_results(search_query)
    if not success:
        # 如果失败，等待一段时间后重试或切换代理
        print("重试...")
        time.sleep(5)
        get_google_search_results(search_query)