使用python搭建代理IP池实现接口设置与整体调度

import requests
from bs4 import BeautifulSoup
 
def get_proxy_ips():
    """
    Get the proxy IPs from zdaye.com
    """
    url = 'https://www.zdaye.com/'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
    html = requests.get(url, headers=headers).text
    soup = BeautifulSoup(html, 'html.parser')
    ips = soup.find_all('tr')
    proxy_ips = []
    for ip in ips[1:]:
        lst = ip.text.strip().split('\n')
        proxy_ip = {'ip': lst[0], 'port': lst[1]}
        proxy_ips.append(proxy_ip)
    return proxy_ips

pip install pymongo

from pymongo import MongoClient
 
def save_proxy_ips(proxy_ips):
    """
    Save the proxy IPs to MongoDB
    """
    client = MongoClient('mongodb://localhost:27017/')
    db = client['proxy_ips']
    coll = db['ips']
    coll.delete_many({})
    coll.insert_many(proxy_ips)

import random
 
class ProxyPool:
    def __init__(self, threshold=5):
        """
        Initialize the proxy pool
        """
        self.threshold = threshold
        self.client = MongoClient('mongodb://localhost:27017/')
        self.db = self.client['proxy_ips']
        self.coll = self.db['ips']
 
    def get_proxy_ip(self):
        """
        Get a random proxy IP from the pool
        """
        count = self.coll.count_documents({})
        if count == 0:
            return None
 
        proxy_ips = self.coll.find({}, {'_id': 0})
        ips = [proxy_ip for proxy_ip in proxy_ips]
        proxy_ip = random.choice(ips)
        ip = 'http://' + proxy_ip['ip'] + ':' + proxy_ip['port']
 
        return {'http': ip}
 
    def delete_proxy_ip(self, proxy_ip):
        """
        Delete the proxy IP from the pool
        """
        self.coll.delete_one(proxy_ip)
 
    def check_proxy_ip(self, proxy_ip):
        """
        Check if the given proxy IP is available
        """
        proxies = {'http': 'http://' + proxy_ip['ip'] + ':' + proxy_ip['port']}
        try:
            requests.get('https://www.baidu.com/', proxies=proxies, timeout=5)
            return True
        except:
            return False
 
    def update_pool(self):
        """
        Update the proxy pool
        """
        count = self.coll.count_documents({})
        if count < self.threshold:
            proxy_ips = get_proxy_ips()
            save_proxy_ips(proxy_ips)

class Scheduler:
    def __init__(self):
        self.proxy_pool = ProxyPool()
 
    def request(self, url):
        """
        Send a request to the given url using a random proxy IP
        """
        while True:
            proxy_ip = self.proxy_pool.get_proxy_ip()
            if proxy_ip is None:
                return None
            try:
                response = requests.get(url, proxies=proxy_ip, timeout=5)
                if response.status_code == 200:
                    return response
                elif response.status_code == 403:
                    self.proxy_pool.delete_proxy_ip(proxy_ip)
                else:
                    continue
            except:
                self.proxy_pool.delete_proxy_ip(proxy_ip)
 
    def run(self):
        """
        Run the scheduler to update the proxy pool
        """
        self.proxy_pool.update_pool()

import time
 
class Spider:
    def __init__(self):
        self.scheduler = Scheduler()
        self.client = MongoClient('mongodb://localhost:27017/')
        self.db = self.client['data']
        self.coll = self.db['info']
 
    def crawl(self):
        """
        Crawl data using the proxy pool
        """
        while True:
            response = self.scheduler.request('https://www.example.com/')
            if response is not None:
                html = response.text
                # parse the html to get the data
                data = {}
                self.coll.insert_one(data)
            time.sleep(1)
 
    def run(self):
        """
        Run the spider to crawl data
        """
        while True:
            self.scheduler.run()
            self.crawl()
            time.sleep(10)