在Python中使用代理IP的方法详解

import urllib.request
# 设置代理IP
proxy_ip = "http://127.0.0.1:8888"
# 构造代理处理器对象
proxy_handler = urllib.request.ProxyHandler({"http": proxy_ip})
# 构造一个自定义的opener对象
opener = urllib.request.build_opener(proxy_handler)
# 使用自定义的opener对象发起访问请求
response = opener.open("http://www.baidu.com")
# 打印请求结果
print(response.read().decode("utf-8"))

import requests
# 设置代理IP
proxy_ip = "http://127.0.0.1:8888"
# 设置代理信息
proxies = {"http": proxy_ip}
# 发起请求
response = requests.get("http://www.baidu.com", proxies=proxies)
# 打印请求结果
print(response.text)

from selenium import webdriver
# 设置代理IP
proxy_ip = "127.0.0.1:8888"
# webdriver设置代理信息
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--proxy-server=http://" + proxy_ip)
driver = webdriver.Chrome(chrome_options=chrome_options)
# 发起请求
driver.get("http://www.baidu.com")
# 打印请求结果
print(driver.page_source)

DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 543,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
    'myproject.middlewares.RandomUserAgentMiddleware': 400,
    'myproject.middlewares.RandomProxyMiddleware': 700,
}

import random
import requests
class RandomProxyMiddleware(object):
    def __init__(self,):
        self.proxies = []
    def get_random_proxy(self):
        if not self.proxies:
            res = requests.get("http://127.0.0.1:8000/get_all/")
            self.proxies = [proxy for proxy in res.text.split("\n") if proxy]
        return random.choice(self.proxies)
    def process_request(self, request, spider):
        proxy = self.get_random_proxy()
        request.meta['proxy'] = "http://{}".format(proxy)