Python搭建代理IP池实现接口设置与整体调度

# 数据库地址
HOST = '127.0.0.1'
# MySql端口
MYSQL_PORT = 3306
# MySQl用户名、密码
MYSQL_USERNAME = '***'
MYSQL_PASSWORD = '***'
# 数据库名
SQL_NAME = 'test'

# 代理等级
MAX_SCORE = 30
MIN_SCORE = 0
INITIAL_SCORE = 10

VALID_STATUS_CODES = [200, 302]

# 代理池数量界限
POOL_UPPER_THRESHOLD = 1000

# 检查周期
TESTER_CYCLE = 20
# 获取周期
GETTER_CYCLE = 300

# 测试API，建议抓哪个网站测哪个
TEST_URL = 'http://www.baidu.com'

# API配置
API_HOST = '0.0.0.0'
API_PORT = 5555

# 开关
TESTER_ENABLED = True
GETTER_ENABLED = True
API_ENABLED = True

# 最大批测试量
BATCH_TEST_SIZE = 10

from flask import Flask, g
from db import MySqlClient

__all__ = ['app']

app = Flask(__name__)

def get_conn():
 if not hasattr(g, 'mysql'):
  g.mysql = MySqlClient()
 return g.mysql

@app.route('/')
def index():
 return '<h2>Welcome to Proxy Pool System</h2>'

# 随机代理
@app.route('/random')
def get_proxy():
 conn = get_conn()
 return conn.random()

# 代理池总量
@app.route('/count')
def get_counts():
 conn = get_conn()
 return str(conn.count())

if __name__ == '__main__':
 app.run()

import time
from multiprocessing import Process
from api import app
from getter import Getter
from tester import Tester
from db import MySqlClient
from setting import *


class Scheduler():

 # 定时测试代理
 def schedule_tester(self, cycle=TESTER_CYCLE):
  tester = Tester()
  while True:
   print('测试器开始运行')
   tester.run()
   time.sleep(cycle)
 
 # 定时获取代理
 def schedule_getter(self, cycle=GETTER_CYCLE):
  getter = Getter()
  while True:
   print('开始抓取代理')
   getter.run()
   time.sleep(cycle)
 
 # 开启API
 def schedule_api(self):
  app.run(API_HOST, API_PORT)
 
 def run(self):
  print('代理池开始运行')
  
  if TESTER_ENABLED:
   tester_process = Process(target=self.schedule_tester)
   tester_process.start()
  
  if GETTER_ENABLED:
   getter_process = Process(target=self.schedule_getter)
   getter_process.start()
  
  if API_ENABLED:
   api_process = Process(target=self.schedule_api)
   api_process.start()

if __name__ == "__main__":
 scheduler = Scheduler()
 scheduler.run()