python解决网站的反爬虫策略总结

#! /usr/bin/env python3.4
#-*- coding:utf-8 -*-
#__author__ == "tyomcat"


import urllib.request
import random
import re

url='http://www.whatismyip.com.tw'
iplist=['121.193.143.249:80','112.126.65.193:80','122.96.59.104:82','115.29.98.139:9999','117.131.216.214:80','116.226.243.166:8118','101.81.22.21:8118','122.96.59.107:843']

proxy_support = urllib.request.ProxyHandler({'http':random.choice(iplist)})
opener=urllib.request.build_opener(proxy_support)
opener.addheaders=[('User-Agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36')]
urllib.request.install_opener(opener)
response = urllib.request.urlopen(url)
html = response.read().decode('utf-8')

pattern = re.compile('<h1>(.*?)</h1>.*?<h2>(.*?)</h2>')
iterms=re.findall(pattern,html)
for item in iterms:
  print(item[0]+":"+item[1])

#! /usr/bin/env python
# -*- coding:utf-8 -*-
#__author__ == "tyomcat"

from selenium import webdriver
import time
import re

drive = webdriver.PhantomJS(executable_path='phantomjs-2.1.1-linux-x86_64/bin/phantomjs')
drive.get('https://mm.taobao.com/self/model_info.htm?user_id=189942305&is_coment=false')

time.sleep(5)

pattern = re.compile(r'<div.*?mm-p-domain-info">.*?class="mm-p-info-cell clearfix">.*?<li>.*?<label>(.*?)</label><span>(.*?)</span>',re.S)
html=drive.page_source.encode('utf-8','ignore')
items=re.findall(pattern,html)
for item in items:
  print item[0],'http:'+item[1]
drive.close()