基于curl数据采集之单页面并行采集函数get_htmls的使用

function get_htmls($urls, $options = array(), $method = 'get'){
     $mh = curl_multi_init();
     if($method == 'get'){//get方式传值 最常用
         foreach($urls as $key=>$url){
             $ch = curl_init($url);
             $options[CURLOPT_RETURNTRANSFER] = true;
             $options[CURLOPT_TIMEOUT] = 5;
             curl_setopt_array($ch,$options);
             $curls[$key] = $ch;
             curl_multi_add_handle($mh,$curls[$key]);
         }
     }elseif($method == 'post'){//post方式传值
         foreach($options as $key=>$option){
             $ch = curl_init($urls);
             $option[CURLOPT_RETURNTRANSFER] = true;
             $option[CURLOPT_TIMEOUT] = 5;
             $option[CURLOPT_POST] = true;
             curl_setopt_array($ch,$option);
             $curls[$key] = $ch;
             curl_multi_add_handle($mh,$curls[$key]);
         }
     }else{
         exit("参数出错!\n");
     }
     do{
         $mrc = curl_multi_exec($mh,$active);
         curl_multi_select($mh);//减少CPU压力 注释掉CPU压力变大
     }while($active);
     foreach($curls as $key=>$ch){
         $html = curl_multi_getcontent($ch);
         curl_multi_remove_handle($mh,$ch);
         curl_close($ch);
         $htmls[$key] = $html;
     }
     curl_multi_close($mh);
     return $htmls;
 }

$urls = array();
 for($i=1; $i<=5; $i++){
     $urls[] = 'http://www.baidu.com/s?wd=shili&pn='.(($i-1)*10).'&ie=utf-8';
 }
 $option[CURLOPT_USERAGENT] = 'Mozilla/5.0 (Windows NT 6.1; rv:19.0) Gecko/20100101 Firefox/19.0';
 $htmls = get_htmls($urls,$option);
 foreach($htmls as $html){
     echo $html;//这里得到html 就可以进行数据处理了
 }

 if(isset($_POST['username']) && isset($_POST['password'])){
     echo '用户名是: '.$_POST['username'].' 密码是: '.$_POST['password'];
 }else{
     echo '请求错误!';
 }

$url = 'http://localhost/yourpath/post.php';//这里是你的路径
 $options = array();
 for($i=1; $i<=5; $i++){
     $option[CURLOPT_POSTFIELDS] = 'username=user'.$i.'&password=pass'.$i;
     $options[] = $option;
 }
 $htmls = get_htmls($url,$options,'post');
 foreach($htmls as $html){
     echo $html;//这里得到html 就可以进行数据处理了
 }