NodeJs爬虫框架Spider基础使用教程

npm i gz-spider --save

const spider = require('gz-spider');
// 每个爬虫是一个方法，需要通过setProcesser注册
spider.setProcesser({
  ['getGoogleSearchResult']: async (fetcher, params) => {
    // fetcher.page是原始的puppeteer page，可以直接用于打开页面
    let resp = await fetcher.axios.get(`https://www.google.com/search?q=${params}`);
    // throw 'Retry', will retry this processer
    // throw 'ChangeProxy', will retry this processer use new proxy
    // throw 'Fail', will finish this processer with message(fail) Immediately
    if (resp.status === 200) {
      // Data processing start
      let result = resp.data + 1;
      // Data processing end
      return result;
    } else {
      throw 'retry';
    }
  }
});
// 开始爬取
spider.getData('getGoogleSearchResult', params).then(userInfo => {
  console.log(userInfo);
});

spider.setFetcher({
  axiosTimeout: 5000,
  proxyTimeout: 180 * 1000
  proxy() {
    // 支持返回Promise，可以从远端拉取代理的配置
    return {
      host: '127.0.0.1',
      port: '9000'
    }
  }
});

spider.setStrategy({
  retryTimes: 2
});

const axios = require('axios');
while (true) {
  // 获取任务
  let resp = await axios.get('http://127.0.0.1:8080/spider/task');
  if (!resp.data.task) break;
  let { id, processerKey, processerInput } = resp.data.task;
  let processerOutput = await spider.getData(processerKey, processerInput);
  // 完成任务并带上处理好的数据
  await axios.put('http://127.0.0.1:8080/spider/task', {
    id, processerOutput,
    status: 'success'
  });
}