Node.js 实现简单爬虫的示例代码

npm init -y

npm install axios cheerio node-xlsx --save

import axios from "axios";
import cheerio from "cheerio";

// 目标网页的URL，这里使用 下厨房 这个美食网站作为测试
const targetUrl = "https://www.xiachufang.com/category/40076/";

// 请求目标网页，获取HTML内容
const getHtml = async () => {
  const response = await axios.get(targetUrl);

  if (response.status !== 200) {
    throw new Error("请求失败");
  }
  return response.data;
};

// 解析HTML内容，获取菜品的标题和图片链接
const getData = async (html) => {
  const $ = cheerio.load(html);
  const list = [];
  $(".normal-recipe-list li").each((i, elem) => {
    const imgUrl = $(elem).find("img").attr("src");
    const title = $(elem).find("p.name a").text();
    list.push({
      title: title.replace(/[\n\s]+/g, ""),
      imgUrl,
    });
  });
  return list;
};

import xlsx from "node-xlsx";
import fs from "fs";

// 根据 表头数据 和 列表数据 转换成二维数组
const transData = (columns, tableList) => {
  const data = columns.reduce(
    (acc, cur) => {
      acc.titles.push(cur.header);
      acc.keys.push(cur.key);
      return acc;
    },
    { titles: [], keys: [] }
  );

  const tableBody = tableList.map((item) => {
    return data.keys.map((key) => item[key]);
  });
  return [data.titles, ...tableBody];
};

const writeExcel = (list) => {
  // 表头
  const columns = [
    { header: "菜名", key: "title" },
    { header: "图片链接", key: "imgUrl" },
  ];

  // 构建表格数据
  const tableData = transData(columns, list);
  const workbook = xlsx.build([
    {
      name: "菜谱",
      data: tableData,
    },
  ]);

  // 写入文件
  fs.writeFileSync("./菜谱.xlsx", workbook, "binary");
};

(async () => {
  const html = await getHtml();
  const list = await getData(html);
  await writeExcel(list);
  console.log("执行完毕");
})();

node crawler.js