TypeScript使用axios库进行高效的网页数据抓取

npm install typescript axios --save

tsc --init

{
  "compilerOptions": {
    "target": "es5",
    "module": "commonjs",
    "strict": true,
    "esModuleInterop": true,
    "outDir": "./dist"
  },
  "include": [
    "./src/**/*"
  ],
  "exclude": [
    "node_modules"
  ]
}

// src/axiosInstance.ts
import axios from 'axios';

const axiosInstance = axios.create({
  baseURL: 'https://www.douyin.com',
  timeout: 1000,
  headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
});

export default axiosInstance;

// src/axiosInstance.ts
import axios from 'axios';
import ProxyAgent from 'axios-http-proxy-fix';

const proxyHost = "www.16yun.cn";
const proxyPort = "5445";
const proxyUser = "16QMSOML";
const proxyPass = "280651";

// 创建一个HTTP代理代理实例
const httpProxyAgent = new ProxyAgent({
  host: proxyHost,
  port: proxyPort,
  protocol: 'http:', // 注意：根据你的代理服务器协议进行设置
  auth: `${proxyUser}:${proxyPass}`
});

// 创建axios实例并应用代理
const axiosInstance = axios.create({
  baseURL: 'https://www.douyin.com',
  timeout: 1000,
  httpAgent: httpProxyAgent, // 应用HTTP代理
  httpsAgent: httpProxyAgent, // 如果axios请求是HTTPS，则应用HTTP代理
  headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}
});

export default axiosInstance;

// src/app.ts
import fetchData from './dataFetcher';

document.addEventListener('DOMContentLoaded', () => {
  const videoUrl = 'https://www.douyin.com/some/video/url';
  fetchData(videoUrl)
    .then(data => {
      console.log('Fetched video data:', data);
      // 处理获取的视频数据
    })
    .catch(error => {
      console.error('Failed to fetch video data:', error);
    });
});

tsc

npm install -g http-server
http-server dist