使用Perl和库WWW::Curl的爬虫程序步骤

cpan WWW::Curl

use strict;
use warnings;
use WWW::Curl::Easy;
# 创建一个 Curl 对象
my $curl = WWW::Curl::Easy->new;
# 设置请求的 URL
my $url = "https://www.example.com";
# 定义一个回调函数来处理获取到的网页内容
my $response_content = '';  # 用于保存网页内容
$curl->setopt(CURLOPT_URL, $url);
$curl->setopt(CURLOPT_WRITEFUNCTION, sub {
    my ($data) = @_;
    $response_content .= $data;  # 将数据追加到响应内容中
    return length($data);
});
# 执行请求
my $retcode = $curl->perform;
# 检查请求是否成功
if ($retcode == 0) {
    print "成功获取网页内容：\n";
    print substr($response_content, 0, 500);  # 打印前500个字符
} else {
    print "请求失败，错误代码：", $retcode, "\n";
}

$curl->setopt(CURLOPT_USERAGENT, 'Mozilla/5.0');

$curl->setopt(CURLOPT_HTTPHEADER, ['Accept-Language: en-US']);

my $post_data = 'key1=value1&key2=value2';
$curl->setopt(CURLOPT_URL, "https://www.example.com/post_endpoint");
$curl->setopt(CURLOPT_POST, 1);
$curl->setopt(CURLOPT_POSTFIELDS, $post_data);

$curl->setopt(CURLOPT_COOKIEJAR, "cookies.txt");
$curl->setopt(CURLOPT_COOKIEFILE, "cookies.txt");

use strict;
use warnings;
use WWW::Curl::Easy;
my $curl = WWW::Curl::Easy->new;
my $url = "https://www.example.com";
my $response_content = '';
# 设置请求 URL
$curl->setopt(CURLOPT_URL, $url);
# 设置 User-Agent
$curl->setopt(CURLOPT_USERAGENT, 'Mozilla/5.0');
# 设置回调函数来处理响应数据
$curl->setopt(CURLOPT_WRITEFUNCTION, sub {
    my ($data) = @_;
    $response_content .= $data;
    return length($data);
});
# 执行请求
my $retcode = $curl->perform;
# 检查请求状态
if ($retcode == 0) {
    print "网页内容获取成功！\n";
    print substr($response_content, 0, 500);  # 打印前500个字符
} else {
    print "请求失败，错误代码：", $retcode, "\n";
}

perl your_script.pl

print $curl->getinfo(CURLINFO_HTTP_CODE);  # 打印 HTTP 状态码