Go语言实现多协程并发下载网页内容的完整代码

package main

import (
    "fmt"
    "io"
    "net/http"
    "strings"
    "sync"
    "time"
)

type Result struct {
    URL    string
    Status string
    Length int
    Error  error
}

// 下载网页内容并写入结果通道
func fetchURL(url string, wg *sync.WaitGroup, resultCh chan<- Result) {
    defer wg.Done()

    client := http.Client{
        Timeout: 5 * time.Second,
    }

    resp, err := client.Get(url)
    if err != nil {
        resultCh <- Result{URL: url, Status: "请求失败", Error: err}
        return
    }
    defer resp.Body.Close()

    body, err := io.ReadAll(resp.Body)
    if err != nil {
        resultCh <- Result{URL: url, Status: "读取失败", Error: err}
        return
    }

    resultCh <- Result{
        URL:    url,
        Status: resp.Status,
        Length: len(body),
    }
}

func main() {
    urls := []string{
        "https://example.com",
        "https://httpbin.org/get",
        "https://golang.org",
        "https://nonexistent.example.com", // 故意的错误URL
    }

    var wg sync.WaitGroup
    resultCh := make(chan Result, len(urls))

    // 启动多个下载协程
    for _, url := range urls {
        wg.Add(1)
        go fetchURL(url, &wg, resultCh)
    }

    // 等待所有任务完成后关闭通道
    go func() {
        wg.Wait()
        close(resultCh)
    }()

    // 读取结果
    for res := range resultCh {
        if res.Error != nil {
            fmt.Printf("[失败] %s：%v\n", res.URL, res.Error)
        } else {
            snippet := fmt.Sprintf("%d 字节", res.Length)
            if res.Length > 0 {
                snippet = fmt.Sprintf("%s 内容预览：%s", snippet, strings.TrimSpace(string([]byte(res.URL)[:min(50, res.Length)])))
            }
            fmt.Printf("[成功] %s：%s\n", res.URL, snippet)
        }
    }

    fmt.Println("所有网页请求已完成。")
}

func min(a, b int) int {
    if a < b {
        return a
    }
    return b
}

[成功] https://example.com：1256 字节 内容预览：https://example.com
[成功] https://httpbin.org/get：349 字节 内容预览：https://httpbin.org/get
[成功] https://golang.org：3578 字节 内容预览：https://golang.org
[失败] https://nonexistent.example.com：Get "https://nonexistent.example.com": dial tcp: ...
所有网页请求已完成。

go fetchURL(url, &wg, resultCh)

wg.Add(1)
defer wg.Done()

resultCh := make(chan Result, len(urls))

go func() {
    wg.Wait()
    close(resultCh)
}()