go语言实现简单的并发网页爬虫示例

go get github.com/PuerkitoBio/goquery

func fetchAndParse(url string) ([]string, error) {
    resp, err := http.Get(url)
    if err != nil {
        return nil, err
    }
    defer resp.Body.Close()
    doc, err := goquery.NewDocumentFromReader(resp.Body)
    if err != nil {
        return nil, err
    }
    var links []string
    doc.Find("a").Each(func(i int, s *goquery.Selection) {
        href, exists := s.Attr("href")
        if exists {
            links = append(links, href)
        }
    })
    return links, nil
}

func crawlLinks(startURL string) {
    // 创建一个带缓冲的 channel，限制同时并发的 goroutine 数量
    worklist := make(chan []string, 20)
    go func() { worklist <- []string{startURL} }()
    seen := make(map[string]bool)
    // 开始并发爬取
    for list := range worklist {
        for _, link := range list {
            if !seen[link] {
                seen[link] = true
                go func(link string) {
                    links, err := fetchAndParse(link)
                    if err == nil {
                        worklist <- links
                    }
                }(link)
            }
        }
    }
}

func main() {
    startURL := "http://example.com"
    crawlLinks(startURL)
}