https://github.com/gocolly/colly

    爬虫

    1. package main
    3. import (
    4.     "fmt"
    5.     "log"
    6.     "strconv"
    7.     "time"
    9.     "github.com/gocolly/colly"
    10. )
    12. func main() {
    13.     t := time.Now()
    14.     c := colly.NewCollector()
    16.     // Limit the number of threads started by colly to two
    17.     // when visiting links which domains' matches "*httpbin.*" glob
    18.     c.Limit(&colly.LimitRule{
    19.         DomainGlob:  "*wufazhuce.*",
    20.         Parallelism: 2,
    21.         RandomDelay: 5 * time.Second,
    22.     })
    24.     // On every a element which has href attribute call callback
    25.     c.OnHTML("div.tab-content", func(e *colly.HTMLElement) {
    26.         // 插画地址
    27.         imageURL := e.ChildAttr("img", "src")
    28.         fmt.Printf("imageURL: %s \n", imageURL)
    29.         // 引言
    30.         citation := e.ChildText("div.one-cita")
    31.         fmt.Printf("citation: %s \n", citation)
    32.         // 标号
    33.         vol := e.ChildText("div.one-titulo")
    34.         fmt.Printf("vol: %s \n", vol)
    35.         // 发布日
    36.         publishDate := e.ChildText("p.dom") + " " + e.ChildText("p.may")
    37.         fmt.Printf("publishDate: %s \n", publishDate)
    38.     })
    40.     // Before making a request print "Visiting ..."
    41.     c.OnRequest(func(r *colly.Request) {
    42.         fmt.Println("Visiting", r.URL.String())
    43.     })
    45.     c.OnScraped(func(r *colly.Response) {
    46.         fmt.Println("Finished", r.Request.URL)
    47.     })
    49.     c.OnError(func(_ *colly.Response, err error) {
    50.         log.Println("Something went wrong:", err)
    51.     })
    53.     for i := 1; i < 2819; i++ {
    54.         c.Visit("http://wufazhuce.com/one/" + strconv.Itoa(i))
    55.     }
    57.     c.Wait()
    58.     fmt.Printf("花费时间:%s", time.Since(t))
    59. }