forked from go/golangs_learn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
33 lines
832 B
33 lines
832 B
package main
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/gocolly/colly"
|
|
)
|
|
|
|
func main() {
|
|
// 实例化默认收集器
|
|
c := colly.NewCollector(
|
|
// Visit only domains: hackerspaces.org, wiki.hackerspaces.org
|
|
colly.AllowedDomains("hackerspaces.org", "wiki.hackerspaces.org"),
|
|
)
|
|
|
|
// On every a element which has href attribute call callback
|
|
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
|
|
link := e.Attr("href")
|
|
// Print link
|
|
fmt.Printf("Link found: %q -> %s\n", e.Text, link)
|
|
// Visit link found on page
|
|
// Only those links are visited which are in AllowedDomains
|
|
c.Visit(e.Request.AbsoluteURL(link))
|
|
})
|
|
|
|
// Before making a request print "Visiting ..."
|
|
c.OnRequest(func(r *colly.Request) {
|
|
fmt.Println("Visiting", r.URL.String())
|
|
})
|
|
|
|
// Start scraping on https://hackerspaces.org
|
|
c.Visit("https://hackerspaces.org/")
|
|
}
|
|
|