Golang Web Crawler Exercise
A sample solution for the final Golang exercise to make a mock web crawler.
package main import ( "fmt" ) //This covers the final exercise, "Web Crawler" type Fetcher interface { // Fetch returns the body of URL and // a slice of URLs found on that page. Fetch(url string) (body string, urls []string, err error) } type FetchedUrl struct { parent string body string urls []string err error depth int } // Crawl uses fetcher to recursively crawl // pages starting with url, to a maximum of depth. func Crawl(url string, depth int, fetcher Fetcher) { results := make(chan *FetchedUrl) alreadyFetched := make(map[string]bool) fetch := func(url string, depth int) { body, urls, err := fetcher.Fetch(url) results <- &FetchedUrl{url, body, urls, err, depth} } go fetch(url, depth) alreadyFetched[url] = true for countFetching := 1; countFetching > 0; countFetching-- { result := <- results if result.err != nil { //skip bad urls fmt.Println(result.err) continue } fmt.Printf("found: %s %q\n", result.parent, result.body) if result.depth > 0 { for _, newUrl := range result.urls { if !alreadyFetched[newUrl] { countFetching++ go fetch(newUrl, depth-1) alreadyFetched[newUrl] = true } } } } close(results) } func main() { Crawl("http://golang.org/", 4, fetcher) } // fakeFetcher is Fetcher that returns canned results. type fakeFetcher map[string]*fakeResult type fakeResult struct { body string urls []string } func (f fakeFetcher) Fetch(url string) (string, []string, error) { if res, ok := f[url]; ok { return res.body, res.urls, nil } return "", nil, fmt.Errorf("not found: %s", url) } // fetcher is a populated fakeFetcher. var fetcher = fakeFetcher{ "http://golang.org/": &fakeResult{ "The Go Programming Language", []string{ "http://golang.org/pkg/", "http://golang.org/cmd/", }, }, "http://golang.org/pkg/": &fakeResult{ "Packages", []string{ "http://golang.org/", "http://golang.org/cmd/", "http://golang.org/pkg/fmt/", "http://golang.org/pkg/os/", }, }, "http://golang.org/pkg/fmt/": &fakeResult{ "Package fmt", []string{ "http://golang.org/", "http://golang.org/pkg/", }, }, "http://golang.org/pkg/os/": &fakeResult{ "Package os", []string{ "http://golang.org/", "http://golang.org/pkg/", }, }, }

This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License.
Download this code in plain text format here