mirror of
https://github.com/ddidderr/wuffs.git
synced 2026-11-18 16:32:11 +01:00
Add script/crawl.go
This commit is contained in:
1 parent
8eb9781a9d
commit
48c7d1c030
1 file changed
+591
+591
@@ -0,0 +1,591 @@
|
||||
// Copyright 2018 The Wuffs Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build ignore
|
||||
|
||||
package main
|
||||
|
||||
// crawl.go crawls a list of HTTP and HTTPS URLs. If the URL yields an HTML
|
||||
// file, that file is parsed and the "<img src=etc>" tags within it are
|
||||
// followed (but not recursively).
|
||||
//
|
||||
// The crawler writes files to disk with filenames based on the hash of the
|
||||
// content (thus de-duplicating e.g. a site's 404 Not Found page even if served
|
||||
// from multiple URLs). It also writes a manifest.tsv file that records the
|
||||
// mapping from the original URL to that content hash.
|
||||
//
|
||||
// Usage: go run crawl.go -outdir foo -infile urls.txt 2>log.txt
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
var (
|
||||
datadirlevelsFlag = flag.Int("datadirlevels", 1,
|
||||
"number of directories in the ab/cd/efgh output data filenames; valid range is [0..8]")
|
||||
flushfreqFlag = flag.Int("flushfreq", 256, "write out in-progress manifest.tsv on every flushfreq entries")
|
||||
httptimeoutFlag = flag.Duration("httptimeout", 30*time.Second, "HTTP Client timeout; zero means unlimited")
|
||||
infileFlag = flag.String("infile", "", "source file containing URLs to crawl")
|
||||
inlimitFlag = flag.Int("inlimit", -1, "if non-negative, the maximum number of input URLs")
|
||||
nworkersFlag = flag.Int("nworkers", 16, "number of concurrent crawler workers")
|
||||
outdirFlag = flag.String("outdir", "", "directory to place crawled data")
|
||||
sleepFlag = flag.Duration("sleep", 1*time.Second,
|
||||
"minimum duration to sleep between HTTP requests to the same domain")
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := main1(); err != nil {
|
||||
os.Stderr.WriteString(err.Error() + "\n")
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func main1() error {
|
||||
flag.Parse()
|
||||
if *datadirlevelsFlag < 0 || 8 < *datadirlevelsFlag {
|
||||
return fmt.Errorf("-datadirlevels out of bounds [0..8]")
|
||||
}
|
||||
if *infileFlag == "" {
|
||||
return fmt.Errorf("-infile not given")
|
||||
}
|
||||
if *outdirFlag == "" {
|
||||
return fmt.Errorf("-outdir not given")
|
||||
}
|
||||
|
||||
urlsGroupedByDomain, err := parseInfile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
global.manifest, err = parseManifest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
urlsChan := make(chan []*url.URL)
|
||||
for i := 0; i < *nworkersFlag; i++ {
|
||||
wg.Add(1)
|
||||
w := worker(i)
|
||||
go w.work(&wg, urlsChan)
|
||||
}
|
||||
for _, u := range urlsGroupedByDomain {
|
||||
urlsChan <- u
|
||||
}
|
||||
close(urlsChan)
|
||||
log.Printf("master: no more work (inlimit is %d)", *inlimitFlag)
|
||||
wg.Wait()
|
||||
|
||||
return flush(global.manifest.m)
|
||||
}
|
||||
|
||||
var global struct {
|
||||
manifest manifest
|
||||
}
|
||||
|
||||
const keySize = sha256.Size // 32 bytes.
|
||||
|
||||
type key [keySize]byte
|
||||
|
||||
func (k key) Cmp(l key) int {
|
||||
for a := 0; a < keySize; a++ {
|
||||
if k[a] < l[a] {
|
||||
return -1
|
||||
} else if k[a] > l[a] {
|
||||
return +1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (k key) Str(levels int) string {
|
||||
const hex = "0123456789abcdef"
|
||||
var b [128]byte
|
||||
n := 0
|
||||
for i, x := range k {
|
||||
b[n+0] = hex[x>>4]
|
||||
b[n+1] = hex[x&0xF]
|
||||
n += 2
|
||||
if i < levels {
|
||||
b[n] = '/'
|
||||
n++
|
||||
}
|
||||
}
|
||||
return string(b[:n])
|
||||
}
|
||||
|
||||
func hash(b []byte) key {
|
||||
return sha256.Sum256(b)
|
||||
}
|
||||
|
||||
func unhex(b byte) uint8 {
|
||||
if '0' <= b && b <= '9' {
|
||||
return b - '0'
|
||||
}
|
||||
if 'a' <= b && b <= 'f' {
|
||||
return b + 10 - 'a'
|
||||
}
|
||||
return 0xFF
|
||||
}
|
||||
|
||||
func parseHash(b []byte) (k key, ok bool) {
|
||||
i := 0
|
||||
for i < keySize && len(b) >= 2 {
|
||||
if b[0] == '/' {
|
||||
b = b[1:]
|
||||
continue
|
||||
}
|
||||
u0 := unhex(b[0])
|
||||
u1 := unhex(b[1])
|
||||
if u0 > 15 || u1 > 15 {
|
||||
return key{}, false
|
||||
}
|
||||
k[i] = u0<<4 | u1
|
||||
i++
|
||||
b = b[2:]
|
||||
}
|
||||
return k, i == keySize && len(b) == 0
|
||||
}
|
||||
|
||||
type entry struct {
|
||||
contentHash key
|
||||
urlHash key
|
||||
httpStatusCode uint32
|
||||
sniffedMimeType string
|
||||
size uint64
|
||||
url *url.URL
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
lock sync.Mutex
|
||||
m map[key]entry // maps from urlHash to entry.
|
||||
numPending int
|
||||
flushing bool
|
||||
}
|
||||
|
||||
func (m *manifest) get(k key) entry {
|
||||
m.lock.Lock()
|
||||
e := m.m[k]
|
||||
m.lock.Unlock()
|
||||
return e
|
||||
}
|
||||
|
||||
func (m *manifest) put(k key, e entry) {
|
||||
clone := map[key]entry(nil)
|
||||
|
||||
m.lock.Lock()
|
||||
if m.m == nil {
|
||||
m.m = map[key]entry{}
|
||||
}
|
||||
m.m[k] = e
|
||||
m.numPending++
|
||||
if m.numPending >= *flushfreqFlag && !m.flushing {
|
||||
m.numPending = 0
|
||||
m.flushing = true
|
||||
clone = make(map[key]entry, len(m.m))
|
||||
for mk, mv := range m.m {
|
||||
clone[mk] = mv
|
||||
}
|
||||
}
|
||||
m.lock.Unlock()
|
||||
|
||||
if clone != nil {
|
||||
if err := flush(clone); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
m.lock.Lock()
|
||||
m.flushing = false
|
||||
m.lock.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func flush(m map[key]entry) error {
|
||||
log.Printf("Write manifest.tsv (%d entries) started...", len(m))
|
||||
defer log.Printf("Write manifest.tsv (%d entries) finished.", len(m))
|
||||
|
||||
keys := make([][2]key, 0, len(m))
|
||||
for _, v := range m {
|
||||
keys = append(keys, [2]key{v.contentHash, v.urlHash})
|
||||
}
|
||||
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if cmp := keys[i][0].Cmp(keys[j][0]); cmp != 0 {
|
||||
return cmp < 0
|
||||
}
|
||||
if cmp := keys[i][1].Cmp(keys[j][1]); cmp != 0 {
|
||||
return cmp < 0
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
filename0 := filepath.Join(*outdirFlag, "manifest.tsv.inprogress")
|
||||
filename1 := filepath.Join(*outdirFlag, "manifest.tsv")
|
||||
|
||||
f, err := os.Create(filename0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w := bufio.NewWriter(f)
|
||||
fmt.Fprintf(w, "#ContentHash\tURLHash\tHTTPStatusCode\tSniffedMIMEType\tSize\tURL\n")
|
||||
for _, k := range keys {
|
||||
e := m[k[1]]
|
||||
fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%d\t%v\n",
|
||||
e.contentHash.Str(*datadirlevelsFlag), e.urlHash.Str(0),
|
||||
e.httpStatusCode, e.sniffedMimeType, e.size, e.url)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(filename0, filename1)
|
||||
}
|
||||
|
||||
func parseInfile() (map[string][]*url.URL, error) {
|
||||
f, err := os.Open(*infileFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
m := map[string][]*url.URL{}
|
||||
|
||||
n := 0
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
b := s.Bytes()
|
||||
|
||||
// Strip leading whitespace (space, tab or other control character).
|
||||
for len(b) > 0 && b[0] <= ' ' {
|
||||
b = b[1:]
|
||||
}
|
||||
|
||||
// Skip empty lines or comments starting with "#".
|
||||
if len(b) == 0 || b[0] == '#' {
|
||||
continue
|
||||
}
|
||||
|
||||
// Strip everything up to the first whitespace.
|
||||
for i, x := range b {
|
||||
if x <= ' ' {
|
||||
b = b[:i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
u, err := url.Parse(string(b))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if *inlimitFlag >= 0 && n == *inlimitFlag {
|
||||
break
|
||||
}
|
||||
n++
|
||||
|
||||
m[u.Host] = append(m[u.Host], u)
|
||||
}
|
||||
if err := s.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func parseManifest() (manifest, error) {
|
||||
f, err := os.Open(filepath.Join(*outdirFlag, "manifest.tsv"))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
err = nil
|
||||
}
|
||||
return manifest{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
m := map[key]entry{}
|
||||
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
b := s.Bytes()
|
||||
|
||||
// Strip leading whitespace (space, tab or other control character).
|
||||
for len(b) > 0 && b[0] <= ' ' {
|
||||
b = b[1:]
|
||||
}
|
||||
|
||||
// Skip empty lines or comments starting with "#".
|
||||
if len(b) == 0 || b[0] == '#' {
|
||||
continue
|
||||
}
|
||||
|
||||
e := parseEntry(b)
|
||||
if e.url == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
m[e.urlHash] = e
|
||||
}
|
||||
if err := s.Err(); err != nil {
|
||||
return manifest{}, err
|
||||
}
|
||||
|
||||
return manifest{m: m}, nil
|
||||
}
|
||||
|
||||
func parseEntry(b []byte) (e entry) {
|
||||
if i := bytes.IndexByte(b, '\t'); i < 0 {
|
||||
return entry{}
|
||||
} else if h, ok := parseHash(b[:i]); !ok {
|
||||
return entry{}
|
||||
} else {
|
||||
e.contentHash = h
|
||||
b = b[i+1:]
|
||||
}
|
||||
|
||||
if i := bytes.IndexByte(b, '\t'); i < 0 {
|
||||
return entry{}
|
||||
} else if h, ok := parseHash(b[:i]); !ok {
|
||||
return entry{}
|
||||
} else {
|
||||
e.urlHash = h
|
||||
b = b[i+1:]
|
||||
}
|
||||
|
||||
if i := bytes.IndexByte(b, '\t'); i < 0 {
|
||||
return entry{}
|
||||
} else if u, err := strconv.ParseUint(string(b[:i]), 10, 32); err != nil {
|
||||
return entry{}
|
||||
} else {
|
||||
e.httpStatusCode = uint32(u)
|
||||
b = b[i+1:]
|
||||
}
|
||||
|
||||
if i := bytes.IndexByte(b, '\t'); i < 0 {
|
||||
return entry{}
|
||||
} else {
|
||||
e.sniffedMimeType = string(b[:i])
|
||||
b = b[i+1:]
|
||||
}
|
||||
|
||||
if i := bytes.IndexByte(b, '\t'); i < 0 {
|
||||
return entry{}
|
||||
} else if u, err := strconv.ParseUint(string(b[:i]), 10, 64); err != nil {
|
||||
return entry{}
|
||||
} else {
|
||||
e.size = uint64(u)
|
||||
b = b[i+1:]
|
||||
}
|
||||
|
||||
if u, err := url.Parse(string(b)); err != nil {
|
||||
return entry{}
|
||||
} else {
|
||||
e.url = u
|
||||
}
|
||||
|
||||
return e
|
||||
}
|
||||
|
||||
func sleep(rng *rand.Rand) {
|
||||
jitter := rng.Int63n(int64(*sleepFlag))
|
||||
time.Sleep(*sleepFlag + time.Duration(jitter))
|
||||
}
|
||||
|
||||
type worker int
|
||||
|
||||
func (w worker) work(wg *sync.WaitGroup, urlsChan chan []*url.URL) {
|
||||
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
defer wg.Done()
|
||||
for urls := range urlsChan {
|
||||
for _, u := range urls {
|
||||
e, links, fetched := w.work1(u, true)
|
||||
if fetched {
|
||||
sleep(rng)
|
||||
}
|
||||
|
||||
for _, u := range links {
|
||||
if ee, _, fetched := w.work1(u, false); fetched {
|
||||
sleep(rng)
|
||||
if ee.url != nil {
|
||||
global.manifest.put(ee.urlHash, ee)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fetched && e.url != nil {
|
||||
global.manifest.put(e.urlHash, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
log.Printf("worker #%03d: no more work", w)
|
||||
}
|
||||
|
||||
func (w worker) work1(u *url.URL, followHTML bool) (e entry, links []*url.URL, fetched bool) {
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return entry{}, nil, false
|
||||
}
|
||||
|
||||
urlString := u.String()
|
||||
urlHash := hash([]byte(urlString))
|
||||
e = global.manifest.get(urlHash)
|
||||
if e.url != nil {
|
||||
log.Printf("worker #%03d: skipping %q", w, urlString)
|
||||
return e, nil, false
|
||||
}
|
||||
log.Printf("worker #%03d: fetching %q", w, urlString)
|
||||
|
||||
statusCode, data, err := fetch(urlString)
|
||||
if err != nil {
|
||||
log.Printf("worker #%03d: %v", w, err)
|
||||
return entry{}, nil, true
|
||||
}
|
||||
e = entry{
|
||||
contentHash: hash(data),
|
||||
urlHash: urlHash,
|
||||
httpStatusCode: statusCode,
|
||||
sniffedMimeType: sniff(data),
|
||||
size: uint64(len(data)),
|
||||
url: u,
|
||||
}
|
||||
|
||||
filename := filepath.Join(*outdirFlag, "data", filepath.FromSlash(e.contentHash.Str(*datadirlevelsFlag)))
|
||||
if _, err := os.Stat(filename); os.IsNotExist(err) {
|
||||
log.Printf("worker #%03d: writing %q", w, urlString)
|
||||
os.MkdirAll(filepath.Dir(filename), 0755)
|
||||
if err := ioutil.WriteFile(filename, data, 0644); err != nil {
|
||||
log.Println(err)
|
||||
return entry{}, nil, true
|
||||
}
|
||||
}
|
||||
|
||||
if followHTML && looksLikeHTML(data) {
|
||||
log.Printf("worker #%03d: parsing %q", w, urlString)
|
||||
links = parseHTML(u, data)
|
||||
}
|
||||
return e, links, true
|
||||
}
|
||||
|
||||
func fetch(urlString string) (statusCode uint32, body []byte, retErr error) {
|
||||
client := &http.Client{
|
||||
Timeout: *httptimeoutFlag,
|
||||
}
|
||||
|
||||
res, err := client.Get(urlString)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
body, err = ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
return uint32(res.StatusCode), body, nil
|
||||
}
|
||||
|
||||
func parseHTML(u *url.URL, data []byte) (links []*url.URL) {
|
||||
z := html.NewTokenizer(bytes.NewReader(data))
|
||||
for {
|
||||
tt := z.Next()
|
||||
if tt == html.ErrorToken {
|
||||
break
|
||||
}
|
||||
if tt != html.StartTagToken && tt != html.SelfClosingTagToken {
|
||||
continue
|
||||
}
|
||||
tn, hasAttr := z.TagName()
|
||||
if len(tn) != 3 || string(tn) != "img" {
|
||||
continue
|
||||
}
|
||||
for hasAttr {
|
||||
key, val, moreAttr := z.TagAttr()
|
||||
if len(key) == 3 && string(key) == "src" {
|
||||
if v, err := url.Parse(string(val)); err == nil {
|
||||
links = append(links, u.ResolveReference(v))
|
||||
}
|
||||
}
|
||||
hasAttr = moreAttr
|
||||
}
|
||||
}
|
||||
return links
|
||||
}
|
||||
|
||||
var (
|
||||
prefixBM = []byte("BM")
|
||||
prefixGIF = []byte("GIF8")
|
||||
prefixJPEG = []byte("\xFF\xD8")
|
||||
prefixPNG = []byte("\x89PNG\r\n\x1A\n")
|
||||
prefixRIFF = []byte("RIFF")
|
||||
prefixTIFFBE = []byte("II\x2A\x00")
|
||||
prefixTIFFLE = []byte("MM\x00\x2A")
|
||||
prefixWEBP = []byte("WEBP")
|
||||
prefixZZZZ = []byte("\x00\x00\x00\x00")
|
||||
)
|
||||
|
||||
func sniff(b []byte) string {
|
||||
switch {
|
||||
case bytes.HasPrefix(b, prefixBM):
|
||||
if len(b) > 6 && bytes.HasPrefix(b[6:], prefixZZZZ) {
|
||||
return "image/bmp"
|
||||
}
|
||||
case bytes.HasPrefix(b, prefixGIF):
|
||||
return "image/gif"
|
||||
case bytes.HasPrefix(b, prefixJPEG):
|
||||
return "image/jpeg"
|
||||
case bytes.HasPrefix(b, prefixPNG):
|
||||
return "image/png"
|
||||
case bytes.HasPrefix(b, prefixRIFF):
|
||||
if len(b) > 8 && bytes.HasPrefix(b[8:], prefixWEBP) {
|
||||
return "image/webp"
|
||||
}
|
||||
case bytes.HasPrefix(b, prefixTIFFBE), bytes.HasPrefix(b, prefixTIFFLE):
|
||||
return "image/tiff"
|
||||
}
|
||||
|
||||
if looksLikeHTML(b) {
|
||||
return "text/html"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func looksLikeHTML(b []byte) bool {
|
||||
for len(b) > 0 {
|
||||
if b[0] > ' ' {
|
||||
return b[0] == '<'
|
||||
}
|
||||
b = b[1:]
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in new issue
Block a user