Skip to content

Commit 0fef760

Browse files
authored
Remove internal rate limiter (#60)
1 parent 8577b6c commit 0fef760

1 file changed

Lines changed: 0 additions & 5 deletions

File tree

scraper/scraper.go

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,6 @@ func InitScraper() {
3333
scraper.AllowURLRevisit = true
3434
scraper.SetRequestTimeout(time.Minute / 2)
3535

36-
scraper.Limit(&colly.LimitRule{
37-
Delay: time.Second,
38-
RandomDelay: 5 * time.Second,
39-
})
40-
4136
proxies := viper.GetStringSlice("proxy")
4237
if p, err := proxy.RoundRobinProxySwitcher(proxies...); err == nil {
4338
scraper.SetProxyFunc(p)

0 commit comments

Comments
 (0)