We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 8577b6c commit 0fef760Copy full SHA for 0fef760
1 file changed
scraper/scraper.go
@@ -33,11 +33,6 @@ func InitScraper() {
33
scraper.AllowURLRevisit = true
34
scraper.SetRequestTimeout(time.Minute / 2)
35
36
- scraper.Limit(&colly.LimitRule{
37
- Delay: time.Second,
38
- RandomDelay: 5 * time.Second,
39
- })
40
-
41
proxies := viper.GetStringSlice("proxy")
42
if p, err := proxy.RoundRobinProxySwitcher(proxies...); err == nil {
43
scraper.SetProxyFunc(p)
0 commit comments