Add elapsed time logging (#62)

Author: man90es

scraper/scraper.go

@@ -45,7 +45,7 @@ func InitScraper() {
 
 	scraper.OnRequest(func(r *colly.Request) {
 		query := r.URL.Query()
-		for _, key := range []string{"taskHash", "taskType", "taskRegion", "taskRetries", "taskClient"} {
+		for _, key := range []string{"taskHash", "taskType", "taskRegion", "taskRetries", "taskClient", "taskAddedAt"} {
 			r.Ctx.Put(key, query.Get(key))
 			query.Del(key)
 		}
@@ -57,7 +57,9 @@ func InitScraper() {
 	})
 
 	scraper.OnResponse(func(r *colly.Response) {
-		logger.Info(fmt.Sprintf("Loaded %v", r.Request.URL))
+		parsedStartTime, _ := time.Parse(time.RFC3339, r.Request.Ctx.Get("taskAddedAt"))
+		elapsed := time.Since(parsedStartTime)
+		logger.Info(fmt.Sprintf("Loaded %v in %v", r.Request.URL, elapsed))
 	})
 
 	scraper.OnHTML("body", func(body *colly.HTMLElement) {

scraper/taskQueue.go

@@ -7,6 +7,7 @@ import (
 )
 
 type Task struct {
+	AddedAt  time.Time
 	ClientIP string
 	Hash     string
 	URL      string
@@ -36,8 +37,9 @@ func NewTaskQueue(bufferSize int) *TaskQueue {
 
 func (q *TaskQueue) AddTask(clientIP, hash, url string) (added bool) {
 	fullURL := utils.BuildRequest(url, map[string]string{
-		"taskClient": clientIP,
-		"taskHash":   hash,
+		"taskAddedAt": time.Now().Format(time.RFC3339),
+		"taskClient":  clientIP,
+		"taskHash":    hash,
 	})
 
 	q.mutex.Lock()