[feat] Add support for time ranges longer than one hour

Author: Evan Damon

cmd/logshare-cli/main.go

@@ -113,18 +113,44 @@ func run(conf *config) func(c *cli.Context) error {
 			if err != nil {
 				return errors.Wrap(err, "failed to fetch field names")
 			}
+		} else if conf.iterate {
+			// Iterate over the time range in 1h (3600s) blocks, making sure to
+			// stay within the request rate limit (1 request/5 seconds)
+			hourStart := conf.startTime
+			hourEnd := hourStart + 3600
+			var previousRequest time.Time
+			var requestGap time.Duration
+			for hourStart < conf.endTime {
+				if hourEnd > conf.endTime {
+					hourEnd = conf.endTime
+				}
+				requestGap = time.Since(previousRequest)
+				if requestGap.Seconds() < 5 {
+					time.Sleep(requestGap)
+				}
+				previousRequest = time.Now()
+				meta, err = client.GetFromTimestamp(
+					conf.zoneID, hourStart, hourEnd, conf.count)
+				if err != nil {
+					return errors.Wrap(err, "failed to fetch via timestamp")
+				}
+				log.Printf("HTTP status %d | %dms | %s",
+					meta.StatusCode, meta.Duration, meta.URL)
+				log.Printf("Retrieved %d logs", meta.Count)
+				hourStart = hourEnd
+				hourEnd += 3600
+			}
 		} else {
 			meta, err = client.GetFromTimestamp(
 				conf.zoneID, conf.startTime, conf.endTime, conf.count)
 			if err != nil {
 				return errors.Wrap(err, "failed to fetch via timestamp")
 			}
+			log.Printf("HTTP status %d | %dms | %s",
+				meta.StatusCode, meta.Duration, meta.URL)
+			log.Printf("Retrieved %d logs", meta.Count)
 		}
 
-		log.Printf("HTTP status %d | %dms | %s",
-			meta.StatusCode, meta.Duration, meta.URL)
-		log.Printf("Retrieved %d logs", meta.Count)
-
 		return nil
 	}
 }
@@ -143,6 +169,7 @@ func parseFlags(conf *config, c *cli.Context) error {
 	conf.listFields = c.Bool("list-fields")
 	conf.googleStorageBucket = c.String("google-storage-bucket")
 	conf.googleProjectID = c.String("google-project-id")
+	conf.iterate = c.Bool("iterate")
 
 	return conf.Validate()
 }
@@ -161,6 +188,7 @@ type config struct {
 	listFields          bool
 	googleStorageBucket string
 	googleProjectID     string
+	iterate             bool
 }
 
 func (conf *config) Validate() error {
@@ -181,6 +209,10 @@ func (conf *config) Validate() error {
 		return errors.New("Both google-storage-bucket and google-project-id must be provided to upload to Google Storage")
 	}
 
+	if conf.endTime > (conf.startTime+3600) && conf.iterate == false {
+		return errors.New("Time range too long; ranges longer than 1h0m0s require the --iterate option")
+	}
+
 	return nil
 }
 
@@ -246,4 +278,8 @@ var flags = []cli.Flag{
 		Name:  "google-project-id",
 		Usage: "Project ID of the Google Cloud Storage Bucket to upload logs to",
 	},
+	cli.BoolFlag{
+		Name:  "iterate",
+		Usage: "Iterate over time ranges longer than normally allowed by the Logpull Rest API",
+	},
 }