network: support ADSL, configurable timout and retry

Author: Grok Compression

src/lib/core/grok.h

@@ -612,6 +612,17 @@ typedef struct _grk_stream_params
   /* 9 Requester pays (S3) */
   char request_payer[64]; /* e.g. "requester" for x-amz-request-payer header */
 
+  /* 10 User agent */
+  char user_agent[256]; /* custom User-Agent header string */
+
+  /* 11 Timeouts (seconds, 0 = use default) */
+  long timeout; /* overall request timeout (CURLOPT_TIMEOUT) */
+  long connect_timeout; /* connection timeout (CURLOPT_CONNECTTIMEOUT) */
+
+  /* 12 Retry configuration (0 = use default) */
+  uint32_t max_retry; /* maximum number of retries */
+  uint32_t retry_delay; /* delay between retries in seconds */
+
 } grk_stream_params;
 
 /**

src/lib/core/stream/StreamGenerator.cpp

@@ -151,17 +151,26 @@ IStream* StreamGenerator::createCurlFetchStream(void)
     auth.proxy_userpwd_ = streamParams_.proxy_userpwd;
   if(streamParams_.request_payer[0])
     auth.request_payer_ = streamParams_.request_payer;
+  if(streamParams_.user_agent[0])
+    auth.user_agent_ = streamParams_.user_agent;
+  auth.timeout_ = streamParams_.timeout;
+  auth.connect_timeout_ = streamParams_.connect_timeout;
+  auth.max_retry_ = streamParams_.max_retry;
+  auth.retry_delay_ = streamParams_.retry_delay;
   grklog.debug("StreamGenerator: s3_allow_insecure: streamParams=%d, auth=%d",
                (int)streamParams_.s3_allow_insecure, (int)auth.s3_allow_insecure_);
   std::string_view file{streamParams_.file};
   bool isS3 = file.starts_with("/vsis3/") || file.starts_with("/vsis3_streaming/");
   bool isAZ = file.starts_with("/vsiaz/");
+  bool isADLS = file.starts_with("/vsiadls/");
   bool isGS = file.starts_with("/vsigs/");
   CurlFetcher* fetcher;
   if(isS3)
     fetcher = new S3Fetcher();
   else if(isAZ)
     fetcher = new AZFetcher();
+  else if(isADLS)
+    fetcher = new ADLSFetcher();
   else if(isGS)
     fetcher = new GSFetcher();
   else

src/lib/core/stream/StreamGenerator.h

@@ -100,7 +100,8 @@ class StreamGenerator
         std::string_view file{streamParams_.file};
         bool isNetwork = file.starts_with("http://") || file.starts_with("https://") ||
                          file.starts_with("/vsis3/") || file.starts_with("/vsicurl/") ||
-                         file.starts_with("/vsiaz/") || file.starts_with("/vsigs/");
+                         file.starts_with("/vsiaz/") || file.starts_with("/vsiadls/") ||
+                         file.starts_with("/vsigs/");
         if(isNetwork && !streamParams_.read_fn)
           return createCurlFetchStream();
       }

src/lib/core/stream/fetchers/CurlFetcher.h

@@ -203,6 +203,10 @@ class CurlFetcher : public IFetcher
   void init(const std::string& path, const FetchAuth& auth) override
   {
     auth_ = auth;
+    if(auth_.max_retry_ > 0)
+      maxRetries_ = auth_.max_retry_;
+    if(auth_.retry_delay_ > 0)
+      retryDelayMs_ = auth_.retry_delay_ * 1000;
     parse(path);
     fetch_total_size();
   }
@@ -598,6 +602,14 @@ class CurlFetcher : public IFetcher
       if(EnvVarManager::get("GRK_CURL_PROXYAUTH"))
         curl_easy_setopt(curl, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
     }
+
+    // User agent
+    if(!auth_.user_agent_.empty())
+      curl_easy_setopt(curl, CURLOPT_USERAGENT, auth_.user_agent_.c_str());
+
+    // Timeouts
+    long connect_timeout = auth_.connect_timeout_ > 0 ? auth_.connect_timeout_ : 10L;
+    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, connect_timeout);
   }
 
   curl_slist* configureHeaders(const std::string& range)
@@ -685,7 +697,6 @@ class CurlFetcher : public IFetcher
       throw std::runtime_error("Failed to initialize CURL easy handle");
 
     curl_easy_setopt(curl, CURLOPT_URL, url_.c_str());
-    curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10L);
     curl_initiate_retry(curl);
     curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);
     curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callback);
@@ -860,7 +871,8 @@ class CurlFetcher : public IFetcher
 
   void curl_initiate_retry(CURL* curl)
   {
-    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L);
+    long timeout = auth_.timeout_ > 0 ? auth_.timeout_ : 30L;
+    curl_easy_setopt(curl, CURLOPT_TIMEOUT, timeout);
   }
 
   /**

src/lib/core/stream/fetchers/FetchCommon.h

@@ -68,6 +68,17 @@ struct FetchAuth
   // S3 requester pays
   std::string request_payer_; // e.g. "requester"
 
+  // User agent
+  std::string user_agent_;
+
+  // Timeouts (seconds, 0 = use default)
+  long timeout_ = 0;
+  long connect_timeout_ = 0;
+
+  // Retry configuration (0 = use default)
+  uint32_t max_retry_ = 0;
+  uint32_t retry_delay_ = 0;
+
   FetchAuth() = default;
   FetchAuth(const std::string& u, const std::string& p, const std::string& t, const std::string& h,
             const std::string& r = "", const std::string& st = "")