googleapis · Anshu6250 · Jul 24, 2026 · Jul 27, 2026 · Jul 28, 2026 · Jul 28, 2026
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "google/cloud/odbc/bq_client_interface/datasets.h"
+#include "google/cloud/odbc/bq_client_interface/utils.h"
 #include "google/cloud/odbc/internal/status_record_or.h"
 #include "google/cloud/bigquery/v2/minimal/internal/dataset_client.h"
 #include "absl/log/log.h"
@@ -58,6 +59,7 @@ StatusRecordOr<std::vector<ListFormatDataset>> ListAllDatasets(
   ListDatasetsRequest request;
   request.set_project_id(project_id);
   request.set_all_datasets(true);
+  request.set_max_results(kMetadataPageSize);
   LOG(INFO) << "ListAllDatasets:: Request body: " << request.DebugString("");
   StreamRange<ListFormatDataset> datasets_response =
       dataset_client.ListDatasets(request, options);
@@ -86,6 +88,7 @@ StatusRecordOr<std::vector<ListFormatDataset>> FilterDatasets(
   request.set_project_id(project_id);
   request.set_all_datasets(dataset_filter.all);
   request.set_filter(dataset_filter.filter);
+  request.set_max_results(kMetadataPageSize);
   LOG(INFO) << "FilterDatasets:: Request body: " << request.DebugString("");
 
   StreamRange<ListFormatDataset> datasets_response =

@@ -14,6 +14,7 @@
 
 #include "google/cloud/odbc/bq_client_interface/projects.h"
 #include "google/cloud/odbc/bq_client_interface/datasets.h"
+#include "google/cloud/odbc/bq_client_interface/utils.h"
 #include "google/cloud/odbc/internal/sql_state_constants.h"
 #include "google/cloud/odbc/internal/status_record_or.h"
 #include "google/cloud/resourcemanager/v3/projects.pb.h"
@@ -92,6 +93,7 @@ bool IsProjectBQEnabled(std::string const& bq_project_id,
 StatusRecordOr<std::vector<Project>> ListAllProjects(
     ProjectClient& project_client, Options const& options) {
   ListProjectsRequest request;
+  request.set_max_results(kMetadataPageSize);
 
   StreamRange<Project> projects_response =
       project_client.ListProjects(request, options);
@@ -112,6 +114,7 @@ StatusRecordOr<Project> GetProject(ProjectClient& project_client,
                                    std::string const& project_id,
                                    Options const& options) {
   ListProjectsRequest request;
+  request.set_max_results(kMetadataPageSize);
 
   StreamRange<Project> projects_response =
       project_client.ListProjects(request, options);
@@ -252,6 +255,7 @@ StatusRecordOr<std::vector<Project>> FilterProjects(
     ProjectClient& project_client, std::vector<std::string> const& project_ids,
     Options const& options) {
   ListProjectsRequest request;
+  request.set_max_results(kMetadataPageSize);
 
   StreamRange<Project> projects_response =
       project_client.ListProjects(request, options);

@@ -69,6 +69,7 @@ StatusRecordOr<std::vector<ListFormatTable>> ListAllTables(
   ListTablesRequest request;
   request.set_project_id(project_id);
   request.set_dataset_id(dataset_id);
+  request.set_max_results(kMetadataPageSize);
 
   LOG(INFO) << "ListAllTables:: Request body: " << request.DebugString("");
   StreamRange<ListFormatTable> tables_response =

@@ -17,6 +17,7 @@
 
 #include "absl/log/log.h"
 #include <cctype>
+#include <cstdint>
 #include <iomanip>
 #include <sstream>
 #include <string>
@@ -27,6 +28,11 @@ struct MaxRetriesOption {
   using Type = int;
 };
 
+// Page size requested from paginated metadata list APIs (projects.list,
+// datasets.list, tables.list). When maxResults is unset the BigQuery REST API
+// returns small pages (50), which multiplies sequential HTTP round trips
+constexpr std::int32_t kMetadataPageSize = 1000;
+
 // URL-encodes a value for safe use as a single path segment.
 inline std::string UrlEncodeSegment(std::string const& value) {
   std::ostringstream os;

@@ -17,6 +17,7 @@
 #include "google/cloud/odbc/bq_driver/internal/trace_utils.h"
 #include "google/cloud/odbc/bq_driver/internal/utils.h"
 #include <algorithm>
+#include <optional>
 
 namespace google::cloud::odbc_bq_driver_internal {
 
@@ -54,6 +55,37 @@ std::string NormalizeRestTableType(std::string type) {
 }
 }  // namespace
 
+std::optional<std::string> LiteralFromOdbcPattern(std::string const& filter,
+                                                  SQLULEN metadata_id) {
+  if (metadata_id == SQL_TRUE) {
+    std::string literal = filter;
+    RTrim(literal);
+    return literal;
+  }
+  if (filter.empty()) {
+    return std::nullopt;
+  }
+  std::string literal;
+  literal.reserve(filter.size());
+  for (size_t i = 0; i < filter.size(); ++i) {
+    char c = filter[i];
+    if (c == '\\') {
+      // Escape: the next char is a literal (consume both). A lone trailing
+      // backslash is dropped, matching CastOdbcRegexToCppRegex().
+      if (i + 1 < filter.size()) {
+        literal.push_back(filter[i + 1]);
+        ++i;
+      }
+    } else if (c == '%' || c == '_') {
+      // A genuine wildcard: the filter must be expanded by enumeration.
+      return std::nullopt;
+    } else {
+      literal.push_back(c);
+    }
+  }
+  return literal;
+}
+
 StatusRecord ValidateInputParameters(
     const SQLCHAR* catalog_name, SQLSMALLINT catalog_name_len,
     const SQLCHAR* schema_name, SQLSMALLINT schema_name_len,
@@ -116,9 +148,19 @@ StatusRecordOr<std::vector<std::string>> GetFilteredDatasetIds(
   StatusRecordOr<std::vector<ListFormatDataset>> datasets =
       bq_client.FilterDatasets(project_id, filter, options);
   if (!datasets) {
-    LOG(ERROR) << "GetFilteredDatasetIds::FilterDatasets:: "
-               << datasets.GetStatusRecord().message;
-    return datasets.GetStatusRecord();
+    auto const& status = datasets.GetStatusRecord();
+    // A catalog passed as a literal (no wildcard) is used directly without
+    // first confirming it via projects.list, so an unknown or inaccessible
+    // project surfaces here. Treat "not found" as an empty catalog rather than
+    // failing the whole call: the pattern-based path simply would not have
+    // matched that project. Mirrors GetFilteredTables' 404 handling.
+    if (status.native_error_code == 404) {
+      LOG(WARNING) << "GetFilteredDatasetIds:: Skipping project not found: '"
+                   << project_id << "': " << status.message;
+      return std::vector<std::string>{};
+    }
+    LOG(ERROR) << "GetFilteredDatasetIds::FilterDatasets:: " << status.message;
+    return status;
   }
   for (auto const& dataset : *datasets) {
     if ((!metadata_id && datasets_filter == "%") ||
@@ -416,12 +458,15 @@ StatusRecordOr<ResultSet> GetResultSetForTables(
     std::string const& project_filter, std::string const& dataset_filter,
     std::string const& table_filter, std::string const& table_type_filter,
     SQLULEN metadata_id) {
-  // When SQL_ATTR_METADATA_ID is true, the catalog argument is an exact project
-  // identifier (not a pattern). Skip enumerating every accessible project via
-  // projects.list and use the name directly.
+  // When the catalog argument is an exact project identifier -- either because
+  // SQL_ATTR_METADATA_ID is true, or because the LIKE pattern contains no
+  // wildcard -- skip enumerating every accessible project via projects.list and
+  // use the name directly. projects.list has no server-side filter, so it
+  // returns every project the caller can see just to select one by name.
   std::vector<std::string> project_list;
-  if (metadata_id == SQL_TRUE) {
-    project_list = {project_filter};
+  if (auto project_literal =
+          LiteralFromOdbcPattern(project_filter, metadata_id)) {
+    project_list = {std::move(*project_literal)};
   } else {
     auto projects_status_record_or =
         GetFilteredProjectIds(bq_client, project_filter, metadata_id);
@@ -444,23 +489,51 @@ StatusRecordOr<ResultSet> GetResultSetForTables(
   };
   std::vector<TaskInput> tasks;
 
-  for (auto const& project_id : project_list) {
-    // When SQL_ATTR_METADATA_ID is true, the schema argument is an exact
-    // dataset identifier (not a pattern). Skip listing every dataset in the
-    // project via datasets.list
-    if (metadata_id == SQL_TRUE) {
-      tasks.push_back({project_id, dataset_filter});
-      continue;
+  std::shared_ptr<TraceOptions> trace_option = TraceOptions::GetTraceOption();
+  int max_threads = trace_option->max_threads;
+
+  // The schema argument is an exact dataset identifier when
+  // SQL_ATTR_METADATA_ID is true, or when the LIKE pattern contains no
+  // wildcard. Computed once: it does not depend on the project.
+  auto const dataset_literal =
+      LiteralFromOdbcPattern(dataset_filter, metadata_id);
+
+  if (dataset_literal) {
+    // For an exact dataset identifier, skip listing every dataset in the
+    // project via datasets.list and address the dataset directly.
+    for (auto const& project_id : project_list) {
+      tasks.push_back({project_id, *dataset_literal});
     }
-    auto datasets_status_record_or = GetFilteredDatasetIds(
-        bq_client, project_id, dataset_filter, metadata_id);
-    if (!datasets_status_record_or) {
-      LOG(ERROR) << "GetResultSetForTables::GetFilteredDatasetIds:: "
-                 << datasets_status_record_or.GetStatusRecord().message;
-      return datasets_status_record_or.GetStatusRecord();
+  } else {
+    // Enumerate datasets for each project in parallel. A serial loop here
+    // issues one datasets.list REST call per project, which dominates
+    // SQLTables latency when the catalog pattern matches many projects
+    // (e.g. catalog = "%").
+    auto dataset_task = [&](std::string const& project_id)
+        -> StatusRecordOr<std::vector<TaskInput>> {
+      auto datasets_status_record_or = GetFilteredDatasetIds(
+          bq_client, project_id, dataset_filter, metadata_id);
+      if (!datasets_status_record_or) {
+        LOG(ERROR) << "GetResultSetForTables::GetFilteredDatasetIds:: "
+                   << datasets_status_record_or.GetStatusRecord().message;
+        return datasets_status_record_or.GetStatusRecord();
+      }
+      std::vector<TaskInput> project_tasks;
+      project_tasks.reserve(datasets_status_record_or->size());
+      for (auto& dataset_id : *datasets_status_record_or) {
+        project_tasks.push_back({project_id, std::move(dataset_id)});
+      }
+      return project_tasks;
+    };
+    auto dataset_results_or =
+        ExecuteParallelTasks<std::string, std::vector<TaskInput>>(
+            max_threads, project_list, dataset_task);
+    if (!dataset_results_or) {
+      return dataset_results_or.GetStatusRecord();
     }
-    for (auto const& dataset_id : *datasets_status_record_or) {
-      tasks.push_back({project_id, dataset_id});
+    for (auto& project_tasks : *dataset_results_or) {
+      tasks.insert(tasks.end(), std::make_move_iterator(project_tasks.begin()),
+                   std::make_move_iterator(project_tasks.end()));
     }
   }
 
@@ -491,8 +564,6 @@ StatusRecordOr<ResultSet> GetResultSetForTables(
   };
 
   // 3. Execute tasks using the generic utility
-  std::shared_ptr<TraceOptions> trace_option = TraceOptions::GetTraceOption();
-  int max_threads = trace_option->max_threads;
   auto parallel_results_or = ExecuteParallelTasks<TaskInput, TaskResult>(
       max_threads, tasks, parallel_func);
 

@@ -18,6 +18,7 @@
 #include "google/cloud/odbc/bq_driver/internal/odbc_sql_execute_utils.h"
 #include "google/cloud/odbc/bq_driver/internal/odbc_stmt_handle.h"
 #include "google/cloud/odbc/internal/status_record_or.h"
+#include <optional>
 
 namespace google::cloud::odbc_bq_driver_internal {
 
@@ -68,6 +69,20 @@ odbc_internal::StatusRecordOr<std::vector<std::string>> GetFilteredDatasetIds(
     ODBCBQClient& bq_client, std::string const& project_id,
     std::string const& datasets_filter, SQLULEN metadata_id);
 
+// Returns the exact identifier a catalog/schema filter denotes, or nullopt when
+// the filter is a genuine search pattern that must be expanded by listing.
+//
+// When SQL_ATTR_METADATA_ID is SQL_TRUE the argument is an exact identifier and
+// is returned verbatim (right-trimmed). Otherwise it is an ODBC LIKE pattern
+// and is a literal only when it has no unescaped '%'/'_' metacharacter, in
+// which case the unescaped form is returned. Lets SQLTables skip a
+// projects.list / datasets.list round trip when a concrete project or dataset
+// name was passed -- projects.list in particular has no server-side filter and
+// enumerates every visible project. Empty patterns return nullopt to preserve
+// the existing (match-nothing) listing path.
+std::optional<std::string> LiteralFromOdbcPattern(std::string const& filter,
+                                                  SQLULEN metadata_id);
+
 // Construct a query to INFORMATION_SCHEMA.TABLES table depending on input
 // parameters. Populate 'named_query_params' with named parameters if needed.
 odbc_internal::StatusRecordOr<std::string> ConstructQuery(

@@ -106,6 +106,49 @@ TEST(ValidateInputParameters, SuccessAllnullsMetadatafalse) {
   EXPECT_TRUE(status.ok());
 }
 
+TEST(LiteralFromOdbcPattern, PlainNameIsLiteral) {
+  auto literal = LiteralFromOdbcPattern("kirltest", SQL_FALSE);
+  ASSERT_TRUE(literal.has_value());
+  EXPECT_EQ(*literal, "kirltest");
+}
+
+TEST(LiteralFromOdbcPattern, HyphensAreLiteral) {
+  auto literal = LiteralFromOdbcPattern("bigquery-devtools-drivers", SQL_FALSE);
+  ASSERT_TRUE(literal.has_value());
+  EXPECT_EQ(*literal, "bigquery-devtools-drivers");
+}
+
+TEST(LiteralFromOdbcPattern, PercentWildcardIsPattern) {
+  EXPECT_FALSE(LiteralFromOdbcPattern("%", SQL_FALSE).has_value());
+  EXPECT_FALSE(LiteralFromOdbcPattern("%timestamp%", SQL_FALSE).has_value());
+}
+
+TEST(LiteralFromOdbcPattern, UnderscoreWildcardIsPattern) {
+  // '_' is an ODBC single-character wildcard, so a name containing it must be
+  // expanded by listing rather than used as an exact identifier.
+  EXPECT_FALSE(
+      LiteralFromOdbcPattern("ODBC_TEST_DATASET", SQL_FALSE).has_value());
+}
+
+TEST(LiteralFromOdbcPattern, EscapedWildcardsAreLiteral) {
+  auto literal = LiteralFromOdbcPattern("my\\_dataset\\%", SQL_FALSE);
+  ASSERT_TRUE(literal.has_value());
+  EXPECT_EQ(*literal, "my_dataset%");
+}
+
+TEST(LiteralFromOdbcPattern, EmptyPatternIsNotLiteral) {
+  // Preserve the prior (match-nothing) listing path for empty filters.
+  EXPECT_FALSE(LiteralFromOdbcPattern("", SQL_FALSE).has_value());
+}
+
+TEST(LiteralFromOdbcPattern, MetadataIdTrueIsAlwaysLiteral) {
+  // With SQL_ATTR_METADATA_ID true, arguments are exact identifiers even when
+  // they contain characters that would otherwise be wildcards.
+  auto literal = LiteralFromOdbcPattern("ODBC_TEST_DATASET ", SQL_TRUE);
+  ASSERT_TRUE(literal.has_value());
+  EXPECT_EQ(*literal, "ODBC_TEST_DATASET");
+}
+
 TEST(ConstructQuery, ConstructWithTwoClausesMetadatafalse) {
   std::vector<QueryParameter> named_query_params;
 

@@ -283,7 +283,9 @@ TraceOptions::CreateTraceOptionsFile(
   int log_level = 0;
   int log_file_count;
   int log_file_size;
-  std::uint32_t max_threads = 8;  // default max_threads
+  // Must track kDefaultMaxThreads: a literal here silently overrides it
+  // whenever the driver config key has no MaxThreads value.
+  std::uint32_t max_threads = kDefaultMaxThreads;
   for (auto const& s : trace_sections) {
     if (s.first == kLogLevel && !s.second.empty()) {
       log_level = std::strtol(s.second.c_str(), nullptr, 10);

@@ -41,7 +41,13 @@ inline std::string const kLogPath = "LogPath";
 inline std::string const kLogFileCount = "LogFileCount";
 inline std::string const kLogFileSize = "LogFileSize";
 inline std::string const kMaxThreadsParam = "MaxThreads";
-inline std::uint32_t const kDefaultMaxThreads = 8;
+// Concurrency for the catalog metadata fan-out (SQLTables / SQLColumns issue
+// one datasets.list / tables.list REST call per project / dataset through
+// ExecuteParallelTasks). These threads spend essentially all their time waiting
+// on network I/O, not CPU, so full-catalog enumeration wall time is
+// ceil(calls / kDefaultMaxThreads) * per_call_latency and the previous value of
+// 8 bounded throughput well below what the service will serve.
+inline std::uint32_t const kDefaultMaxThreads = 16;
 inline std::string const kDefaultMaxFiles = "50";
 inline std::string const kDefaultMaxSize = "2000";
 

@@ -979,6 +979,19 @@ std::string CastOdbcRegexToCppRegex(std::string const& str) {
   return result;
 }
 
+std::string EscapeOdbcPattern(std::string const& identifier) {
+  std::string escaped;
+  // Worst case: every character needs a backslash.
+  escaped.reserve(identifier.size() * 2);
+  for (char c : identifier) {
+    if (c == '%' || c == '_' || c == '\\') {
+      escaped.push_back('\\');
+    }
+    escaped.push_back(c);
+  }
+  return escaped;
+}
+
 std::unique_ptr<re2::RE2> BuildRegex(std::string filter_pattern,
                                      SQLULEN metadata_id) {
   if (metadata_id == SQL_TRUE) {