Skip to content

improve(bq_driver): Optimize Log macros for less response time #1624

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
NeerajDwivedii wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

improve(bq_driver): Optimize Log macros for less response time #1624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f1766d8
improve(bq_driver): Optimize Log macros for less response time
NeerajDwivedii Jul 31, 2026
2a0cf23
small change
NeerajDwivedii Jul 31, 2026
71579e2
small change
NeerajDwivedii Jul 31, 2026
072d377
small new change
NeerajDwivedii Jul 31, 2026
297fab2
new commit
NeerajDwivedii Jul 31, 2026
edfc05d
new changes
NeerajDwivedii Jul 31, 2026
33a912a
change
NeerajDwivedii Jul 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions google/cloud/odbc/bq_client_interface/datasets.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

#include "google/cloud/odbc/bq_client_interface/datasets.h"
#include "google/cloud/odbc/internal/commons.h"
#include "google/cloud/odbc/internal/status_record_or.h"
#include "google/cloud/bigquery/v2/minimal/internal/dataset_client.h"
#include "absl/log/log.h"
Expand All @@ -25,6 +26,8 @@ using ::google::cloud::bigquery_v2_minimal_internal::DatasetClient;
using ::google::cloud::bigquery_v2_minimal_internal::GetDatasetRequest;
using ::google::cloud::bigquery_v2_minimal_internal::ListDatasetsRequest;
using ::google::cloud::bigquery_v2_minimal_internal::ListFormatDataset;
using ::google::cloud::odbc_internal::LogLevel;
using ::google::cloud::odbc_internal::ShouldLog;
using google::cloud::odbc_internal::StatusRecord;
using google::cloud::odbc_internal::StatusRecordOr;

Expand All @@ -37,15 +40,16 @@ StatusRecordOr<Dataset> GetDataset(DatasetClient& dataset_client,
GetDatasetRequest request;
request.set_project_id(project_id);
request.set_dataset_id(dataset_id);
LOG(INFO) << "GetDataSet:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetDataSet:: Request body: " << request.DebugString("");

auto response = dataset_client.GetDataset(request, options);
if (!response.ok()) {
LOG(WARNING) << "GetDataSet:: Request failed: " << response.status();
return StatusRecordOr<Dataset>::ConvertFromStatusOr(response.status());
}
LOG(INFO) << "GetDataSet:: Response body: "
<< GetJsonRegResp<Dataset>(*response);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetDataSet:: Response body: " << GetJsonRegResp<Dataset>(*response);
return StatusRecordOr<Dataset>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand All @@ -58,7 +62,8 @@ StatusRecordOr<std::vector<ListFormatDataset>> ListAllDatasets(
ListDatasetsRequest request;
request.set_project_id(project_id);
request.set_all_datasets(true);
LOG(INFO) << "ListAllDatasets:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllDatasets:: Request body: " << request.DebugString("");
StreamRange<ListFormatDataset> datasets_response =
dataset_client.ListDatasets(request, options);

Expand All @@ -68,8 +73,9 @@ StatusRecordOr<std::vector<ListFormatDataset>> ListAllDatasets(
LOG(ERROR) << "ListAllDatasets:: " << dataset.status().message();
return StatusRecord::ConvertFrom(dataset.status());
}
LOG(INFO) << "ListAllDatasets:: Response body: "
<< GetJsonRegResp<ListFormatDataset>(*dataset);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllDatasets:: Response body: "
<< GetJsonRegResp<ListFormatDataset>(*dataset);
datasets.push_back(*dataset);
}

Expand All @@ -86,7 +92,8 @@ StatusRecordOr<std::vector<ListFormatDataset>> FilterDatasets(
request.set_project_id(project_id);
request.set_all_datasets(dataset_filter.all);
request.set_filter(dataset_filter.filter);
LOG(INFO) << "FilterDatasets:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "FilterDatasets:: Request body: " << request.DebugString("");

StreamRange<ListFormatDataset> datasets_response =
dataset_client.ListDatasets(request, options);
Expand All @@ -97,8 +104,9 @@ StatusRecordOr<std::vector<ListFormatDataset>> FilterDatasets(
LOG(ERROR) << "FilterDatasets:: " << dataset.status().message();
return StatusRecord::ConvertFrom(dataset.status());
}
LOG(INFO) << "FilterDatasets:: Response body: "
<< GetJsonRegResp<ListFormatDataset>(*dataset);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "FilterDatasets:: Response body: "
<< GetJsonRegResp<ListFormatDataset>(*dataset);
datasets.push_back(*dataset);
}

Expand Down
77 changes: 47 additions & 30 deletions google/cloud/odbc/bq_client_interface/jobs.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "google/cloud/odbc/bq_client_interface/jobs.h"
#include "google/cloud/odbc/bq_client_interface/datasets.h"
#include "google/cloud/odbc/bq_client_interface/utils.h"
#include "google/cloud/odbc/internal/commons.h"
#include "google/cloud/odbc/internal/status_record_or.h"
#include "google/cloud/bigquery/v2/minimal/internal/job_client.h"
#include "google/cloud/bigquery/v2/minimal/internal/job_request.h"
Expand All @@ -38,6 +39,8 @@ using ::google::cloud::bigquery_v2_minimal_internal::PostQueryResults;
using ::google::cloud::bigquery_v2_minimal_internal::Projection;
using ::google::cloud::bigquery_v2_minimal_internal::QueryRequest;
using ::google::cloud::internal::ExponentialBackoffPolicy;
using ::google::cloud::odbc_internal::LogLevel;
using ::google::cloud::odbc_internal::ShouldLog;
using google::cloud::odbc_internal::SQLStates;
using google::cloud::odbc_internal::StatusRecord;
using google::cloud::odbc_internal::StatusRecordOr;
Expand Down Expand Up @@ -119,7 +122,8 @@ StatusRecordOr<Job> GetJob(JobClient& job_client, std::string const& project_id,
get_job_request.set_project_id(project_id);
get_job_request.set_job_id(job_id);
get_job_request.set_location(location);
LOG(INFO) << "GetJob:: Request body: " << get_job_request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetJob:: Request body: " << get_job_request.DebugString("");
auto max_retries = options.get<MaxRetriesOption>();
auto response =
RetryLoop([&] { return job_client.GetJob(get_job_request, options); },
Expand All @@ -129,7 +133,8 @@ StatusRecordOr<Job> GetJob(JobClient& job_client, std::string const& project_id,
LOG(WARNING) << "GetJob:: Request failed: " << response.status();
return StatusRecordOr<Job>::ConvertFromStatusOr(response.status());
}
LOG(INFO) << "GetJob:: Response body: " << GetJsonRegResp<Job>(*response);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetJob:: Response body: " << GetJsonRegResp<Job>(*response);
return StatusRecordOr<Job>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand Down Expand Up @@ -157,7 +162,8 @@ StatusRecordOr<std::vector<ListFormatJob>> ListAllJobs(
request.set_all_users(false);
request.set_max_results(kMaxChildJobsResults);
request.set_projection(Projection::Full());
LOG(INFO) << "ListAllJobs:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllJobs:: Request body: " << request.DebugString("");
StreamRange<ListFormatJob> jobs_response =
job_client.ListJobs(request, options);

Expand All @@ -167,8 +173,9 @@ StatusRecordOr<std::vector<ListFormatJob>> ListAllJobs(
LOG(ERROR) << "ListAllJobs:: " << job.status().message();
return StatusRecord::ConvertFrom(job.status());
}
LOG(INFO) << "ListAllJobs:: Response body: "
<< GetJsonRegResp<ListFormatJob>(*job);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllJobs:: Response body: "
<< GetJsonRegResp<ListFormatJob>(*job);
jobs.push_back(*job);
}

Expand All @@ -186,7 +193,8 @@ StatusRecordOr<std::vector<ListFormatJob>> ListAllJobs(
request.set_all_users(false);
request.set_max_results(kMaxChildJobsResults);
request.set_projection(Projection::Full());
LOG(INFO) << "ListAllJobs:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllJobs:: Request body: " << request.DebugString("");

StreamRange<ListFormatJob> jobs_response =
job_client.ListJobs(request, options);
Expand All @@ -197,8 +205,9 @@ StatusRecordOr<std::vector<ListFormatJob>> ListAllJobs(
LOG(ERROR) << "ListAllJobs:: " << job.status().message();
return StatusRecord::ConvertFrom(job.status());
}
LOG(INFO) << "ListAllJobs:: Response body: "
<< GetJsonRegResp<ListFormatJob>(*job);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllJobs:: Response body: "
<< GetJsonRegResp<ListFormatJob>(*job);
jobs.push_back(*job);
}

Expand All @@ -219,8 +228,8 @@ StatusRecordOr<std::vector<ListFormatJob>> FilterJobs(
request.set_state_filter(job_filter.state_filter);
request.set_parent_job_id(job_filter.parent_job_id);
request.set_projection(job_filter.projection);

LOG(INFO) << "FilterJobs:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "FilterJobs:: Request body: " << request.DebugString("");
StreamRange<ListFormatJob> jobs_response =
job_client.ListJobs(request, options);

Expand All @@ -230,8 +239,9 @@ StatusRecordOr<std::vector<ListFormatJob>> FilterJobs(
LOG(ERROR) << "FilterJobs:: " << job.status().message();
return StatusRecord::ConvertFrom(job.status());
}
LOG(INFO) << "FilterJobs:: Response body: "
<< GetJsonRegResp<ListFormatJob>(*job);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "FilterJobs:: Response body: "
<< GetJsonRegResp<ListFormatJob>(*job);
jobs.push_back(*job);
}

Expand All @@ -248,8 +258,8 @@ StatusRecordOr<Job> InsertJob(JobClient& job_client,
request.set_project_id(project_id);
request.set_job(job);
request.set_json_filter_keys(CreateKeysToFilterOut(job));

LOG(INFO) << "InsertJob:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "InsertJob:: Request body: " << request.DebugString("");
auto max_retries = options.get<MaxRetriesOption>();
auto response =
RetryLoop([&] { return job_client.InsertJob(request, options); },
Expand All @@ -259,7 +269,8 @@ StatusRecordOr<Job> InsertJob(JobClient& job_client,
LOG(WARNING) << "InsertJob:: Request failed: " << response.status();
return StatusRecordOr<Job>::ConvertFromStatusOr(response.status());
}
LOG(INFO) << "InsertJob: Response body: " << GetJsonRegResp<Job>(*response);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "InsertJob: Response body: " << GetJsonRegResp<Job>(*response);
return StatusRecordOr<Job>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand All @@ -280,7 +291,8 @@ StatusRecordOr<Job> CancelJob(JobClient& job_client,
if (!location.empty()) {
request.set_location(location);
}
LOG(INFO) << "CancelJob:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "CancelJob:: Request body: " << request.DebugString("");
auto max_retries = options.get<MaxRetriesOption>();
auto response =
RetryLoop([&] { return job_client.CancelJob(request, options); },
Expand All @@ -290,7 +302,8 @@ StatusRecordOr<Job> CancelJob(JobClient& job_client,
LOG(WARNING) << "CancelJob:: Request failed: " << response.status();
return StatusRecordOr<Job>::ConvertFromStatusOr(response.status());
}
LOG(INFO) << "CancelJob:: Response body: " << GetJsonRegResp<Job>(*response);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "CancelJob:: Response body: " << GetJsonRegResp<Job>(*response);
return StatusRecordOr<Job>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand All @@ -305,8 +318,8 @@ StatusRecordOr<PostQueryResults> Query(JobClient& job_client,
post_query_request.set_project_id(project_id);
post_query_request.set_query_request(query_request);
post_query_request.set_json_filter_keys(CreateKeysToFilterOut(query_request));

LOG(INFO) << "Query:: Request body: " << post_query_request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "Query:: Request body: " << post_query_request.DebugString("");
auto max_retries = options.get<MaxRetriesOption>();
auto response =
RetryLoop([&] { return job_client.Query(post_query_request, options); },
Expand All @@ -317,8 +330,9 @@ StatusRecordOr<PostQueryResults> Query(JobClient& job_client,
return StatusRecordOr<PostQueryResults>::ConvertFromStatusOr(
response.status());
}
LOG(INFO) << "Query:: Response body: "
<< GetJsonRegResp<PostQueryResults>(*response);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "Query:: Response body: "
<< GetJsonRegResp<PostQueryResults>(*response);
return StatusRecordOr<PostQueryResults>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand Down Expand Up @@ -364,8 +378,9 @@ StatusRecordOr<GetQueryResults> GetAllQueryResults(
<< get_query_results_partial.status().message();
return StatusRecord::ConvertFrom(get_query_results_partial.status());
}
LOG(INFO) << "GetAllQueryResults::QueryResults:: Response body: "
<< get_query_results_partial->DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetAllQueryResults::QueryResults:: Response body: "
<< get_query_results_partial->DebugString("");
// If job_complete is false, there would be no rows and we should wait for
// job completion
if (!get_query_results_partial->job_complete &&
Expand All @@ -389,8 +404,9 @@ StatusRecordOr<GetQueryResults> GetAllQueryResults(
get_query_results_request.set_page_token(
get_query_results_partial->page_token);
}
LOG(INFO) << "GetAllQueryResults:: Request body: "
<< get_query_results_request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetAllQueryResults:: Request body: "
<< get_query_results_request.DebugString("");
return get_query_results;
}
#pragma clang attribute pop
Expand All @@ -411,9 +427,9 @@ StatusRecordOr<GetQueryResults> FilterQueryResults(
chrono_ms(query_results_filter.query_timeout_ms));
get_query_results_request.set_max_results(query_results_filter.max_results);
get_query_results_request.set_page_token(query_results_filter.page_token);

LOG(INFO) << "FilterQueryResults:: Request body: "
<< get_query_results_request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "FilterQueryResults:: Request body: "
<< get_query_results_request.DebugString("");
auto max_retries = options.get<MaxRetriesOption>();
auto response = RetryLoop(
[&] {
Expand All @@ -427,8 +443,9 @@ StatusRecordOr<GetQueryResults> FilterQueryResults(
return StatusRecordOr<GetQueryResults>::ConvertFromStatusOr(
response.status());
}
LOG(INFO) << "FilterQueryResults::QueryResults:: Response body: "
<< response->DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "FilterQueryResults::QueryResults:: Response body: "
<< response->DebugString("");
return StatusRecordOr<GetQueryResults>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand Down
8 changes: 6 additions & 2 deletions google/cloud/odbc/bq_client_interface/projects.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

#include "google/cloud/odbc/bq_client_interface/projects.h"
#include "google/cloud/odbc/bq_client_interface/datasets.h"
#include "google/cloud/odbc/internal/commons.h"
#include "google/cloud/odbc/internal/sql_state_constants.h"
#include "google/cloud/odbc/internal/status_record_or.h"
#include "google/cloud/resourcemanager/v3/projects.pb.h"
Expand All @@ -27,6 +28,8 @@ using ::google::cloud::Options;
using ::google::cloud::bigquery_v2_minimal_internal::ListProjectsRequest;
using ::google::cloud::bigquery_v2_minimal_internal::Project;
using ::google::cloud::bigquery_v2_minimal_internal::ProjectClient;
using ::google::cloud::odbc_internal::LogLevel;
using ::google::cloud::odbc_internal::ShouldLog;
using google::cloud::odbc_internal::StatusRecord;
using google::cloud::odbc_internal::StatusRecordOr;
using ::google::cloud::resourcemanager_v3::ProjectsClient;
Expand Down Expand Up @@ -70,8 +73,9 @@ StatusRecordOr<Project> ConvertFrom(
}
std::string s_numeric_id = rm_project.name().substr(index + 1);
bq_project.numeric_id = std::stoll(s_numeric_id);
LOG(INFO) << "ConvertFrom::Project:: Request body: "
<< GetJsonRegResp<Project>(bq_project);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ConvertFrom::Project:: Request body: "
<< GetJsonRegResp<Project>(bq_project);
return bq_project;
}

Expand Down
18 changes: 12 additions & 6 deletions google/cloud/odbc/bq_client_interface/tables.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include "google/cloud/odbc/bq_client_interface/tables.h"
#include "google/cloud/odbc/bq_client_interface/datasets.h"
#include "google/cloud/odbc/bq_client_interface/utils.h"
#include "google/cloud/odbc/internal/commons.h"
#include "google/cloud/odbc/internal/status_record_or.h"
#include "google/cloud/bigquery/v2/minimal/internal/table_client.h"
#include "absl/log/log.h"
Expand All @@ -27,6 +28,8 @@ using ::google::cloud::bigquery_v2_minimal_internal::ListFormatTable;
using ::google::cloud::bigquery_v2_minimal_internal::ListTablesRequest;
using ::google::cloud::bigquery_v2_minimal_internal::Table;
using ::google::cloud::bigquery_v2_minimal_internal::TableClient;
using ::google::cloud::odbc_internal::LogLevel;
using ::google::cloud::odbc_internal::ShouldLog;
using google::cloud::odbc_internal::StatusRecord;
using google::cloud::odbc_internal::StatusRecordOr;

Expand All @@ -45,8 +48,8 @@ StatusRecordOr<Table> GetTable(TableClient& table_client,
request.set_selected_fields(table_filter.selected_fields);
request.set_view(table_filter.view);

LOG(INFO) << "GetTable:: Request body: " << request.DebugString("");

LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetTable:: Request body: " << request.DebugString("");
auto max_retries = options.get<MaxRetriesOption>();
auto response =
RetryLoop([&] { return table_client.GetTable(request, options); },
Expand All @@ -56,7 +59,8 @@ StatusRecordOr<Table> GetTable(TableClient& table_client,
LOG(WARNING) << "GetTable:: Request failed: " << response.status();
return StatusRecordOr<Table>::ConvertFromStatusOr(response.status());
}
LOG(INFO) << "GetTable:: Response body: " << GetJsonRegResp<Table>(*response);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "GetTable:: Response body: " << GetJsonRegResp<Table>(*response);
return StatusRecordOr<Table>::ConvertFromStatusOr(*response);
}
#pragma clang attribute pop
Expand All @@ -70,7 +74,8 @@ StatusRecordOr<std::vector<ListFormatTable>> ListAllTables(
request.set_project_id(project_id);
request.set_dataset_id(dataset_id);

LOG(INFO) << "ListAllTables:: Request body: " << request.DebugString("");
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllTables:: Request body: " << request.DebugString("");
StreamRange<ListFormatTable> tables_response =
table_client.ListTables(request, options);

Expand All @@ -80,8 +85,9 @@ StatusRecordOr<std::vector<ListFormatTable>> ListAllTables(
LOG(ERROR) << "ListAllTables:: " << table.status().message();
return StatusRecord::ConvertFrom(table.status());
}
LOG(INFO) << "ListAllTables:: Response body: "
<< GetJsonRegResp<ListFormatTable>(*table);
LOG_IF(INFO, ShouldLog(LogLevel::kLogInfo))
<< "ListAllTables:: Response body: "
<< GetJsonRegResp<ListFormatTable>(*table);
tables.push_back(*table);
}

Expand Down
Loading
Loading