Merge branch 'feature/yg/meego-6739737753-add-debugger' into feat/assume_role_and_oidc_and_saml

Author: yikefan

SDK_Integration_zh.md

@@ -31,6 +31,8 @@
     - [自定义退避策略](#自定义退避策略)
 - [异常处理](#异常处理)
 - [Debug机制](#debug机制)
+  - [开启Debug模式](#开启debug模式)
+  - [设置Debug级别](#设置debug级别)
 - [指定日志Logger](#指定日志logger)
 
 # 集成SDK
@@ -478,6 +480,7 @@ except ApiException as e:
 ## 配置Http(s)代理
 
 ```python
+import volcenginesdkcore,volcenginesdkecs
 configuration = volcenginesdkcore.Configuration()
 configuration.ak = "Your AK"
 configuration.sk = "Your SK"
@@ -748,6 +751,8 @@ except Exception as e:
 
 为便于客户在处理请求时进行问题排查和调试，SDK 支持日志功能，并提供多种日志级别设置。客户可根据实际需求配置日志级别，获取详细的请求与响应信息，以提升排障效率和系统可 observability（可观测性）。
 
+## 开启Debug模式
+
 > **默认**  
 > * `debug` - `False`  
 
@@ -761,6 +766,39 @@ configuration.debug = True  # 开启debug模式
 volcenginesdkcore.Configuration.set_default(configuration)
 ```
 
+## 设置debug级别
+默认情况下开启debug日志后，会输出所有的debug日志；为了按需输出日志，可以调用`configuration.log_level`进行以下设置：
+
+```python
+import volcenginesdkcore
+from volcenginesdkcore.observability.debugger import LogLevel
+configuration = volcenginesdkcore.Configuration()
+configuration.ak = "Your AK"
+configuration.sk = "Your SK"
+configuration.debug = True  # 开启debug模式
+configuration.log_level = LogLevel.LOG_DEBUG_WITH_CONFIG.mask | LogLevel.LOG_DEBUG_WITH_REQUEST.mask | LogLevel.LOG_DEBUG_WITH_RESPONSE.mask
+volcenginesdkcore.Configuration.set_default(configuration)
+
+```
+
+**支持的日志级别**
+
+
+| 枚举项                              | 父级日志(同时打印父级日志) | 打印的内容                                |
+| -------------------------------- |-----|--------------------------------------|
+| `LOG_DEBUG_WITH_REQUEST`         | —   | 请求行与基础请求信息：`HTTP方法`、`URL（含查询参数）`、`请求头` |
+| `LOG_DEBUG_WITH_REQUEST_BODY`    | `LOG_DEBUG_WITH_REQUEST` | `请求体`                                |
+| `LOG_DEBUG_WITH_REQUEST_ID`      | `LOG_DEBUG_WITH_REQUEST` | `RequestId`                          |
+| `LOG_DEBUG_WITH_RESPONSE`        | `LOG_DEBUG_WITH_REQUEST` | `响应状态码` `响应头`                        |
+| `LOG_DEBUG_WITH_RESPONSE_BODY`   | `LOG_DEBUG_WITH_RESPONSE` | `响应体`                                |
+| `LOG_DEBUG_WITH_SIGNING`         | `LOG_DEBUG_WITH_REQUEST` | `签名过程`                               |
+| `LOG_DEBUG_WITH_ENDPOINT`        | `LOG_DEBUG_WITH_REQUEST` | ` Endpoint 选择过程`                     |
+| `LOG_DEBUG_WITH_REQUEST_RETRIES` | `LOG_DEBUG_WITH_REQUEST` | `重试信息`                               |
+| `LOG_DEBUG_WITH_CONFIG`          | `LOG_DEBUG_WITH_REQUEST` | `关键配置信息`                             |
+| `LOG_DEBUG_ALL`                  |   —     | `包含上面所有信息`                           |
+
+
+
 # 指定日志Logger
 
 > **默认**  

volcenginesdkcore/api_client.py

@@ -3,7 +3,6 @@
 from __future__ import absolute_import
 
 import datetime
-import logging
 from multiprocessing.pool import ThreadPool
 from time import sleep
 
@@ -17,9 +16,7 @@
 from volcenginesdkcore.interceptor import DeserializedResponseInterceptor
 from volcenginesdkcore.interceptor import InterceptorChain, InterceptorContext
 from volcenginesdkcore.interceptor import Request, Response
-
-logger = logging.getLogger(__name__)
-
+from volcenginesdkcore.observability.debugger import sdk_core_logger, LogLevel
 
 class ApiClient(object):
     """Generic API client for Swagger client library builds.
@@ -126,6 +123,8 @@ def __call_api(
 
         interceptor_context = self.interceptor_chain.execute_request(interceptor_context)
 
+        self._log_config(interceptor_context.get_request())
+
         retry_count = 0
         response_data = None
         retry_err = None
@@ -145,7 +144,7 @@ def __call_api(
                     _request_timeout=interceptor_context.request.request_timeout)
                 self.last_response = response_data
             except Exception as e:
-                logger.warning("request error: {}".format(e))
+                sdk_core_logger.warning("request error: {}".format(e))
                 retry_err = e
                 if retry_count >= num_max_retries:
                     raise e
@@ -171,7 +170,7 @@ def request_retry(self, response_data, retry_count, retry_err, retryer):
             delay = retryer.get_backoff_delay(retry_count)
             sleep(delay / 1000)
             if self.configuration.debug:
-                logger.debug(
+                sdk_core_logger.debug_config(
                     "retry backoff strategy:%s, retry condition: %s, max retry count:%d, current retry count: %d, retry delay(ms):%f",
                     type(retryer.backoff_strategy).__name__, type(retryer.retry_condition).__name__,
                     retryer.num_max_retries, retry_count + 1, delay)
@@ -331,6 +330,74 @@ def select_header_content_type(self, content_types):
         else:
             return content_types[0]
 
+    def _log_config(self, request):
+
+        if not sdk_core_logger.is_enabled_for_loglevel(LogLevel.LOG_DEBUG_WITH_CONFIG):
+            return
+
+        sb = []
+
+        sb.append("SDK Version        : ")
+        sb.append(self.user_agent + "\n")
+
+        # 连接池配置
+        sb.append("[Connection Pool]" + "\n")
+        sb.append("  Number of pools           : ")
+        sb.append(str(self.configuration.num_pools) + "\n")
+        sb.append("  Connection pool maxsize   : ")
+        sb.append(str(self.configuration.connection_pool_maxsize) + "\n")
+
+        # SSL设置
+        sb.append("[Scheme]" + "\n")
+        sb.append("  Scheme      : ")
+        sb.append(str(request.scheme) + "\n")
+
+        # 代理设置（隐藏部分信息避免敏感泄露）
+        sb.append("[Proxy]" + "\n");
+        sb.append("  HTTP Proxy       : ")
+        sb.append(str(self.configuration.http_proxy) + "\n")
+        sb.append("  HTTPS Proxy      : ")
+        sb.append(str(self.configuration.https_proxy) + "\n")
+
+        # 超时设置
+        sb.append("[Timeout]" + "\n");
+        sb.append("  Connect Timeout(ms)  : ")
+        sb.append(str(self.configuration.connect_timeout) + "\n")
+        sb.append("  Read Timeout(ms)     : ")
+        sb.append(str(self.configuration.read_timeout) + "\n")
+
+        # 重试设置
+        sb.append("[Retry]" + "\n");
+        sb.append("  Auto Retry       : ")
+        sb.append(str(request.auto_retry) + "\n")
+        if request.auto_retry and request.retryer is not None:
+            sb.append("  Max Retries      : ")
+            sb.append(str(request.retryer.num_max_retries) + "\n")
+            sb.append("  Min Delay (ms)   : ")
+            sb.append(str(request.retryer.backoff_strategy.min_retry_delay_ms) + "\n")
+            sb.append("  Max Delay (ms)   : ")
+            sb.append(str(request.retryer.backoff_strategy.max_retry_delay_ms) + "\n")
+            sb.append("  Retry Condition  : ")
+            sb.append(type(request.retryer.retry_condition).__name__ if request.retryer.retry_condition is not None else "None" + "\n")
+            sb.append("  Backoff Strategy : ")
+            sb.append(type(request.retryer.backoff_strategy).__name__ if request.retryer.backoff_strategy is not None else "None" + "\n")
+            sb.append("  Retry ErrorCodes : ")
+            sb.append(str(request.retryer.retry_condition.retry_error_codes) + "\n")
+
+        # EndpointResolver设置
+        sb.append("[Endpoint Resolver]" + "\n")
+        sb.append("  Region           : ")
+        sb.append(str(request.region) + "\n")
+        sb.append("  Endpoint         : ")
+        sb.append(str(request.host) + "\n")
+        sb.append("  Use DualStack    : ")
+        sb.append(str(request.use_dual_stack) + "\n")
+        sb.append("  Bootstrap Region : ")
+        sb.append(str(request.custom_bootstrap_region) + "\n")
+        sb.append("  Resolver Class   : ")
+        sb.append(type(request.endpoint_provider).__name__ if request.endpoint_provider is not None else "None" + "\n")
+
+        sdk_core_logger.debug_config("".join(sb))
 
 def metadata(self):
     return self._metadata

volcenginesdkcore/configuration.py

@@ -12,6 +12,7 @@
 from six.moves import http_client as httplib
 
 from volcenginesdkcore.endpoint import DefaultEndpointProvider
+from volcenginesdkcore.observability.debugger import sdk_core_logger
 from volcenginesdkcore.retryer.retryer import DEFAULT_RETRYER
 
 
@@ -87,7 +88,7 @@ def __init__(self):
 
         # Logging Settings
         self.logger = {}
-        self.logger["package_logger"] = logging.getLogger("volcenginesdkcore")
+        self.logger["package_logger"] = sdk_core_logger
         self.logger["urllib3_logger"] = logging.getLogger("urllib3")
         # Log format
         self.logger_format = '%(asctime)s %(levelname)s %(message)s'
@@ -209,14 +210,14 @@ def debug(self, value):
             for _, logger in six.iteritems(self.logger):
                 logger.setLevel(logging.DEBUG)
             # turn on httplib debug
-            httplib.HTTPConnection.debuglevel = 1
+            #httplib.HTTPConnection.debuglevel = 1
         else:
             # if debug status is False, turn off debug logging,
             # setting log level to default `logging.WARNING`
             for _, logger in six.iteritems(self.logger):
                 logger.setLevel(logging.WARNING)
             # turn off httplib debug
-            httplib.HTTPConnection.debuglevel = 0
+            #httplib.HTTPConnection.debuglevel = 0
 
     @property
     def logger_format(self):
@@ -240,6 +241,9 @@ def logger_format(self, value):
         """
         self.__logger_format = value
         self.logger_formatter = logging.Formatter(self.__logger_format)
+        for _, lg in six.iteritems(self.logger):
+            for h in lg.handlers:
+                h.setFormatter(self.logger_formatter)
 
     def get_api_key_with_prefix(self, identifier):
         """Gets API key (with prefix if set).
@@ -335,3 +339,11 @@ def max_retry_delay_ms(self, value):
     @property
     def retryer(self):
         return self.__retryer
+
+    @property
+    def log_level(self):
+        return sdk_core_logger.get_log_level()
+
+    @log_level.setter
+    def log_level(self, value):
+        sdk_core_logger.set_log_level(value)

volcenginesdkcore/endpoint/providers/default_provider.py

@@ -3,6 +3,7 @@
 import warnings
 
 from volcenginesdkcore.endpoint.endpoint_provider import EndpointProvider, ResolvedEndpoint
+from volcenginesdkcore.observability.debugger import sdk_core_logger
 
 open_prefix = 'open'
 endpoint_suffix = '.volcengineapi.com'
@@ -29,14 +30,36 @@ def __standardize_domain_service_code(self):
         return self.service.lower().replace('_', '-')
 
     def get_endpoint_for(self, region, suffix=endpoint_suffix):
+        sdk_core_logger.debug_endpoint(
+            "get_endpoint_for start: service=%s, region=%s, suffix=%s",
+            self.service, region, suffix
+        )
         if self.is_global:
             if self.global_endpoint:
+                sdk_core_logger.debug_endpoint(
+                    "use global endpoint: service=%s, endpoint=%s",
+                    self.service, self.global_endpoint
+                )
                 return self.global_endpoint
-            return self.__standardize_domain_service_code + suffix
+
+            endpoint = self.__standardize_domain_service_code + suffix
+            sdk_core_logger.debug_endpoint(
+                "build global endpoint from service code: %s", endpoint
+            )
+            return endpoint
         if region in self.region_endpoint_map:
-            return self.region_endpoint_map[region]
+            endpoint = self.region_endpoint_map[region]
+            sdk_core_logger.debug_endpoint(
+                "use region endpoint from map: service=%s, region=%s, endpoint=%s",
+                self.service, region, endpoint
+            )
+            return endpoint
 
-        return self.__standardize_domain_service_code + '.' + region + suffix
+        endpoint = self.__standardize_domain_service_code + '.' + region + suffix
+        sdk_core_logger.debug_endpoint(
+            "build region endpoint by default rule: %s", endpoint
+        )
+        return endpoint
 
 
 default_endpoint = {
@@ -483,9 +506,21 @@ def __init__(self, custom_endpoints=None):
         self.custom_endpoints = custom_endpoints or {}
 
     def get_default_endpoint(self, service, region, suffix=endpoint_suffix):
+        sdk_core_logger.debug_endpoint(
+            "get_default_endpoint: service=%s, region=%s, suffix=%s",
+            service, region, suffix
+        )
         if service in default_endpoint:
             e = default_endpoint[service]
-            return e.get_endpoint_for(region, suffix)
+            endpoint = e.get_endpoint_for(region, suffix)
+            sdk_core_logger.debug_endpoint(
+                "resolved default endpoint: service=%s, endpoint=%s",
+                service, endpoint
+            )
+            return endpoint
+        sdk_core_logger.debug_endpoint(
+            "service not found in default_endpoint, use fallback=%s", fallback_endpoint
+        )
         return fallback_endpoint
 
     def __in_bootstrap_region_list(self, region, custom_bootstrap_region):
@@ -525,20 +560,36 @@ def __has_enabled_dualstack(use_dual_stack):
         return use_dual_stack
 
     def endpoint_for(self, service, region, custom_bootstrap_region=None, use_dual_stack=None, **kwargs):
+        sdk_core_logger.debug_endpoint(
+            "endpoint_for called: service=%s, region=%s, custom_bootstrap_region=%s, use_dual_stack=%s",
+            service, region, custom_bootstrap_region, use_dual_stack
+        )
         if service in self.custom_endpoints:
             conf = self.custom_endpoints[service]
             host = conf.get_endpoint_for(region)
+            sdk_core_logger.debug_endpoint(
+                "use custom endpoint: service=%s, region=%s, host=%s",
+                service, region, host
+            )
             return ResolvedEndpoint(host)
 
         if custom_bootstrap_region is None:
             custom_bootstrap_region = {}
 
         if not self.__in_bootstrap_region_list(region, custom_bootstrap_region):
+            sdk_core_logger.debug_endpoint(
+                "region=%s not in bootstrap list, fallback=%s",
+                region, fallback_endpoint
+            )
             return ResolvedEndpoint(fallback_endpoint)
 
         suffix = dualstack_endpoint_suffix if self.__has_enabled_dualstack(use_dual_stack) else endpoint_suffix
         host = self.get_default_endpoint(service=service, region=region, suffix=suffix)
 
+        sdk_core_logger.debug_endpoint(
+            "final resolved endpoint: service=%s, region=%s, host=%s",
+            service, region, host
+        )
         return ResolvedEndpoint(host)
 
 
@@ -547,4 +598,8 @@ def __init__(self, host):
         self.host = host
 
     def endpoint_for(self, service, region, **kwargs):
+        sdk_core_logger.debug_endpoint(
+            "HostEndpointProvider.endpoint_for: service=%s, region=%s, host=%s",
+            service, region, self.host
+        )
         return ResolvedEndpoint(self.host)