diff --git a/.circleci/config.yml b/.circleci/config.yml index a214c35c..0fd0a474 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,6 +74,7 @@ commands: steps: - store_test_results: path: test-results + run_sonarqube: steps: - attach_workspace: @@ -103,8 +104,8 @@ commands: -Dsonar.login="${SONARQUBE_LOGIN}" \ -Dsonar.branch.name="${CIRCLE_BRANCH}" fi - store_artifacts: - path: htmlcov + - store_artifacts: + path: htmlcov store-coverage-report: steps: @@ -114,7 +115,7 @@ commands: jobs: python38: docker: - - image: cimg/python:3.8.17 + - image: cimg/python:3.8 - image: cimg/postgres:9.6.24 environment: POSTGRES_USER: root @@ -138,7 +139,7 @@ jobs: python39: docker: - - image: cimg/python:3.9.17 + - image: cimg/python:3.9 - image: cimg/postgres:9.6.24 environment: POSTGRES_USER: root @@ -162,7 +163,7 @@ jobs: python310: docker: - - image: cimg/python:3.10.12 + - image: cimg/python:3.10 - image: cimg/postgres:9.6.24 environment: POSTGRES_USER: root @@ -187,7 +188,7 @@ jobs: python311: docker: - - image: cimg/python:3.11.4 + - image: cimg/python:3.11 - image: cimg/postgres:9.6.24 environment: POSTGRES_USER: root @@ -212,7 +213,7 @@ jobs: python312: docker: - - image: cimg/python:3.12.0 + - image: cimg/python:3.12 - image: cimg/postgres:9.6.24 environment: POSTGRES_USER: root @@ -235,9 +236,34 @@ jobs: - store-pytest-results - store-coverage-report + python313: + docker: + - image: python:3.13.0b2-bookworm + - image: cimg/postgres:9.6.24 + environment: + POSTGRES_USER: root + POSTGRES_PASSWORD: passw0rd + POSTGRES_DB: instana_test_db + - image: cimg/mariadb:10.11.2 + environment: + MYSQL_ROOT_PASSWORD: passw0rd + MYSQL_DATABASE: instana_test_db + - image: cimg/redis:5.0.14 + - image: rabbitmq:3.9.13 + - image: mongo:4.2.3 + - image: vanmoof/pubsub-emulator + working_directory: ~/repo + steps: + - checkout + - pip-install-deps: + requirements: "tests/requirements-313.txt" + - run-tests-with-coverage-report + - store-pytest-results + - store-coverage-report + py39couchbase: docker: - - image: cimg/python:3.9.17 + - image: cimg/python:3.9 - image: couchbase/server-sandbox:5.5.0 working_directory: ~/repo steps: @@ -253,7 +279,7 @@ jobs: py39cassandra: docker: - - image: cimg/python:3.9.17 + - image: cimg/python:3.9 - image: cassandra:3.11 environment: MAX_HEAP_SIZE: 2048m @@ -303,9 +329,10 @@ workflows: - python310 - python311 - python312 - - py39cassandra - - py39couchbase - - py39gevent_starlette + - python313 + # - py39cassandra + # - py39couchbase + # - py39gevent_starlette - final_job: requires: - python38 @@ -313,6 +340,10 @@ workflows: - python310 - python311 - python312 - - py39cassandra - - py39couchbase - - py39gevent_starlette + - python313 + # - py39cassandra + # - py39couchbase + # - py39gevent_starlette + filters: + branches: + only: master diff --git a/.tekton/.currency/resources/table.json b/.tekton/.currency/resources/table.json index 05b4f058..07658f41 100644 --- a/.tekton/.currency/resources/table.json +++ b/.tekton/.currency/resources/table.json @@ -2,7 +2,7 @@ "table": [ { "Package name": "ASGI", - "Support Policy": "0-day", + "Support Policy": "30-days", "Beta version": "No", "Last Supported Version": "3.0", "Cloud Native": "No" @@ -21,13 +21,13 @@ }, { "Package name": "FastAPI", - "Support Policy": "0-day", + "Support Policy": "30-days", "Beta version": "No", "Cloud Native": "No" }, { "Package name": "Flask", - "Support Policy": "0-day", + "Support Policy": "30-days", "Beta version": "No", "Cloud Native": "No" }, @@ -66,7 +66,7 @@ { "Package name": "WSGI", "Support Policy": "0-day", - "Beta version": "No", + "Beta version": "Yes", "Last Supported Version": "1.0.1", "Cloud Native": "No" }, @@ -85,7 +85,7 @@ }, { "Package name": "Boto3", - "Support Policy": "0-day", + "Support Policy": "30-days", "Beta version": "No", "Cloud Native": "Yes" }, @@ -145,7 +145,7 @@ }, { "Package name": "Requests", - "Support Policy": "0-day", + "Support Policy": "30-days", "Beta version": "No", "Cloud Native": "Yes" }, @@ -157,7 +157,7 @@ }, { "Package name": "Urllib3", - "Support Policy": "0-day", + "Support Policy": "30-days", "Beta version": "No", "Cloud Native": "No" } diff --git a/.tekton/pipeline.yaml b/.tekton/pipeline.yaml index 953a364d..17191508 100644 --- a/.tekton/pipeline.yaml +++ b/.tekton/pipeline.yaml @@ -25,16 +25,18 @@ spec: params: - name: imageDigest value: - # 3.8.18-bookworm - - "sha256:625008535504ab68868ca06d1bdd868dee92a9878d5b55fc240af7ceb38b7183" - # 3.9.18-bookworm - - "sha256:530d4ba717be787c0e2d011aa107edac6d721f8c06fe6d44708d4aa5e9bc5ec9" - # 3.10.13-bookworm - - "sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc" - # 3.11.8-bookworm - - "sha256:72afb375030b13c8c9cb72ba1d8c410f25307c2dbbd7d59f9c6ccea5cb152ff9" - # 3.12.2-bookworm - - "sha256:35eff340c0acd837b7962f77ee4b8869385dd6fe7d3928375a08f0a3bdd18beb" + # 3.8.19-bookworm + - "sha256:4d3590657cf443010b58ae94a09c59505a750744ed70d2028b35dac101df5e3a" + # 3.9.19-bookworm + - "sha256:e298e2e898691a938073f670dac8ef1a551c83344b67b5d8e32d1fbc8e0b57f8" + # 3.10.14-bookworm + - "sha256:c0352a2c64efe4cc08b198e90b97ed7e08897518c4bee99647e3eaf676e84951" + # 3.11.9-bookworm + - "sha256:0c2928128a96e544a1ee248e50ee8ecbe840bf48ef5a49065812e3d06b6e1bcc" + # 3.12.4-bookworm + - "sha256:83f5f8714b6881d3e0e91023d9fe9e43aa6ad5a04e9f9a94ee180b18b021c72a" + # 3.13.0b2-bookworm + - "sha256:6502f02f8a02313f582928ec7159623b54d7c3d627a7e355ca46f4aace406a6a" taskRef: name: python-tracer-unittest-default-task workspaces: @@ -47,8 +49,8 @@ spec: params: - name: imageDigest value: - # 3.9.18-bookworm - - "sha256:530d4ba717be787c0e2d011aa107edac6d721f8c06fe6d44708d4aa5e9bc5ec9" + # 3.9.19-bookworm + - "sha256:e298e2e898691a938073f670dac8ef1a551c83344b67b5d8e32d1fbc8e0b57f8" taskRef: name: python-tracer-unittest-cassandra-task workspaces: @@ -61,8 +63,8 @@ spec: params: - name: imageDigest value: - # 3.9.18-bookworm - - "sha256:530d4ba717be787c0e2d011aa107edac6d721f8c06fe6d44708d4aa5e9bc5ec9" + # 3.9.19-bookworm + - "sha256:e298e2e898691a938073f670dac8ef1a551c83344b67b5d8e32d1fbc8e0b57f8" taskRef: name: python-tracer-unittest-couchbase-task workspaces: diff --git a/.tekton/python-tracer-prepuller.yaml b/.tekton/python-tracer-prepuller.yaml index 3b66ef71..29b14679 100644 --- a/.tekton/python-tracer-prepuller.yaml +++ b/.tekton/python-tracer-prepuller.yaml @@ -49,41 +49,29 @@ spec: # postgres:16.2-bookworm image: postgres@sha256:3bfb87432e26badf72d727a0c5f5bb7b81438cd9baec5be8531c70a42b07adc6 command: ["sh", "-c", "'true'"] - - name: prepuller-30 - # 3.0.6-bullseye - image: ruby@sha256:3166618469ad8a3190d80f43b322818fafb4bfac0b4882255eee3346af2a0a35 - command: ["sh", "-c", "'true'"] - - name: prepuller-31 - # 3.1.4-bookworm - image: ruby@sha256:ec69284bcbceb0a23ffc070ef2e0e8eb0fe495c20efbd51846b103338c3da1e4 - command: ["sh", "-c", "'true'"] - - name: prepuller-32 - # 3.2.3-bookworm - image: ruby@sha256:007d2edd515f9cfc8c5c571486aca4fc4a25c903d004decee302961bb8c636ed - command: ["sh", "-c", "'true'"] - - name: prepuller-33 - # 3.3.1-bookworm - image: ruby@sha256:5cf0004738f54bd67e4c4316394208ca38a6726eda7a1b0586d95601aad86e5d - command: ["sh", "-c", "'true'"] - name: prepuller-38 - # 3.8.18-bookworm - image: "python@sha256:625008535504ab68868ca06d1bdd868dee92a9878d5b55fc240af7ceb38b7183" + # 3.8.19-bookworm + image: "python@sha256:4d3590657cf443010b58ae94a09c59505a750744ed70d2028b35dac101df5e3a" command: ["sh", "-c", "'true'"] - name: prepuller-39 - # 3.9.18-bookworm - image: "python@sha256:530d4ba717be787c0e2d011aa107edac6d721f8c06fe6d44708d4aa5e9bc5ec9" + # 3.9.19-bookworm + image: "python@sha256:e298e2e898691a938073f670dac8ef1a551c83344b67b5d8e32d1fbc8e0b57f8" command: ["sh", "-c", "'true'"] - name: prepuller-310 - # 3.10.13-bookworm - image: "python@sha256:c970ff53939772f47b0672e380328afb50d8fd1c0568ed4f82c22effc54244fc" + # 3.10.14-bookworm + image: "python@sha256:c0352a2c64efe4cc08b198e90b97ed7e08897518c4bee99647e3eaf676e84951" command: ["sh", "-c", "'true'"] - name: prepuller-311 - # 3.11.8-bookworm - image: "python@sha256:72afb375030b13c8c9cb72ba1d8c410f25307c2dbbd7d59f9c6ccea5cb152ff9" + # 3.11.9-bookworm + image: "python@sha256:0c2928128a96e544a1ee248e50ee8ecbe840bf48ef5a49065812e3d06b6e1bcc" command: ["sh", "-c", "'true'"] - name: prepuller-312 - # 3.12.2-bookworm - image: "python@sha256:35eff340c0acd837b7962f77ee4b8869385dd6fe7d3928375a08f0a3bdd18beb" + # 3.12.4-bookworm + image: "python@sha256:83f5f8714b6881d3e0e91023d9fe9e43aa6ad5a04e9f9a94ee180b18b021c72a" + command: ["sh", "-c", "'true'"] + - name: prepuller-313 + # 3.13.0b2-bookworm + image: "python@sha256:6502f02f8a02313f582928ec7159623b54d7c3d627a7e355ca46f4aace406a6a" command: ["sh", "-c", "'true'"] # Use the pause container to ensure the Pod goes into a `Running` phase diff --git a/.tekton/run_unittests.sh b/.tekton/run_unittests.sh index 0a6054f8..9e1ecd37 100755 --- a/.tekton/run_unittests.sh +++ b/.tekton/run_unittests.sh @@ -22,6 +22,8 @@ default) export REQUIREMENTS='requirements-310.txt' ;; 12) export REQUIREMENTS='requirements-312.txt' ;; + 13) + export REQUIREMENTS='requirements-313.txt' ;; *) export REQUIREMENTS='requirements.txt' ;; esac @@ -59,6 +61,7 @@ if [[ -n "${COUCHBASE_TEST}" ]]; then apt update apt install libcouchbase-dev -y fi + python -m venv /tmp/venv # shellcheck disable=SC1091 source /tmp/venv/bin/activate diff --git a/pyproject.toml b/pyproject.toml index 1d19b98d..d27b498a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.8" license = "MIT" keywords = [ "performance", - "opentracing", + "opentelemetry", "metrics", "monitoring", "tracing", @@ -44,13 +44,12 @@ classifiers = [ ] dependencies = [ "autowrapt>=1.0", - "basictracer>=3.1.0", "fysom>=2.1.2", - "opentracing>=2.3.0", "protobuf<5.0.0", "requests>=2.6.0", "six>=1.12.0", "urllib3>=1.26.5", + "opentelemetry-api>=1.23.0", ] [project.optional-dependencies] diff --git a/pytest.ini b/pytest.ini index 52835b1d..be615810 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,3 +3,4 @@ log_cli = 1 log_cli_level = WARN log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %H:%M:%S +pythonpath = src diff --git a/src/instana/collector/aws_eks_fargate.py b/src/instana/collector/aws_eks_fargate.py index c6a2d8f0..dac335e9 100644 --- a/src/instana/collector/aws_eks_fargate.py +++ b/src/instana/collector/aws_eks_fargate.py @@ -5,15 +5,17 @@ """ from time import time -from instana.log import logger + from instana.collector.base import BaseCollector from instana.collector.helpers.eks.process import EKSFargateProcessHelper from instana.collector.helpers.runtime import RuntimeHelper +from instana.collector.utils import format_trace_and_span_ids +from instana.log import logger from instana.util import DictionaryOfStan class EKSFargateCollector(BaseCollector): - """ Collector for EKS Pods on AWS Fargate """ + """Collector for EKS Pods on AWS Fargate""" def __init__(self, agent): super(EKSFargateCollector, self).__init__(agent) @@ -35,7 +37,7 @@ def prepare_payload(self): try: if not self.span_queue.empty(): - payload["spans"] = self.queued_spans() + payload["spans"] = format_trace_and_span_ids(self.queued_spans()) with_snapshot = self.should_send_snapshot_data() diff --git a/src/instana/collector/aws_fargate.py b/src/instana/collector/aws_fargate.py index 74c54c59..d3168d32 100644 --- a/src/instana/collector/aws_fargate.py +++ b/src/instana/collector/aws_fargate.py @@ -4,25 +4,27 @@ """ AWS Fargate Collector: Manages the periodic collection of metrics & snapshot data """ -import os + import json +import os from time import time -import requests -from ..log import logger -from .base import BaseCollector -from ..util import DictionaryOfStan, validate_url -from ..singletons import env_is_test +import requests -from .helpers.fargate.process import FargateProcessHelper -from .helpers.runtime import RuntimeHelper -from .helpers.fargate.task import TaskHelper -from .helpers.fargate.docker import DockerHelper -from .helpers.fargate.container import ContainerHelper +from instana.collector.base import BaseCollector +from instana.collector.helpers.fargate.container import ContainerHelper +from instana.collector.helpers.fargate.docker import DockerHelper +from instana.collector.helpers.fargate.process import FargateProcessHelper +from instana.collector.helpers.fargate.task import TaskHelper +from instana.collector.helpers.runtime import RuntimeHelper +from instana.collector.utils import format_trace_and_span_ids +from instana.log import logger +from instana.singletons import env_is_test +from instana.util import DictionaryOfStan, validate_url class AWSFargateCollector(BaseCollector): - """ Collector for AWS Fargate """ + """Collector for AWS Fargate""" def __init__(self, agent): super(AWSFargateCollector, self).__init__(agent) @@ -35,14 +37,16 @@ def __init__(self, agent): self.ecmu = os.environ.get("ECS_CONTAINER_METADATA_URI", "") if self.ecmu == "" or validate_url(self.ecmu) is False: - logger.warning("AWSFargateCollector: ECS_CONTAINER_METADATA_URI not in environment or invalid URL. " - "Instana will not be able to monitor this environment") + logger.warning( + "AWSFargateCollector: ECS_CONTAINER_METADATA_URI not in environment or invalid URL. " + "Instana will not be able to monitor this environment" + ) self.ready_to_start = False self.ecmu_url_root = self.ecmu - self.ecmu_url_task = self.ecmu + '/task' - self.ecmu_url_stats = self.ecmu + '/stats' - self.ecmu_url_task_stats = self.ecmu + '/task/stats' + self.ecmu_url_task = self.ecmu + "/task" + self.ecmu_url_stats = self.ecmu + "/stats" + self.ecmu_url_task_stats = self.ecmu + "/task/stats" # Timestamp in seconds of the last time we fetched all ECMU data self.last_ecmu_full_fetch = 0 @@ -84,7 +88,9 @@ def __init__(self, agent): def start(self): if self.ready_to_start is False: - logger.warning("AWS Fargate Collector is missing requirements and cannot monitor this environment.") + logger.warning( + "AWS Fargate Collector is missing requirements and cannot monitor this environment." + ) return super(AWSFargateCollector, self).start() @@ -122,7 +128,9 @@ def get_ecs_metadata(self): # Response from the last call to # ${ECS_CONTAINER_METADATA_URI}/task/stats - json_body = self.http_client.get(self.ecmu_url_task_stats, timeout=1).content + json_body = self.http_client.get( + self.ecmu_url_task_stats, timeout=1 + ).content self.task_stats_metadata = json.loads(json_body) except Exception: logger.debug("AWSFargateCollector.get_ecs_metadata", exc_info=True) @@ -137,7 +145,7 @@ def prepare_payload(self): try: if not self.span_queue.empty(): - payload["spans"] = self.queued_spans() + payload["spans"] = format_trace_and_span_ids(self.queued_spans()) with_snapshot = self.should_send_snapshot_data() diff --git a/src/instana/collector/aws_lambda.py b/src/instana/collector/aws_lambda.py index 5964e301..c0e4b731 100644 --- a/src/instana/collector/aws_lambda.py +++ b/src/instana/collector/aws_lambda.py @@ -4,14 +4,17 @@ """ AWS Lambda Collector: Manages the periodic collection of metrics & snapshot data """ -from ..log import logger -from .base import BaseCollector -from ..util import DictionaryOfStan -from ..util.aws import normalize_aws_lambda_arn + +from instana.collector.base import BaseCollector +from instana.collector.utils import format_trace_and_span_ids +from instana.log import logger +from instana.util import DictionaryOfStan +from instana.util.aws import normalize_aws_lambda_arn class AWSLambdaCollector(BaseCollector): - """ Collector for AWS Lambda """ + """Collector for AWS Lambda""" + def __init__(self, agent): super(AWSLambdaCollector, self).__init__(agent) logger.debug("Loading AWS Lambda Collector") @@ -47,7 +50,7 @@ def prepare_payload(self): payload["metrics"] = None if not self.span_queue.empty(): - payload["spans"] = self.queued_spans() + payload["spans"] = format_trace_and_span_ids(self.queued_spans()) if self.should_send_snapshot_data(): payload["metrics"] = self.snapshot_data @@ -60,8 +63,10 @@ def get_fq_arn(self): return self._fq_arn if self.context is None: - logger.debug("Attempt to get qualified ARN before the context object is available") - return '' + logger.debug( + "Attempt to get qualified ARN before the context object is available" + ) + return "" self._fq_arn = normalize_aws_lambda_arn(self.context) return self._fq_arn diff --git a/src/instana/collector/base.py b/src/instana/collector/base.py index c1576688..8c6df344 100644 --- a/src/instana/collector/base.py +++ b/src/instana/collector/base.py @@ -5,15 +5,16 @@ A Collector launches a background thread and continually collects & reports data. The data can be any combination of metrics, snapshot data and spans. """ -import sys -import threading -from ..log import logger -from ..singletons import env_is_test -from ..util import every, DictionaryOfStan +import queue # pylint: disable=import-error +import threading +from os import environ +from instana.log import logger +from instana.util import DictionaryOfStan, every -import queue # pylint: disable=import-error +# TODO: Use mock.patch() or unittest.mock to mock the testing env +env_is_test = "INSTANA_TEST" in environ class BaseCollector(object): @@ -21,6 +22,7 @@ class BaseCollector(object): Base class to handle the collection & reporting of snapshot and metric data This class launches a background thread to do this work. """ + def __init__(self, agent): # The agent for this process. Can be Standard, AWSLambda or Fargate self.agent = agent @@ -35,6 +37,7 @@ def __init__(self, agent): # others in background processes. This multiprocessing queue allows us to collect # up spans from all sources. import multiprocessing + self.span_queue = multiprocessing.Queue() else: self.span_queue = queue.Queue() @@ -92,7 +95,10 @@ def start(self): timer.name = "Collector Timed Start" timer.start() return - logger.debug("BaseCollector.start non-fatal: call but thread already running (started: %s)", self.started) + logger.debug( + "BaseCollector.start non-fatal: call but thread already running (started: %s)", + self.started, + ) return if self.agent.can_send(): @@ -104,7 +110,9 @@ def start(self): self.reporting_thread.start() self.started = True else: - logger.warning("BaseCollector.start: the agent tells us we can't send anything out") + logger.warning( + "BaseCollector.start: the agent tells us we can't send anything out" + ) def shutdown(self, report_final=True): """ @@ -123,7 +131,11 @@ def thread_loop(self): Just a loop that is run in the background thread. @return: None """ - every(self.report_interval, self.background_report, "Instana Collector: prepare_and_report_data") + every( + self.report_interval, + self.background_report, + "Instana Collector: prepare_and_report_data", + ) def background_report(self): """ @@ -131,13 +143,17 @@ def background_report(self): @return: Boolean """ if self.thread_shutdown.is_set(): - logger.debug("Thread shutdown signal is active: Shutting down reporting thread") + logger.debug( + "Thread shutdown signal is active: Shutting down reporting thread" + ) return False self.prepare_and_report_data() if self.thread_shutdown.is_set(): - logger.debug("Thread shutdown signal is active: Shutting down reporting thread") + logger.debug( + "Thread shutdown signal is active: Shutting down reporting thread" + ) return False return True @@ -188,7 +204,6 @@ def queued_spans(self): spans.append(span) return spans - def queued_profiles(self): """ Get all of the queued profiles diff --git a/src/instana/collector/google_cloud_run.py b/src/instana/collector/google_cloud_run.py index 27f5ec29..ffcbd984 100644 --- a/src/instana/collector/google_cloud_run.py +++ b/src/instana/collector/google_cloud_run.py @@ -4,19 +4,24 @@ """ Google Cloud Run Collector: Manages the periodic collection of metrics & snapshot data """ + import os from time import time + import requests -from instana.log import logger from instana.collector.base import BaseCollector -from instana.util import DictionaryOfStan, validate_url +from instana.collector.helpers.google_cloud_run.instance_entity import ( + InstanceEntityHelper, +) from instana.collector.helpers.google_cloud_run.process import GCRProcessHelper -from instana.collector.helpers.google_cloud_run.instance_entity import InstanceEntityHelper +from instana.collector.utils import format_trace_and_span_ids +from instana.log import logger +from instana.util import DictionaryOfStan, validate_url class GCRCollector(BaseCollector): - """ Collector for Google Cloud Run """ + """Collector for Google Cloud Run""" def __init__(self, agent, service, configuration, revision): super(GCRCollector, self).__init__(agent) @@ -29,15 +34,23 @@ def __init__(self, agent, service, configuration, revision): self.service = service self.configuration = configuration # Prepare the URLS that we will collect data from - self._gcr_md_uri = os.environ.get("GOOGLE_CLOUD_RUN_METADATA_ENDPOINT", "http://metadata.google.internal") + self._gcr_md_uri = os.environ.get( + "GOOGLE_CLOUD_RUN_METADATA_ENDPOINT", "http://metadata.google.internal" + ) if self._gcr_md_uri == "" or validate_url(self._gcr_md_uri) is False: - logger.warning("GCRCollector: GOOGLE_CLOUD_RUN_METADATA_ENDPOINT not in environment or invalid URL. " - "Instana will not be able to monitor this environment") + logger.warning( + "GCRCollector: GOOGLE_CLOUD_RUN_METADATA_ENDPOINT not in environment or invalid URL. " + "Instana will not be able to monitor this environment" + ) self.ready_to_start = False - self._gcr_md_project_uri = self._gcr_md_uri + '/computeMetadata/v1/project/?recursive=true' - self._gcr_md_instance_uri = self._gcr_md_uri + '/computeMetadata/v1/instance/?recursive=true' + self._gcr_md_project_uri = ( + self._gcr_md_uri + "/computeMetadata/v1/project/?recursive=true" + ) + self._gcr_md_instance_uri = ( + self._gcr_md_uri + "/computeMetadata/v1/instance/?recursive=true" + ) # Timestamp in seconds of the last time we fetched all GCR metadata self.__last_gcr_md_full_fetch = 0 @@ -65,7 +78,9 @@ def __init__(self, agent, service, configuration, revision): def start(self): if self.ready_to_start is False: - logger.warning("Google Cloud Run Collector is missing requirements and cannot monitor this environment.") + logger.warning( + "Google Cloud Run Collector is missing requirements and cannot monitor this environment." + ) return super(GCRCollector, self).start() @@ -81,15 +96,19 @@ def __get_project_instance_metadata(self): headers = {"Metadata-Flavor": "Google"} # Response from the last call to # ${GOOGLE_CLOUD_RUN_METADATA_ENDPOINT}/computeMetadata/v1/project/?recursive=true - self.project_metadata = self._http_client.get(self._gcr_md_project_uri, timeout=1, - headers=headers).json() + self.project_metadata = self._http_client.get( + self._gcr_md_project_uri, timeout=1, headers=headers + ).json() # Response from the last call to # ${GOOGLE_CLOUD_RUN_METADATA_ENDPOINT}/computeMetadata/v1/instance/?recursive=true - self.instance_metadata = self._http_client.get(self._gcr_md_instance_uri, timeout=1, - headers=headers).json() + self.instance_metadata = self._http_client.get( + self._gcr_md_instance_uri, timeout=1, headers=headers + ).json() except Exception: - logger.debug("GoogleCloudRunCollector.get_project_instance_metadata", exc_info=True) + logger.debug( + "GoogleCloudRunCollector.get_project_instance_metadata", exc_info=True + ) def should_send_snapshot_data(self): return int(time()) - self.snapshot_data_last_sent > self.snapshot_data_interval @@ -100,9 +119,8 @@ def prepare_payload(self): payload["metrics"]["plugins"] = [] try: - if not self.span_queue.empty(): - payload["spans"] = self.queued_spans() + payload["spans"] = format_trace_and_span_ids(self.queued_spans()) self.fetching_start_time = int(time()) delta = self.fetching_start_time - self.__last_gcr_md_full_fetch @@ -119,8 +137,12 @@ def prepare_payload(self): plugins = [] for helper in self.helpers: plugins.extend( - helper.collect_metrics(with_snapshot=with_snapshot, instance_metadata=self.instance_metadata, - project_metadata=self.project_metadata)) + helper.collect_metrics( + with_snapshot=with_snapshot, + instance_metadata=self.instance_metadata, + project_metadata=self.project_metadata, + ) + ) payload["metrics"]["plugins"] = plugins diff --git a/src/instana/collector/helpers/base.py b/src/instana/collector/helpers/base.py index 682617a2..e2cea3d5 100644 --- a/src/instana/collector/helpers/base.py +++ b/src/instana/collector/helpers/base.py @@ -6,13 +6,15 @@ in the data collection for various entities such as host, hardware, AWS Task, ec2, memory, cpu, docker etc etc.. """ -from ...log import logger + +from instana.log import logger class BaseHelper(object): """ Base class for all helpers. Descendants must override and implement `self.collect_metrics`. """ + def __init__(self, collector): self.collector = collector @@ -73,6 +75,6 @@ def apply_delta(self, source, previous, new, metric, with_snapshot): if previous_value != new_value or with_snapshot is True: previous[dst_metric] = new[dst_metric] = new_value - + def collect_metrics(self, **kwargs): logger.debug("BaseHelper.collect_metrics must be overridden") diff --git a/src/instana/collector/helpers/eks/process.py b/src/instana/collector/helpers/eks/process.py index 09198532..86239520 100644 --- a/src/instana/collector/helpers/eks/process.py +++ b/src/instana/collector/helpers/eks/process.py @@ -1,13 +1,15 @@ # (c) Copyright IBM Corp. 2024 -""" Module to handle the collection of containerized process metrics for EKS Pods on AWS Fargate """ +"""Module to handle the collection of containerized process metrics for EKS Pods on AWS Fargate""" + import os + from instana.collector.helpers.process import ProcessHelper from instana.log import logger def get_pod_name(): - podname = os.environ.get('HOSTNAME', '') + podname = os.environ.get("HOSTNAME", "") if not podname: logger.warning("Failed to determine podname from EKS hostname.") @@ -15,7 +17,7 @@ def get_pod_name(): class EKSFargateProcessHelper(ProcessHelper): - """ Helper class to extend the generic process helper class with the corresponding fargate attributes """ + """Helper class to extend the generic process helper class with the corresponding fargate attributes""" def collect_metrics(self, **kwargs): plugin_data = dict() diff --git a/src/instana/collector/helpers/fargate/container.py b/src/instana/collector/helpers/fargate/container.py index 90981298..82ad7bea 100644 --- a/src/instana/collector/helpers/fargate/container.py +++ b/src/instana/collector/helpers/fargate/container.py @@ -1,14 +1,16 @@ # (c) Copyright IBM Corp. 2021 # (c) Copyright Instana Inc. 2020 -""" Module to handle the collection of container metrics in AWS Fargate """ -from ....log import logger -from ....util import DictionaryOfStan -from ..base import BaseHelper +"""Module to handle the collection of container metrics in AWS Fargate""" + +from instana.collector.helpers.base import BaseHelper +from instana.log import logger +from instana.util import DictionaryOfStan class ContainerHelper(BaseHelper): - """ This class acts as a helper to collect container snapshot and metric information """ + """This class acts as a helper to collect container snapshot and metric information""" + def collect_metrics(self, **kwargs): """ Collect and return metrics (and optionally snapshot data) for every container in this task @@ -31,27 +33,55 @@ def collect_metrics(self, **kwargs): plugin_data["data"] = DictionaryOfStan() if self.collector.root_metadata["Name"] == name: plugin_data["data"]["instrumented"] = True - plugin_data["data"]["dockerId"] = container.get("DockerId", None) - plugin_data["data"]["taskArn"] = labels.get("com.amazonaws.ecs.task-arn", None) + plugin_data["data"]["dockerId"] = container.get( + "DockerId", None + ) + plugin_data["data"]["taskArn"] = labels.get( + "com.amazonaws.ecs.task-arn", None + ) if kwargs.get("with_snapshot"): plugin_data["data"]["runtime"] = "python" - plugin_data["data"]["dockerName"] = container.get("DockerName", None) - plugin_data["data"]["containerName"] = container.get("Name", None) + plugin_data["data"]["dockerName"] = container.get( + "DockerName", None + ) + plugin_data["data"]["containerName"] = container.get( + "Name", None + ) plugin_data["data"]["image"] = container.get("Image", None) - plugin_data["data"]["imageId"] = container.get("ImageID", None) - plugin_data["data"]["taskDefinition"] = labels.get("com.amazonaws.ecs.task-definition-family", None) - plugin_data["data"]["taskDefinitionVersion"] = labels.get("com.amazonaws.ecs.task-definition-version", None) - plugin_data["data"]["clusterArn"] = labels.get("com.amazonaws.ecs.cluster", None) - plugin_data["data"]["desiredStatus"] = container.get("DesiredStatus", None) - plugin_data["data"]["knownStatus"] = container.get("KnownStatus", None) + plugin_data["data"]["imageId"] = container.get( + "ImageID", None + ) + plugin_data["data"]["taskDefinition"] = labels.get( + "com.amazonaws.ecs.task-definition-family", None + ) + plugin_data["data"]["taskDefinitionVersion"] = labels.get( + "com.amazonaws.ecs.task-definition-version", None + ) + plugin_data["data"]["clusterArn"] = labels.get( + "com.amazonaws.ecs.cluster", None + ) + plugin_data["data"]["desiredStatus"] = container.get( + "DesiredStatus", None + ) + plugin_data["data"]["knownStatus"] = container.get( + "KnownStatus", None + ) plugin_data["data"]["ports"] = container.get("Ports", None) - plugin_data["data"]["createdAt"] = container.get("CreatedAt", None) - plugin_data["data"]["startedAt"] = container.get("StartedAt", None) + plugin_data["data"]["createdAt"] = container.get( + "CreatedAt", None + ) + plugin_data["data"]["startedAt"] = container.get( + "StartedAt", None + ) plugin_data["data"]["type"] = container.get("Type", None) limits = container.get("Limits", {}) - plugin_data["data"]["limits"]["cpu"] = limits.get("CPU", None) - plugin_data["data"]["limits"]["memory"] = limits.get("Memory", None) + plugin_data["data"]["limits"]["cpu"] = limits.get( + "CPU", None + ) + plugin_data["data"]["limits"]["memory"] = limits.get( + "Memory", None + ) except Exception: logger.debug("_collect_container_snapshots: ", exc_info=True) finally: diff --git a/src/instana/collector/helpers/process.py b/src/instana/collector/helpers/process.py index 073842f2..2f2115cb 100644 --- a/src/instana/collector/helpers/process.py +++ b/src/instana/collector/helpers/process.py @@ -1,19 +1,21 @@ # (c) Copyright IBM Corp. 2021 # (c) Copyright Instana Inc. 2020 -""" Collection helper for the process """ +"""Collection helper for the process""" + +import grp import os import pwd -import grp + +from instana.collector.helpers.base import BaseHelper from instana.log import logger from instana.util import DictionaryOfStan from instana.util.runtime import get_proc_cmdline from instana.util.secrets import contains_secret -from .base import BaseHelper class ProcessHelper(BaseHelper): - """ Helper class to collect metrics for this process """ + """Helper class to collect metrics for this process""" def collect_metrics(self, **kwargs): plugin_data = dict() @@ -33,9 +35,11 @@ def _collect_process_snapshot(self, plugin_data): try: env = dict() for key in os.environ: - if contains_secret(key, - self.collector.agent.options.secrets_matcher, - self.collector.agent.options.secrets_list): + if contains_secret( + key, + self.collector.agent.options.secrets_matcher, + self.collector.agent.options.secrets_list, + ): env[key] = "" else: env[key] = os.environ[key] diff --git a/src/instana/collector/helpers/runtime.py b/src/instana/collector/helpers/runtime.py index f6111bb0..8aef48e3 100644 --- a/src/instana/collector/helpers/runtime.py +++ b/src/instana/collector/helpers/runtime.py @@ -2,21 +2,22 @@ # (c) Copyright Instana Inc. 2020 """ Collection helper for the Python runtime """ +import gc import importlib.metadata import os -import gc -import sys import platform import resource +import sys import threading from types import ModuleType +from instana.collector.helpers.base import BaseHelper from instana.log import logger -from instana.version import VERSION from instana.util import DictionaryOfStan from instana.util.runtime import determine_service_name +from instana.version import VERSION -from .base import BaseHelper +PATH_OF_DEPRECATED_INSTALLATION_VIA_HOST_AGENT = "/tmp/.instana/python" PATH_OF_AUTOTRACE_WEBHOOK_SITEDIR = '/opt/instana/instrumentation/python/' @@ -29,7 +30,7 @@ def is_webhook_instrumented(): class RuntimeHelper(BaseHelper): - """ Helper class to collect snapshot and metrics for this Python runtime """ + """Helper class to collect snapshot and metrics for this Python runtime""" def __init__(self, collector): super(RuntimeHelper, self).__init__(collector) @@ -66,7 +67,7 @@ def collect_metrics(self, **kwargs): return [plugin_data] def _collect_runtime_metrics(self, plugin_data, with_snapshot): - if os.environ.get('INSTANA_DISABLE_METRICS_COLLECTION', False): + if os.environ.get("INSTANA_DISABLE_METRICS_COLLECTION", False): return """ Collect up and return the runtime metrics """ @@ -78,61 +79,141 @@ def _collect_runtime_metrics(self, plugin_data, with_snapshot): self._collect_thread_metrics(plugin_data, with_snapshot) value_diff = rusage.ru_utime - self.previous_rusage.ru_utime - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_utime", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_utime", + with_snapshot, + ) value_diff = rusage.ru_stime - self.previous_rusage.ru_stime - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_stime", with_snapshot) - - self.apply_delta(rusage.ru_maxrss, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_maxrss", with_snapshot) - self.apply_delta(rusage.ru_ixrss, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_ixrss", with_snapshot) - self.apply_delta(rusage.ru_idrss, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_idrss", with_snapshot) - self.apply_delta(rusage.ru_isrss, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_isrss", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_stime", + with_snapshot, + ) + + self.apply_delta( + rusage.ru_maxrss, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_maxrss", + with_snapshot, + ) + self.apply_delta( + rusage.ru_ixrss, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_ixrss", + with_snapshot, + ) + self.apply_delta( + rusage.ru_idrss, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_idrss", + with_snapshot, + ) + self.apply_delta( + rusage.ru_isrss, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_isrss", + with_snapshot, + ) value_diff = rusage.ru_minflt - self.previous_rusage.ru_minflt - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_minflt", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_minflt", + with_snapshot, + ) value_diff = rusage.ru_majflt - self.previous_rusage.ru_majflt - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_majflt", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_majflt", + with_snapshot, + ) value_diff = rusage.ru_nswap - self.previous_rusage.ru_nswap - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_nswap", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_nswap", + with_snapshot, + ) value_diff = rusage.ru_inblock - self.previous_rusage.ru_inblock - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_inblock", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_inblock", + with_snapshot, + ) value_diff = rusage.ru_oublock - self.previous_rusage.ru_oublock - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_oublock", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_oublock", + with_snapshot, + ) value_diff = rusage.ru_msgsnd - self.previous_rusage.ru_msgsnd - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_msgsnd", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_msgsnd", + with_snapshot, + ) value_diff = rusage.ru_msgrcv - self.previous_rusage.ru_msgrcv - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_msgrcv", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_msgrcv", + with_snapshot, + ) value_diff = rusage.ru_nsignals - self.previous_rusage.ru_nsignals - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_nsignals", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_nsignals", + with_snapshot, + ) value_diff = rusage.ru_nvcsw - self.previous_rusage.ru_nvcsw - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_nvcsw", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_nvcsw", + with_snapshot, + ) value_diff = rusage.ru_nivcsw - self.previous_rusage.ru_nivcsw - self.apply_delta(value_diff, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "ru_nivcsw", with_snapshot) + self.apply_delta( + value_diff, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "ru_nivcsw", + with_snapshot, + ) except Exception: logger.debug("_collect_runtime_metrics", exc_info=True) finally: @@ -143,19 +224,49 @@ def _collect_gc_metrics(self, plugin_data, with_snapshot): gc_count = gc.get_count() gc_threshold = gc.get_threshold() - self.apply_delta(gc_count[0], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "collect0", with_snapshot) - self.apply_delta(gc_count[1], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "collect1", with_snapshot) - self.apply_delta(gc_count[2], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "collect2", with_snapshot) - - self.apply_delta(gc_threshold[0], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "threshold0", with_snapshot) - self.apply_delta(gc_threshold[1], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "threshold1", with_snapshot) - self.apply_delta(gc_threshold[2], self.previous['data']['metrics']['gc'], - plugin_data['data']['metrics']['gc'], "threshold2", with_snapshot) + self.apply_delta( + gc_count[0], + self.previous["data"]["metrics"]["gc"], + plugin_data["data"]["metrics"]["gc"], + "collect0", + with_snapshot, + ) + self.apply_delta( + gc_count[1], + self.previous["data"]["metrics"]["gc"], + plugin_data["data"]["metrics"]["gc"], + "collect1", + with_snapshot, + ) + self.apply_delta( + gc_count[2], + self.previous["data"]["metrics"]["gc"], + plugin_data["data"]["metrics"]["gc"], + "collect2", + with_snapshot, + ) + + self.apply_delta( + gc_threshold[0], + self.previous["data"]["metrics"]["gc"], + plugin_data["data"]["metrics"]["gc"], + "threshold0", + with_snapshot, + ) + self.apply_delta( + gc_threshold[1], + self.previous["data"]["metrics"]["gc"], + plugin_data["data"]["metrics"]["gc"], + "threshold1", + with_snapshot, + ) + self.apply_delta( + gc_threshold[2], + self.previous["data"]["metrics"]["gc"], + plugin_data["data"]["metrics"]["gc"], + "threshold2", + with_snapshot, + ) except Exception: logger.debug("_collect_gc_metrics", exc_info=True) @@ -163,55 +274,77 @@ def _collect_thread_metrics(self, plugin_data, with_snapshot): try: threads = threading.enumerate() daemon_threads = [thread.daemon is True for thread in threads].count(True) - self.apply_delta(daemon_threads, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "daemon_threads", with_snapshot) + self.apply_delta( + daemon_threads, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "daemon_threads", + with_snapshot, + ) alive_threads = [thread.daemon is False for thread in threads].count(True) - self.apply_delta(alive_threads, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "alive_threads", with_snapshot) - - dummy_threads = [isinstance(thread, threading._DummyThread) for thread in threads].count( - True) # pylint: disable=protected-access - self.apply_delta(dummy_threads, self.previous['data']['metrics'], - plugin_data['data']['metrics'], "dummy_threads", with_snapshot) + self.apply_delta( + alive_threads, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "alive_threads", + with_snapshot, + ) + + dummy_threads = [ + isinstance(thread, threading._DummyThread) for thread in threads + ].count(True) # pylint: disable=protected-access + self.apply_delta( + dummy_threads, + self.previous["data"]["metrics"], + plugin_data["data"]["metrics"], + "dummy_threads", + with_snapshot, + ) except Exception: logger.debug("_collect_thread_metrics", exc_info=True) def _collect_runtime_snapshot(self, plugin_data): - """ Gathers Python specific Snapshot information for this process """ + """Gathers Python specific Snapshot information for this process""" snapshot_payload = {} try: - snapshot_payload['name'] = determine_service_name() - snapshot_payload['version'] = sys.version - snapshot_payload['f'] = platform.python_implementation() # flavor - snapshot_payload['a'] = platform.architecture()[0] # architecture - snapshot_payload['versions'] = self.gather_python_packages() - snapshot_payload['iv'] = VERSION + snapshot_payload["name"] = determine_service_name() + snapshot_payload["version"] = sys.version + snapshot_payload["f"] = platform.python_implementation() # flavor + snapshot_payload["a"] = platform.architecture()[0] # architecture + snapshot_payload["versions"] = self.gather_python_packages() + snapshot_payload["iv"] = VERSION if is_autowrapt_instrumented(): snapshot_payload['m'] = 'Autowrapt' elif is_webhook_instrumented(): snapshot_payload['m'] = 'AutoTrace' else: - snapshot_payload['m'] = 'Manual' + snapshot_payload["m"] = "Manual" try: - from django.conf import settings # pylint: disable=import-outside-toplevel - if hasattr(settings, 'MIDDLEWARE') and settings.MIDDLEWARE is not None: - snapshot_payload['djmw'] = settings.MIDDLEWARE - elif hasattr(settings, 'MIDDLEWARE_CLASSES') and settings.MIDDLEWARE_CLASSES is not None: - snapshot_payload['djmw'] = settings.MIDDLEWARE_CLASSES + from django.conf import ( + settings, # pylint: disable=import-outside-toplevel + ) + + if hasattr(settings, "MIDDLEWARE") and settings.MIDDLEWARE is not None: + snapshot_payload["djmw"] = settings.MIDDLEWARE + elif ( + hasattr(settings, "MIDDLEWARE_CLASSES") + and settings.MIDDLEWARE_CLASSES is not None + ): + snapshot_payload["djmw"] = settings.MIDDLEWARE_CLASSES except Exception: pass except Exception: logger.debug("collect_snapshot: ", exc_info=True) - plugin_data['data']['snapshot'] = snapshot_payload + plugin_data["data"]["snapshot"] = snapshot_payload def gather_python_packages(self): - """ Collect up the list of modules in use """ - if os.environ.get('INSTANA_DISABLE_PYTHON_PACKAGE_COLLECTION'): - return {'instana': VERSION} + """Collect up the list of modules in use""" + if os.environ.get("INSTANA_DISABLE_PYTHON_PACKAGE_COLLECTION"): + return {"instana": VERSION} versions = {} try: @@ -220,7 +353,7 @@ def gather_python_packages(self): for pkg_name in sys_packages: # Don't report submodules (e.g. django.x, django.y, django.z) # Skip modules that begin with underscore - if ('.' in pkg_name) or pkg_name[0] == '_': + if ("." in pkg_name) or pkg_name[0] == "_": continue # Skip builtins @@ -234,7 +367,9 @@ def gather_python_packages(self): if isinstance(pkg_info["__version__"], str): versions[pkg_name] = pkg_info["__version__"] else: - versions[pkg_name] = self.jsonable(pkg_info["__version__"]) + versions[pkg_name] = self.jsonable( + pkg_info["__version__"] + ) elif "version" in pkg_info: versions[pkg_name] = self.jsonable(pkg_info["version"]) else: @@ -242,10 +377,13 @@ def gather_python_packages(self): except importlib.metadata.PackageNotFoundError: pass except Exception: - logger.debug("gather_python_packages: could not process module: %s", pkg_name) + logger.debug( + "gather_python_packages: could not process module: %s", + pkg_name, + ) # Manually set our package version - versions['instana'] = VERSION + versions["instana"] = VERSION except Exception: logger.debug("gather_python_packages", exc_info=True) @@ -257,7 +395,7 @@ def jsonable(self, value): try: result = value() except Exception: - result = 'Unknown' + result = "Unknown" elif isinstance(value, ModuleType): result = value else: diff --git a/src/instana/collector/host.py b/src/instana/collector/host.py index 2ec2bd8b..d5e3b77f 100644 --- a/src/instana/collector/host.py +++ b/src/instana/collector/host.py @@ -4,16 +4,21 @@ """ Host Collector: Manages the periodic collection of metrics & snapshot data """ + from time import time -from ..log import logger -from .base import BaseCollector -from ..util import DictionaryOfStan -from .helpers.runtime import RuntimeHelper +from typing import DefaultDict, Any + +from instana.collector.base import BaseCollector +from instana.collector.helpers.runtime import RuntimeHelper +from instana.collector.utils import format_trace_and_span_ids +from instana.log import logger +from instana.util import DictionaryOfStan class HostCollector(BaseCollector): - """ Collector for host agent """ - def __init__(self, agent): + """Collector for host agent""" + + def __init__(self, agent) -> None: super(HostCollector, self).__init__(agent) logger.debug("Loading Host Collector") @@ -23,14 +28,16 @@ def __init__(self, agent): # Populate the collection helpers self.helpers.append(RuntimeHelper(self)) - def start(self): + def start(self) -> None: if self.ready_to_start is False: - logger.warning("Host Collector is missing requirements and cannot monitor this environment.") + logger.warning( + "Host Collector is missing requirements and cannot monitor this environment." + ) return super(HostCollector, self).start() - def prepare_and_report_data(self): + def prepare_and_report_data(self) -> None: """ We override this method from the base class so that we can handle the wait4init state machine case. @@ -45,21 +52,28 @@ def prepare_and_report_data(self): else: return - if self.agent.machine.fsm.current == "good2go" and self.agent.is_timed_out(): - logger.info("The Instana host agent has gone offline or is no longer reachable for > 1 min. Will retry periodically.") + if ( + self.agent.machine.fsm.current == "good2go" + and self.agent.is_timed_out() + ): + logger.info( + "The Instana host agent has gone offline or is no longer reachable for > 1 min. Will retry periodically." + ) self.agent.reset() except Exception: - logger.debug('Harmless state machine thread disagreement. Will self-correct on next timer cycle.') + logger.debug( + "Harmless state machine thread disagreement. Will self-correct on next timer cycle." + ) super(HostCollector, self).prepare_and_report_data() - def should_send_snapshot_data(self): + def should_send_snapshot_data(self) -> bool: delta = int(time()) - self.snapshot_data_last_sent if delta > self.snapshot_data_interval: return True return False - def prepare_payload(self): + def prepare_payload(self) -> DefaultDict[Any, Any]: payload = DictionaryOfStan() payload["spans"] = [] payload["profiles"] = [] @@ -67,7 +81,7 @@ def prepare_payload(self): try: if not self.span_queue.empty(): - payload["spans"] = self.queued_spans() + payload["spans"] = format_trace_and_span_ids(self.queued_spans()) if not self.profile_queue.empty(): payload["profiles"] = self.queued_profiles() diff --git a/src/instana/collector/utils.py b/src/instana/collector/utils.py new file mode 100644 index 00000000..4bb4e9ff --- /dev/null +++ b/src/instana/collector/utils.py @@ -0,0 +1,24 @@ +# (c) Copyright IBM Corp. 2024 + +from typing import List + +from opentelemetry.trace.span import format_span_id + +from instana.span import InstanaSpan + + +def format_trace_and_span_ids( + queued_spans: List[InstanaSpan], +) -> List[InstanaSpan]: + """ + Format the Trace, Parent Span, and Span IDs of Spans to be a 64-bit + Hexadecimal String instead of Integer before being pushed to a + Collector (or Instana Agent). + """ + spans = [] + for span in queued_spans: + span.t = format_span_id(span.t) + span.p = format_span_id(span.p) + span.s = format_span_id(span.s) + spans.append(span) + return spans diff --git a/src/instana/propagators/base_propagator.py b/src/instana/propagators/base_propagator.py index a70d032d..25397ec3 100644 --- a/src/instana/propagators/base_propagator.py +++ b/src/instana/propagators/base_propagator.py @@ -2,8 +2,8 @@ # (c) Copyright Instana Inc. 2020 -import sys import os +import typing from instana.log import logger from instana.util.ids import header_to_id, header_to_long_id @@ -11,8 +11,15 @@ from instana.w3c_trace_context.traceparent import Traceparent from instana.w3c_trace_context.tracestate import Tracestate +from opentelemetry.trace import ( + INVALID_SPAN_ID, + INVALID_TRACE_ID, + NonRecordingSpan, + set_span_in_context, +) +from opentelemetry.context.context import Context -# The carrier can be a dict or a list. +# The carrier, typed here as CarrierT, can be a dict, a list, or a tuple. # Using the trace header as an example, it can be in the following forms # for extraction: # X-Instana-T @@ -23,6 +30,7 @@ # # For injection, we only support the standard format: # X-Instana-T +CarrierT = typing.TypeVar("CarrierT", typing.Dict, typing.List, typing.Tuple) class BasePropagator(object): @@ -154,7 +162,7 @@ def __determine_span_context(self, trace_id, span_id, level, synthetic, tracepar correlation = False disable_traceparent = os.environ.get("INSTANA_DISABLE_W3C_TRACE_CORRELATION", "") instana_ancestor = None - ctx = SpanContext() + ctx = SpanContext(trace_id=trace_id, span_id=span_id, is_remote=False) if level and "correlationType" in level: trace_id, span_id = [None] * 2 correlation = True @@ -166,7 +174,12 @@ def __determine_span_context(self, trace_id, span_id, level, synthetic, tracepar ctx.correlation_type = None ctx.correlation_id = None - if trace_id and span_id: + if ( + trace_id + and span_id + and trace_id != INVALID_TRACE_ID + and span_id != INVALID_SPAN_ID + ): ctx.trace_id = trace_id[-16:] # only the last 16 chars ctx.span_id = span_id[-16:] # only the last 16 chars ctx.synthetic = synthetic is not None @@ -290,9 +303,22 @@ def extract(self, carrier, disable_w3c_trace_context=False): if traceparent: traceparent = self._tp.validate(traceparent) - ctx = self.__determine_span_context(trace_id, span_id, level, synthetic, traceparent, tracestate, - disable_w3c_trace_context) - + if trace_id is None: + trace_id = INVALID_TRACE_ID + if span_id is None: + span_id = INVALID_SPAN_ID + + span_context = self.__determine_span_context( + trace_id, + span_id, + level, + synthetic, + traceparent, + tracestate, + disable_w3c_trace_context, + ) + ctx = set_span_in_context(NonRecordingSpan(span_context), Context()) return ctx + except Exception: logger.debug("extract error:", exc_info=True) diff --git a/src/instana/propagators/exceptions.py b/src/instana/propagators/exceptions.py new file mode 100644 index 00000000..7e613c5f --- /dev/null +++ b/src/instana/propagators/exceptions.py @@ -0,0 +1,11 @@ +# (c) Copyright IBM Corp. 2024 + + +class UnsupportedFormatException(Exception): + """UnsupportedFormatException should be used when the provided format + value is unknown or disallowed by the :class:`InstanaTracer`. + + See :meth:`InstanaTracer.inject()` and :meth:`InstanaTracer.extract()`. + """ + + pass diff --git a/src/instana/propagators/format.py b/src/instana/propagators/format.py new file mode 100644 index 00000000..9049c4e1 --- /dev/null +++ b/src/instana/propagators/format.py @@ -0,0 +1,52 @@ +# (c) Copyright IBM Corp. 2024 + + +class Format(object): + """A namespace for builtin carrier formats. + + These static constants are intended for use in the :meth:`Tracer.inject()` + and :meth:`Tracer.extract()` methods. E.g.:: + + tracer.inject(span.context, Format.BINARY, binary_carrier) + + """ + + BINARY = "binary" + """ + The BINARY format represents SpanContexts in an opaque bytearray carrier. + + For both :meth:`Tracer.inject()` and :meth:`Tracer.extract()` the carrier + should be a bytearray instance. :meth:`Tracer.inject()` must append to the + bytearray carrier (rather than replace its contents). + """ + + TEXT_MAP = "text_map" + """ + The TEXT_MAP format represents :class:`SpanContext`\\ s in a python + ``dict`` mapping from strings to strings. + + Both the keys and the values have unrestricted character sets (unlike the + HTTP_HEADERS format). + + NOTE: The TEXT_MAP carrier ``dict`` may contain unrelated data (e.g., + arbitrary gRPC metadata). As such, the :class:`Tracer` implementation + should use a prefix or other convention to distinguish tracer-specific + key:value pairs. + """ + + HTTP_HEADERS = "http_headers" + """ + The HTTP_HEADERS format represents :class:`SpanContext`\\ s in a python + ``dict`` mapping from character-restricted strings to strings. + + Keys and values in the HTTP_HEADERS carrier must be suitable for use as + HTTP headers (without modification or further escaping). That is, the + keys have a greatly restricted character set, casing for the keys may not + be preserved by various intermediaries, and the values should be + URL-escaped. + + NOTE: The HTTP_HEADERS carrier ``dict`` may contain unrelated data (e.g., + arbitrary gRPC metadata). As such, the :class:`Tracer` implementation + should use a prefix or other convention to distinguish tracer-specific + key:value pairs. + """ diff --git a/src/instana/recorder.py b/src/instana/recorder.py index b5875805..74926705 100644 --- a/src/instana/recorder.py +++ b/src/instana/recorder.py @@ -5,48 +5,70 @@ import os import queue -import sys - -from basictracer import Sampler from .span import RegisteredSpan, SDKSpan - class StanRecorder(object): THREAD_NAME = "Instana Span Reporting" - REGISTERED_SPANS = ("aiohttp-client", "aiohttp-server", "aws.lambda.entry", - "boto3", "cassandra", "celery-client", "celery-worker", - "couchbase", "django", "gcs", "gcps-producer", - "gcps-consumer", "log", "memcache", "mongo", "mysql", - "postgres", "pymongo", "rabbitmq", "redis","render", - "rpc-client", "rpc-server", "sqlalchemy", "tornado-client", - "tornado-server", "urllib3", "wsgi", "asgi") + REGISTERED_SPANS = ( + "aiohttp-client", + "aiohttp-server", + "aws.lambda.entry", + "boto3", + "cassandra", + "celery-client", + "celery-worker", + "couchbase", + "django", + "gcs", + "gcps-producer", + "gcps-consumer", + "log", + "memcache", + "mongo", + "mysql", + "postgres", + "pymongo", + "rabbitmq", + "redis", + "render", + "rpc-client", + "rpc-server", + "sqlalchemy", + "tornado-client", + "tornado-server", + "urllib3", + "wsgi", + "asgi", + ) # Recorder thread for collection/reporting of spans thread = None - def __init__(self, agent = None): + def __init__(self, agent=None): if agent is None: # Late import to avoid circular import # pylint: disable=import-outside-toplevel from .singletons import get_agent + self.agent = get_agent() else: self.agent = agent def queue_size(self): - """ Return the size of the queue; how may spans are queued, """ + """Return the size of the queue; how may spans are queued,""" return self.agent.collector.span_queue.qsize() def queued_spans(self): - """ Get all of the spans in the queue """ + """Get all of the spans in the queue""" span = None spans = [] import time from .singletons import env_is_test + if env_is_test is True: time.sleep(1) @@ -63,8 +85,8 @@ def queued_spans(self): return spans def clear_spans(self): - """ Clear the queue of spans """ - if self.agent.collector.span_queue.empty() == False: + """Clear the queue of spans""" + if not self.agent.collector.span_queue.empty(): self.queued_spans() def record_span(self, span): @@ -88,9 +110,3 @@ def record_span(self, span): # logger.debug("Recorded span: %s", json_span) self.agent.collector.span_queue.put(json_span) - - -class InstanaSampler(Sampler): - def sampled(self, _): - # We never sample - return False diff --git a/src/instana/sampling.py b/src/instana/sampling.py new file mode 100644 index 00000000..7f84f786 --- /dev/null +++ b/src/instana/sampling.py @@ -0,0 +1,43 @@ +# (c) Copyright IBM Corp. 2024 + +import abc +import enum + + +class SamplingPolicy(enum.Enum): + # IsRecording() == False + # Span will not be recorded and all events and attributes will be dropped. + # https://opentelemetry.io/docs/specs/otel/trace/api/#isrecording + DROP = 0 + # IsRecording() == True, but Sampled flag MUST NOT be set. + RECORD_ONLY = 1 + # IsRecording() == True AND Sampled flag MUST be set. + RECORD_AND_SAMPLE = 2 + + +class Sampler(abc.ABC): + """Samplers choose whether the span is recorded or dropped. + + A variety of sampling algorithms are available, and choosing which sampler + to use and how to configure it is one of the most confusing parts of + setting up a tracing system. + """ + + @abc.abstractmethod + def sampled(self) -> bool: + """ + Returns if a span was dropped (False) or recorded (True). + + Calling a span “sampled” can mean it was “sampled out” (dropped) + or “sampled in” (recorded). + """ + pass + + +class InstanaSampler(Sampler): + def __init__(self) -> None: + # Instana never samples. + self._sampled: SamplingPolicy = SamplingPolicy.DROP + + def sampled(self) -> bool: + return False if self._sampled == SamplingPolicy.DROP else True diff --git a/src/instana/singletons.py b/src/instana/singletons.py index c93416ca..de605439 100644 --- a/src/instana/singletons.py +++ b/src/instana/singletons.py @@ -3,11 +3,10 @@ import os -import opentracing +from opentelemetry import trace from .autoprofile.profiler import Profiler -from .log import logger -from .tracer import InstanaTracer +from .tracer import InstanaTracerProvider agent = None tracer = None @@ -96,34 +95,23 @@ def set_agent(new_agent): agent = new_agent -# The global OpenTracing compatible tracer used internally by +# The global OpenTelemetry compatible tracer used internally by # this package. -tracer = InstanaTracer(recorder=span_recorder) +provider = InstanaTracerProvider(recorder=span_recorder) +provider.add_span_processor(agent) -try: - from opentracing.scope_managers.contextvars import ContextVarsScopeManager +# Sets the global default tracer provider +trace.set_tracer_provider(provider) - async_tracer = InstanaTracer( - scope_manager=ContextVarsScopeManager(), recorder=span_recorder - ) -except Exception: - logger.debug("Error setting up async_tracer:", exc_info=True) - -# Mock the tornado tracer until tornado is detected and instrumented first -tornado_tracer = tracer +# Creates a tracer from the global tracer provider +tracer = trace.get_tracer("instana.tracer") +async_tracer = trace.get_tracer("instana.async.tracer") +tornado_tracer = None def setup_tornado_tracer(): global tornado_tracer - from opentracing.scope_managers.tornado import TornadoScopeManager - - tornado_tracer = InstanaTracer( - scope_manager=TornadoScopeManager(), recorder=span_recorder - ) - - -# Set ourselves as the tracer. -opentracing.tracer = tracer + tornado_tracer = trace.get_tracer("instana.tornado.tracer") def get_tracer(): diff --git a/src/instana/span.py b/src/instana/span.py index 5398fbd3..8c3206c9 100644 --- a/src/instana/span.py +++ b/src/instana/span.py @@ -4,7 +4,7 @@ """ This module contains the classes that represents spans. -InstanaSpan - the OpenTracing based span used during tracing +InstanaSpan - the OpenTelemetry based span used during tracing When an InstanaSpan is finished, it is converted into either an SDKSpan or RegisteredSpan depending on type. @@ -14,103 +14,313 @@ - RegisteredSpan: Class that represents a Registered type span """ import six - -from basictracer.span import BasicSpan -import opentracing.ext.tags as ot_tags - +from typing import Dict, Optional, Union, Sequence, Tuple +from threading import Lock +from time import time_ns + +from opentelemetry.trace import ( + Span, + DEFAULT_TRACE_OPTIONS, + DEFAULT_TRACE_STATE, + INVALID_SPAN_ID, + INVALID_TRACE_ID, + _SPAN_KEY, +) +from opentelemetry.util import types +from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.trace.span import NonRecordingSpan +from opentelemetry.context import get_value +from opentelemetry.context.context import Context + +from .span_context import SpanContext from .log import logger from .util import DictionaryOfStan -class InstanaSpan(BasicSpan): +class Event: + def __init__( + self, + name: str, + attributes: types.Attributes = None, + timestamp: Optional[int] = None, + ) -> None: + self._name = name + self._attributes = attributes + if timestamp is None: + self._timestamp = time_ns() + else: + self._timestamp = timestamp + + @property + def name(self) -> str: + return self._name + + @property + def timestamp(self) -> int: + return self._timestamp + + @property + def attributes(self) -> types.Attributes: + return self._attributes + + +class InstanaSpan(Span): stack = None synthetic = False - def mark_as_errored(self, tags=None): + def __init__( + self, + name: str, + context: SpanContext, + parent_id: Optional[str] = None, + start_time: Optional[int] = None, + end_time: Optional[int] = None, + attributes: types.Attributes = {}, + events: Sequence[Event] = [], + status: Optional[Status] = Status(StatusCode.UNSET), + ) -> None: + self._name = name + self._context = context + self._lock = Lock() + self._start_time = start_time or time_ns() + self._end_time = end_time + self._duration = 0 + self._attributes = attributes + self._events = events + self._parent_id = parent_id + self._status = status + + if context.synthetic: + self.synthetic = True + + + @property + def name(self) -> str: + return self._name + + def get_span_context(self) -> SpanContext: + return self._context + + @property + def context(self) -> SpanContext: + return self._context + + @property + def start_time(self) -> Optional[int]: + return self._start_time + + @property + def end_time(self) -> Optional[int]: + return self._end_time + + @property + def duration(self) -> int: + return self._duration + + @property + def attributes(self) -> types.Attributes: + return self._attributes + + def set_attributes(self, attributes: Dict[str, types.AttributeValue]) -> None: + if not self._attributes: + self._attributes = {} + + with self._lock: + for key, value in attributes.items(): + self._attributes[key] = value + + def set_attribute(self, key: str, value: types.AttributeValue) -> None: + return self.set_attributes({key: value}) + + @property + def events(self) -> Sequence[Event]: + return self._events + + @property + def status(self) -> Status: + return self._status + + @property + def parent_id(self) -> int: + return self._parent_id + + def update_name(self, name: str) -> None: + with self._lock: + self._name = name + + def is_recording(self) -> bool: + return self._end_time is None + + def set_status( + self, + status: Union[Status, StatusCode], + description: Optional[str] = None, + ) -> None: + # Ignore future calls if status is already set to OK + # Ignore calls to set to StatusCode.UNSET + if isinstance(status, Status): + if ( + self._status + and self._status.status_code is StatusCode.OK + or status.status_code is StatusCode.UNSET + ): + return + if description is not None: + logger.warning( + "Description %s ignored. Use either `Status` or `(StatusCode, Description)`", + description, + ) + self._status = status + elif isinstance(status, StatusCode): + if ( + self._status + and self._status.status_code is StatusCode.OK + or status is StatusCode.UNSET + ): + return + self._status = Status(status, description) + + def add_event( + self, + name: str, + attributes: types.Attributes = None, + timestamp: Optional[int] = None, + ) -> None: + + event = Event( + name=name, + attributes=attributes, + timestamp=timestamp, + ) + + self._events.append(event) + + def record_exception( + self, + exception: Exception, + attributes: types.Attributes = None, + timestamp: Optional[int] = None, + escaped: bool = False, + ) -> None: + """ + Records an exception as a span event. This will record pertinent info from the exception and + assure that this span is marked as errored. + """ + try: + message = "" + self.mark_as_errored() + if hasattr(exception, "__str__") and len(str(exception)) > 0: + message = str(exception) + elif hasattr(exception, "message") and exception.message is not None: + message = exception.message + else: + message = repr(exception) + + if self.name in ["rpc-server", "rpc-client"]: + self.set_attribute("rpc.error", message) + elif self.name == "mysql": + self.set_attribute("mysql.error", message) + elif self.name == "postgres": + self.set_attribute("pg.error", message) + elif self.name in RegisteredSpan.HTTP_SPANS: + self.set_attribute("http.error", message) + elif self.name in ["celery-client", "celery-worker"]: + self.set_attribute("error", message) + elif self.name == "sqlalchemy": + self.set_attribute("sqlalchemy.err", message) + elif self.name == "aws.lambda.entry": + self.set_attribute("lambda.error", message) + else: + _attributes = {"message": message} + if attributes: + _attributes.update(attributes) + self.add_event( + name="exception", attributes=_attributes, timestamp=timestamp + ) + except Exception: + logger.debug("span.record_exception", exc_info=True) + raise + + def end(self, end_time: Optional[int] = None) -> None: + with self._lock: + self._end_time = end_time if end_time is not None else time_ns() + self._duration = self._end_time - self._start_time + + def mark_as_errored(self, attributes: types.Attributes = None) -> None: """ Mark this span as errored. - @param tags: optional tags to add to the span + @param attributes: optional attributes to add to the span """ try: - ec = self.tags.get('ec', 0) - self.set_tag('ec', ec + 1) + ec = self.attributes.get("ec", 0) + self.set_attribute("ec", ec + 1) - if tags is not None and isinstance(tags, dict): - for key in tags: - self.set_tag(key, tags[key]) + if attributes is not None and isinstance(attributes, dict): + for key in attributes: + self.set_attribute(key, attributes[key]) except Exception: - logger.debug('span.mark_as_errored', exc_info=True) + logger.debug("span.mark_as_errored", exc_info=True) - def assure_errored(self): + def assure_errored(self) -> None: """ Make sure that this span is marked as errored. @return: None """ try: - ec = self.tags.get('ec', None) + ec = self.attributes.get("ec", None) if ec is None or ec == 0: - self.set_tag('ec', 1) + self.set_attribute("ec", 1) except Exception: - logger.debug('span.assure_errored', exc_info=True) + logger.debug("span.assure_errored", exc_info=True) - def log_exception(self, exc): - """ - Log an exception onto this span. This will log pertinent info from the exception and - assure that this span is marked as errored. - @param e: the exception to log - """ - try: - message = "" - self.mark_as_errored() - if hasattr(exc, '__str__') and len(str(exc)) > 0: - message = str(exc) - elif hasattr(exc, 'message') and exc.message is not None: - message = exc.message - else: - message = repr(exc) - - if self.operation_name in ['rpc-server', 'rpc-client']: - self.set_tag('rpc.error', message) - elif self.operation_name == "mysql": - self.set_tag('mysql.error', message) - elif self.operation_name == "postgres": - self.set_tag('pg.error', message) - elif self.operation_name in RegisteredSpan.HTTP_SPANS: - self.set_tag('http.error', message) - elif self.operation_name in ["celery-client", "celery-worker"]: - self.set_tag('error', message) - elif self.operation_name == "sqlalchemy": - self.set_tag('sqlalchemy.err', message) - elif self.operation_name == "aws.lambda.entry": - self.set_tag('lambda.error', message) - else: - self.log_kv({'message': message}) - except Exception: - logger.debug("span.log_exception", exc_info=True) - raise +INVALID_SPAN_CONTEXT = SpanContext( + trace_id=INVALID_TRACE_ID, + span_id=INVALID_SPAN_ID, + is_remote=False, + trace_flags=DEFAULT_TRACE_OPTIONS, + trace_state=DEFAULT_TRACE_STATE, +) +INVALID_SPAN = NonRecordingSpan(INVALID_SPAN_CONTEXT) + + +def get_current_span(context: Optional[Context] = None) -> InstanaSpan: + """Retrieve the current span. + + Args: + context: A Context object. If one is not passed, the + default current context is used instead. + + Returns: + The Span set in the context if it exists. INVALID_SPAN otherwise. + """ + span = get_value(_SPAN_KEY, context=context) + if span is None or not isinstance(span, InstanaSpan): + return INVALID_SPAN + return span class BaseSpan(object): sy = None - def __str__(self): + def __str__(self) -> str: return "BaseSpan(%s)" % self.__dict__.__str__() - def __repr__(self): + def __repr__(self) -> str: return self.__dict__.__str__() - def __init__(self, span, source, service_name, **kwargs): + def __init__(self, span, source, service_name, **kwargs) -> None: # pylint: disable=invalid-name self.t = span.context.trace_id self.p = span.parent_id + # self.p = span.context.span_id if span.context.is_remote else None self.s = span.context.span_id self.l = span.context.level - self.ts = int(round(span.start_time * 1000)) - self.d = int(round(span.duration * 1000)) + self.ts = round(span.start_time / 10**6) + self.d = round(span.duration / 10**6) self.f = source - self.ec = span.tags.pop('ec', None) + self.ec = span.attributes.pop("ec", None) self.data = DictionaryOfStan() self.stack = span.stack @@ -119,7 +329,7 @@ def __init__(self, span, source, service_name, **kwargs): self.__dict__.update(kwargs) - def _populate_extra_span_attributes(self, span): + def _populate_extra_span_attributes(self, span) -> None: if span.context.trace_parent: self.tp = span.context.trace_parent if span.context.instana_ancestor: @@ -131,60 +341,70 @@ def _populate_extra_span_attributes(self, span): if span.context.correlation_id: self.crid = span.context.correlation_id - def _validate_tags(self, tags): + def _validate_attributes(self, attributes): """ - This method will loop through a set of tags to validate each key and value. + This method will loop through a set of attributes to validate each key and value. - :param tags: dict of tags - :return: dict - a filtered set of tags + :param attributes: dict of attributes + :return: dict - a filtered set of attributes """ - filtered_tags = DictionaryOfStan() - for key in tags.keys(): - validated_key, validated_value = self._validate_tag(key, tags[key]) + filtered_attributes = DictionaryOfStan() + for key in attributes.keys(): + validated_key, validated_value = self._validate_attribute( + key, attributes[key] + ) if validated_key is not None and validated_value is not None: - filtered_tags[validated_key] = validated_value - return filtered_tags + filtered_attributes[validated_key] = validated_value + return filtered_attributes - def _validate_tag(self, key, value): + def _validate_attribute(self, key, value): """ - This method will assure that and are valid to set as a tag. + This method will assure that and are valid to set as a attribute. If fails the check, an attempt will be made to convert it into something useful. - On check failure, this method will return None values indicating that the tag is + On check failure, this method will return None values indicating that the attribute is not valid and could not be converted into something useful - :param key: The tag key - :param value: The tag value + :param key: The attribute key + :param value: The attribute value :return: Tuple (key, value) """ validated_key = None validated_value = None try: - # Tag keys must be some type of text or string type + # Attribute keys must be some type of text or string type if isinstance(key, (six.text_type, six.string_types)): validated_key = key[0:1024] # Max key length of 1024 characters - if isinstance(value, (bool, float, int, list, dict, six.text_type, six.string_types)): + if isinstance( + value, + (bool, float, int, list, dict, six.text_type, six.string_types), + ): validated_value = value else: - validated_value = self._convert_tag_value(value) + validated_value = self._convert_attribute_value(value) else: - logger.debug("(non-fatal) tag names must be strings. tag discarded for %s", type(key)) + logger.debug( + "(non-fatal) attribute names must be strings. attribute discarded for %s", + type(key), + ) except Exception: - logger.debug("instana.span._validate_tag: ", exc_info=True) + logger.debug("instana.span._validate_attribute: ", exc_info=True) return (validated_key, validated_value) - def _convert_tag_value(self, value): + def _convert_attribute_value(self, value): final_value = None try: final_value = repr(value) except Exception: - final_value = "(non-fatal) span.set_tag: values must be one of these types: bool, float, int, list, " \ - "set, str or alternatively support 'repr'. tag discarded" + final_value = ( + "(non-fatal) span.set_attribute: values must be one of these types: bool, float, int, list, " + "set, str or alternatively support 'repr'. attribute discarded" + ) logger.debug(final_value, exc_info=True) return None return final_value @@ -194,7 +414,7 @@ class SDKSpan(BaseSpan): ENTRY_KIND = ["entry", "server", "consumer"] EXIT_KIND = ["exit", "client", "producer"] - def __init__(self, span, source, service_name, **kwargs): + def __init__(self, span, source, service_name, **kwargs) -> None: # pylint: disable=invalid-name super(SDKSpan, self).__init__(span, source, service_name, **kwargs) @@ -206,306 +426,417 @@ def __init__(self, span, source, service_name, **kwargs): if service_name is not None: self.data["service"] = service_name - self.data["sdk"]["name"] = span.operation_name + self.data["sdk"]["name"] = span.name self.data["sdk"]["type"] = span_kind[0] - self.data["sdk"]["custom"]["tags"] = self._validate_tags(span.tags) + self.data["sdk"]["custom"]["attributes"] = self._validate_attributes( + span.attributes + ) - if span.logs is not None and len(span.logs) > 0: - logs = DictionaryOfStan() - for log in span.logs: - filtered_key_values = self._validate_tags(log.key_values) - if len(filtered_key_values.keys()) > 0: - logs[repr(log.timestamp)] = filtered_key_values - self.data["sdk"]["custom"]["logs"] = logs + if span.events is not None and len(span.events) > 0: + events = DictionaryOfStan() + for event in span.events: + filtered_attributes = self._validate_attributes(event.attributes) + if len(filtered_attributes.keys()) > 0: + events[repr(event.timestamp)] = filtered_attributes + self.data["sdk"]["custom"]["events"] = events - if "arguments" in span.tags: - self.data['sdk']['arguments'] = span.tags["arguments"] + if "arguments" in span.attributes: + self.data["sdk"]["arguments"] = span.attributes["arguments"] - if "return" in span.tags: - self.data['sdk']['return'] = span.tags["return"] + if "return" in span.attributes: + self.data["sdk"]["return"] = span.attributes["return"] - if len(span.context.baggage) > 0: - self.data["baggage"] = span.context.baggage + # if len(span.context.baggage) > 0: + # self.data["baggage"] = span.context.baggage - def get_span_kind(self, span): + def get_span_kind(self, span) -> Tuple[str, int]: """ - Will retrieve the `span.kind` tag and return a tuple containing the appropriate string and integer + Will retrieve the `span.kind` attribute and return a tuple containing the appropriate string and integer values for the Instana backend - :param span: The span to search for the `span.kind` tag + :param span: The span to search for the `span.kind` attribute :return: Tuple (String, Int) """ kind = ("intermediate", 3) - if "span.kind" in span.tags: - if span.tags["span.kind"] in self.ENTRY_KIND: + if "span.kind" in span.attributes: + if span.attributes["span.kind"] in self.ENTRY_KIND: kind = ("entry", 1) - elif span.tags["span.kind"] in self.EXIT_KIND: + elif span.attributes["span.kind"] in self.EXIT_KIND: kind = ("exit", 2) return kind class RegisteredSpan(BaseSpan): - HTTP_SPANS = ("aiohttp-client", "aiohttp-server", "django", "http", "tornado-client", - "tornado-server", "urllib3", "wsgi", "asgi") - - EXIT_SPANS = ("aiohttp-client", "boto3", "cassandra", "celery-client", "couchbase", "log", "memcache", - "mongo", "mysql", "postgres", "rabbitmq", "redis", "rpc-client", "sqlalchemy", - "tornado-client", "urllib3", "pymongo", "gcs", "gcps-producer") - - ENTRY_SPANS = ("aiohttp-server", "aws.lambda.entry", "celery-worker", "django", "wsgi", "rabbitmq", - "rpc-server", "tornado-server", "gcps-consumer", "asgi") - - LOCAL_SPANS = ("render") - - def __init__(self, span, source, service_name, **kwargs): + HTTP_SPANS = ( + "aiohttp-client", + "aiohttp-server", + "django", + "http", + "tornado-client", + "tornado-server", + "urllib3", + "wsgi", + "asgi", + ) + + EXIT_SPANS = ( + "aiohttp-client", + "boto3", + "cassandra", + "celery-client", + "couchbase", + "log", + "memcache", + "mongo", + "mysql", + "postgres", + "rabbitmq", + "redis", + "rpc-client", + "sqlalchemy", + "tornado-client", + "urllib3", + "pymongo", + "gcs", + "gcps-producer", + ) + + ENTRY_SPANS = ( + "aiohttp-server", + "aws.lambda.entry", + "celery-worker", + "django", + "wsgi", + "rabbitmq", + "rpc-server", + "tornado-server", + "gcps-consumer", + "asgi", + ) + + LOCAL_SPANS = "render" + + def __init__(self, span, source, service_name, **kwargs) -> None: # pylint: disable=invalid-name super(RegisteredSpan, self).__init__(span, source, service_name, **kwargs) - self.n = span.operation_name + self.n = span.name self.k = 1 self.data["service"] = service_name - if span.operation_name in self.ENTRY_SPANS: + if span.name in self.ENTRY_SPANS: # entry self._populate_entry_span_data(span) self._populate_extra_span_attributes(span) - elif span.operation_name in self.EXIT_SPANS: + elif span.name in self.EXIT_SPANS: self.k = 2 # exit self._populate_exit_span_data(span) - elif span.operation_name in self.LOCAL_SPANS: + elif span.name in self.LOCAL_SPANS: self.k = 3 # intermediate span self._populate_local_span_data(span) if "rabbitmq" in self.data and self.data["rabbitmq"]["sort"] == "publish": self.k = 2 # exit - # unify the span operation_name for gcps-producer and gcps-consumer - if "gcps" in span.operation_name: - self.n = 'gcps' + # unify the span name for gcps-producer and gcps-consumer + if "gcps" in span.name: + self.n = "gcps" - # Store any leftover tags in the custom section - if len(span.tags) > 0: - self.data["custom"]["tags"] = self._validate_tags(span.tags) + # Store any leftover attributes in the custom section + if len(span.attributes) > 0: + self.data["custom"]["attributes"] = self._validate_attributes( + span.attributes + ) - def _populate_entry_span_data(self, span): - if span.operation_name in self.HTTP_SPANS: - self._collect_http_tags(span) + def _populate_entry_span_data(self, span) -> None: + if span.name in self.HTTP_SPANS: + self._collect_http_attributes(span) - elif span.operation_name == "aws.lambda.entry": - self.data["lambda"]["arn"] = span.tags.pop('lambda.arn', "Unknown") + elif span.name == "aws.lambda.entry": + self.data["lambda"]["arn"] = span.attributes.pop("lambda.arn", "Unknown") self.data["lambda"]["alias"] = None self.data["lambda"]["runtime"] = "python" - self.data["lambda"]["functionName"] = span.tags.pop('lambda.name', "Unknown") - self.data["lambda"]["functionVersion"] = span.tags.pop('lambda.version', "Unknown") - self.data["lambda"]["trigger"] = span.tags.pop('lambda.trigger', None) - self.data["lambda"]["error"] = span.tags.pop('lambda.error', None) + self.data["lambda"]["functionName"] = span.attributes.pop( + "lambda.name", "Unknown" + ) + self.data["lambda"]["functionVersion"] = span.attributes.pop( + "lambda.version", "Unknown" + ) + self.data["lambda"]["trigger"] = span.attributes.pop("lambda.trigger", None) + self.data["lambda"]["error"] = span.attributes.pop("lambda.error", None) trigger_type = self.data["lambda"]["trigger"] if trigger_type in ["aws:api.gateway", "aws:application.load.balancer"]: - self._collect_http_tags(span) - elif trigger_type == 'aws:cloudwatch.events': - self.data["lambda"]["cw"]["events"]["id"] = span.tags.pop('data.lambda.cw.events.id', None) - self.data["lambda"]["cw"]["events"]["more"] = span.tags.pop('lambda.cw.events.more', False) - self.data["lambda"]["cw"]["events"]["resources"] = span.tags.pop('lambda.cw.events.resources', None) - - elif trigger_type == 'aws:cloudwatch.logs': - self.data["lambda"]["cw"]["logs"]["group"] = span.tags.pop('lambda.cw.logs.group', None) - self.data["lambda"]["cw"]["logs"]["stream"] = span.tags.pop('lambda.cw.logs.stream', None) - self.data["lambda"]["cw"]["logs"]["more"] = span.tags.pop('lambda.cw.logs.more', None) - self.data["lambda"]["cw"]["logs"]["events"] = span.tags.pop('lambda.cw.logs.events', None) - - elif trigger_type == 'aws:s3': - self.data["lambda"]["s3"]["events"] = span.tags.pop('lambda.s3.events', None) - elif trigger_type == 'aws:sqs': - self.data["lambda"]["sqs"]["messages"] = span.tags.pop('lambda.sqs.messages', None) - - elif span.operation_name == "celery-worker": - self.data["celery"]["task"] = span.tags.pop('task', None) - self.data["celery"]["task_id"] = span.tags.pop('task_id', None) - self.data["celery"]["scheme"] = span.tags.pop('scheme', None) - self.data["celery"]["host"] = span.tags.pop('host', None) - self.data["celery"]["port"] = span.tags.pop('port', None) - self.data["celery"]["retry-reason"] = span.tags.pop('retry-reason', None) - self.data["celery"]["error"] = span.tags.pop('error', None) - - elif span.operation_name == "gcps-consumer": - self.data["gcps"]["op"] = span.tags.pop('gcps.op', None) - self.data["gcps"]["projid"] = span.tags.pop('gcps.projid', None) - self.data["gcps"]["sub"] = span.tags.pop('gcps.sub', None) - - elif span.operation_name == "rabbitmq": - self.data["rabbitmq"]["exchange"] = span.tags.pop('exchange', None) - self.data["rabbitmq"]["queue"] = span.tags.pop('queue', None) - self.data["rabbitmq"]["sort"] = span.tags.pop('sort', None) - self.data["rabbitmq"]["address"] = span.tags.pop('address', None) - self.data["rabbitmq"]["key"] = span.tags.pop('key', None) - - elif span.operation_name == "rpc-server": - self.data["rpc"]["flavor"] = span.tags.pop('rpc.flavor', None) - self.data["rpc"]["host"] = span.tags.pop('rpc.host', None) - self.data["rpc"]["port"] = span.tags.pop('rpc.port', None) - self.data["rpc"]["call"] = span.tags.pop('rpc.call', None) - self.data["rpc"]["call_type"] = span.tags.pop('rpc.call_type', None) - self.data["rpc"]["params"] = span.tags.pop('rpc.params', None) - self.data["rpc"]["baggage"] = span.tags.pop('rpc.baggage', None) - self.data["rpc"]["error"] = span.tags.pop('rpc.error', None) + self._collect_http_attributes(span) + elif trigger_type == "aws:cloudwatch.events": + self.data["lambda"]["cw"]["events"]["id"] = span.attributes.pop( + "data.lambda.cw.events.id", None + ) + self.data["lambda"]["cw"]["events"]["more"] = span.attributes.pop( + "lambda.cw.events.more", False + ) + self.data["lambda"]["cw"]["events"]["resources"] = span.attributes.pop( + "lambda.cw.events.resources", None + ) + + elif trigger_type == "aws:cloudwatch.logs": + self.data["lambda"]["cw"]["logs"]["group"] = span.attributes.pop( + "lambda.cw.logs.group", None + ) + self.data["lambda"]["cw"]["logs"]["stream"] = span.attributes.pop( + "lambda.cw.logs.stream", None + ) + self.data["lambda"]["cw"]["logs"]["more"] = span.attributes.pop( + "lambda.cw.logs.more", None + ) + self.data["lambda"]["cw"]["logs"]["events"] = span.attributes.pop( + "lambda.cw.logs.events", None + ) + + elif trigger_type == "aws:s3": + self.data["lambda"]["s3"]["events"] = span.attributes.pop( + "lambda.s3.events", None + ) + elif trigger_type == "aws:sqs": + self.data["lambda"]["sqs"]["messages"] = span.attributes.pop( + "lambda.sqs.messages", None + ) + + elif span.name == "celery-worker": + self.data["celery"]["task"] = span.attributes.pop("task", None) + self.data["celery"]["task_id"] = span.attributes.pop("task_id", None) + self.data["celery"]["scheme"] = span.attributes.pop("scheme", None) + self.data["celery"]["host"] = span.attributes.pop("host", None) + self.data["celery"]["port"] = span.attributes.pop("port", None) + self.data["celery"]["retry-reason"] = span.attributes.pop( + "retry-reason", None + ) + self.data["celery"]["error"] = span.attributes.pop("error", None) + + elif span.name == "gcps-consumer": + self.data["gcps"]["op"] = span.attributes.pop("gcps.op", None) + self.data["gcps"]["projid"] = span.attributes.pop("gcps.projid", None) + self.data["gcps"]["sub"] = span.attributes.pop("gcps.sub", None) + + elif span.name == "rabbitmq": + self.data["rabbitmq"]["exchange"] = span.attributes.pop("exchange", None) + self.data["rabbitmq"]["queue"] = span.attributes.pop("queue", None) + self.data["rabbitmq"]["sort"] = span.attributes.pop("sort", None) + self.data["rabbitmq"]["address"] = span.attributes.pop("address", None) + self.data["rabbitmq"]["key"] = span.attributes.pop("key", None) + + elif span.name == "rpc-server": + self.data["rpc"]["flavor"] = span.attributes.pop("rpc.flavor", None) + self.data["rpc"]["host"] = span.attributes.pop("rpc.host", None) + self.data["rpc"]["port"] = span.attributes.pop("rpc.port", None) + self.data["rpc"]["call"] = span.attributes.pop("rpc.call", None) + self.data["rpc"]["call_type"] = span.attributes.pop("rpc.call_type", None) + self.data["rpc"]["params"] = span.attributes.pop("rpc.params", None) + # self.data["rpc"]["baggage"] = span.attributes.pop("rpc.baggage", None) + self.data["rpc"]["error"] = span.attributes.pop("rpc.error", None) else: - logger.debug("SpanRecorder: Unknown entry span: %s" % span.operation_name) - - def _populate_local_span_data(self, span): - if span.operation_name == "render": - self.data["render"]["name"] = span.tags.pop('name', None) - self.data["render"]["type"] = span.tags.pop('type', None) - self.data["log"]["message"] = span.tags.pop('message', None) - self.data["log"]["parameters"] = span.tags.pop('parameters', None) + logger.debug("SpanRecorder: Unknown entry span: %s" % span.name) + + def _populate_local_span_data(self, span) -> None: + if span.name == "render": + self.data["render"]["name"] = span.attributes.pop("name", None) + self.data["render"]["type"] = span.attributes.pop("type", None) + self.data["event"]["message"] = span.attributes.pop("message", None) + self.data["event"]["parameters"] = span.attributes.pop("parameters", None) else: - logger.debug("SpanRecorder: Unknown local span: %s" % span.operation_name) + logger.debug("SpanRecorder: Unknown local span: %s" % span.name) - def _populate_exit_span_data(self, span): - if span.operation_name in self.HTTP_SPANS: - self._collect_http_tags(span) + def _populate_exit_span_data(self, span) -> None: + if span.name in self.HTTP_SPANS: + self._collect_http_attributes(span) - elif span.operation_name == "boto3": - # boto3 also sends http tags - self._collect_http_tags(span) + elif span.name == "boto3": + # boto3 also sends http attributes + self._collect_http_attributes(span) - for tag in ['op', 'ep', 'reg', 'payload', 'error']: - value = span.tags.pop(tag, None) + for attribute in ["op", "ep", "reg", "payload", "error"]: + value = span.attributes.pop(attribute, None) if value is not None: - if tag == 'payload': - self.data["boto3"][tag] = self._validate_tags(value) + if attribute == "payload": + self.data["boto3"][attribute] = self._validate_attributes(value) else: - self.data["boto3"][tag] = value - - elif span.operation_name == "cassandra": - self.data["cassandra"]["cluster"] = span.tags.pop('cassandra.cluster', None) - self.data["cassandra"]["query"] = span.tags.pop('cassandra.query', None) - self.data["cassandra"]["keyspace"] = span.tags.pop('cassandra.keyspace', None) - self.data["cassandra"]["fetchSize"] = span.tags.pop('cassandra.fetchSize', None) - self.data["cassandra"]["achievedConsistency"] = span.tags.pop('cassandra.achievedConsistency', None) - self.data["cassandra"]["triedHosts"] = span.tags.pop('cassandra.triedHosts', None) - self.data["cassandra"]["fullyFetched"] = span.tags.pop('cassandra.fullyFetched', None) - self.data["cassandra"]["error"] = span.tags.pop('cassandra.error', None) - - elif span.operation_name == "celery-client": - self.data["celery"]["task"] = span.tags.pop('task', None) - self.data["celery"]["task_id"] = span.tags.pop('task_id', None) - self.data["celery"]["scheme"] = span.tags.pop('scheme', None) - self.data["celery"]["host"] = span.tags.pop('host', None) - self.data["celery"]["port"] = span.tags.pop('port', None) - self.data["celery"]["error"] = span.tags.pop('error', None) - - elif span.operation_name == "couchbase": - self.data["couchbase"]["hostname"] = span.tags.pop('couchbase.hostname', None) - self.data["couchbase"]["bucket"] = span.tags.pop('couchbase.bucket', None) - self.data["couchbase"]["type"] = span.tags.pop('couchbase.type', None) - self.data["couchbase"]["error"] = span.tags.pop('couchbase.error', None) - self.data["couchbase"]["error_type"] = span.tags.pop('couchbase.error_type', None) - self.data["couchbase"]["sql"] = span.tags.pop('couchbase.sql', None) - - elif span.operation_name == "rabbitmq": - self.data["rabbitmq"]["exchange"] = span.tags.pop('exchange', None) - self.data["rabbitmq"]["queue"] = span.tags.pop('queue', None) - self.data["rabbitmq"]["sort"] = span.tags.pop('sort', None) - self.data["rabbitmq"]["address"] = span.tags.pop('address', None) - self.data["rabbitmq"]["key"] = span.tags.pop('key', None) - - elif span.operation_name == "redis": - self.data["redis"]["connection"] = span.tags.pop('connection', None) - self.data["redis"]["driver"] = span.tags.pop('driver', None) - self.data["redis"]["command"] = span.tags.pop('command', None) - self.data["redis"]["error"] = span.tags.pop('redis.error', None) - self.data["redis"]["subCommands"] = span.tags.pop('subCommands', None) - - elif span.operation_name == "rpc-client": - self.data["rpc"]["flavor"] = span.tags.pop('rpc.flavor', None) - self.data["rpc"]["host"] = span.tags.pop('rpc.host', None) - self.data["rpc"]["port"] = span.tags.pop('rpc.port', None) - self.data["rpc"]["call"] = span.tags.pop('rpc.call', None) - self.data["rpc"]["call_type"] = span.tags.pop('rpc.call_type', None) - self.data["rpc"]["params"] = span.tags.pop('rpc.params', None) - self.data["rpc"]["baggage"] = span.tags.pop('rpc.baggage', None) - self.data["rpc"]["error"] = span.tags.pop('rpc.error', None) - - elif span.operation_name == "sqlalchemy": - self.data["sqlalchemy"]["sql"] = span.tags.pop('sqlalchemy.sql', None) - self.data["sqlalchemy"]["eng"] = span.tags.pop('sqlalchemy.eng', None) - self.data["sqlalchemy"]["url"] = span.tags.pop('sqlalchemy.url', None) - self.data["sqlalchemy"]["err"] = span.tags.pop('sqlalchemy.err', None) - - elif span.operation_name == "mysql": - self.data["mysql"]["host"] = span.tags.pop('host', None) - self.data["mysql"]["port"] = span.tags.pop('port', None) - self.data["mysql"]["db"] = span.tags.pop(ot_tags.DATABASE_INSTANCE, None) - self.data["mysql"]["user"] = span.tags.pop(ot_tags.DATABASE_USER, None) - self.data["mysql"]["stmt"] = span.tags.pop(ot_tags.DATABASE_STATEMENT, None) - self.data["mysql"]["error"] = span.tags.pop('mysql.error', None) - - elif span.operation_name == "postgres": - self.data["pg"]["host"] = span.tags.pop('host', None) - self.data["pg"]["port"] = span.tags.pop('port', None) - self.data["pg"]["db"] = span.tags.pop(ot_tags.DATABASE_INSTANCE, None) - self.data["pg"]["user"] = span.tags.pop(ot_tags.DATABASE_USER, None) - self.data["pg"]["stmt"] = span.tags.pop(ot_tags.DATABASE_STATEMENT, None) - self.data["pg"]["error"] = span.tags.pop('pg.error', None) - - elif span.operation_name == "mongo": - service = "%s:%s" % (span.tags.pop('host', None), span.tags.pop('port', None)) - namespace = "%s.%s" % (span.tags.pop('db', "?"), span.tags.pop('collection', "?")) + self.data["boto3"][attribute] = value + + elif span.name == "cassandra": + self.data["cassandra"]["cluster"] = span.attributes.pop( + "cassandra.cluster", None + ) + self.data["cassandra"]["query"] = span.attributes.pop( + "cassandra.query", None + ) + self.data["cassandra"]["keyspace"] = span.attributes.pop( + "cassandra.keyspace", None + ) + self.data["cassandra"]["fetchSize"] = span.attributes.pop( + "cassandra.fetchSize", None + ) + self.data["cassandra"]["achievedConsistency"] = span.attributes.pop( + "cassandra.achievedConsistency", None + ) + self.data["cassandra"]["triedHosts"] = span.attributes.pop( + "cassandra.triedHosts", None + ) + self.data["cassandra"]["fullyFetched"] = span.attributes.pop( + "cassandra.fullyFetched", None + ) + self.data["cassandra"]["error"] = span.attributes.pop( + "cassandra.error", None + ) + + elif span.name == "celery-client": + self.data["celery"]["task"] = span.attributes.pop("task", None) + self.data["celery"]["task_id"] = span.attributes.pop("task_id", None) + self.data["celery"]["scheme"] = span.attributes.pop("scheme", None) + self.data["celery"]["host"] = span.attributes.pop("host", None) + self.data["celery"]["port"] = span.attributes.pop("port", None) + self.data["celery"]["error"] = span.attributes.pop("error", None) + + elif span.name == "couchbase": + self.data["couchbase"]["hostname"] = span.attributes.pop( + "couchbase.hostname", None + ) + self.data["couchbase"]["bucket"] = span.attributes.pop( + "couchbase.bucket", None + ) + self.data["couchbase"]["type"] = span.attributes.pop("couchbase.type", None) + self.data["couchbase"]["error"] = span.attributes.pop( + "couchbase.error", None + ) + self.data["couchbase"]["error_type"] = span.attributes.pop( + "couchbase.error_type", None + ) + self.data["couchbase"]["sql"] = span.attributes.pop("couchbase.sql", None) + + elif span.name == "rabbitmq": + self.data["rabbitmq"]["exchange"] = span.attributes.pop("exchange", None) + self.data["rabbitmq"]["queue"] = span.attributes.pop("queue", None) + self.data["rabbitmq"]["sort"] = span.attributes.pop("sort", None) + self.data["rabbitmq"]["address"] = span.attributes.pop("address", None) + self.data["rabbitmq"]["key"] = span.attributes.pop("key", None) + + elif span.name == "redis": + self.data["redis"]["connection"] = span.attributes.pop("connection", None) + self.data["redis"]["driver"] = span.attributes.pop("driver", None) + self.data["redis"]["command"] = span.attributes.pop("command", None) + self.data["redis"]["error"] = span.attributes.pop("redis.error", None) + self.data["redis"]["subCommands"] = span.attributes.pop("subCommands", None) + + elif span.name == "rpc-client": + self.data["rpc"]["flavor"] = span.attributes.pop("rpc.flavor", None) + self.data["rpc"]["host"] = span.attributes.pop("rpc.host", None) + self.data["rpc"]["port"] = span.attributes.pop("rpc.port", None) + self.data["rpc"]["call"] = span.attributes.pop("rpc.call", None) + self.data["rpc"]["call_type"] = span.attributes.pop("rpc.call_type", None) + self.data["rpc"]["params"] = span.attributes.pop("rpc.params", None) + # self.data["rpc"]["baggage"] = span.attributes.pop("rpc.baggage", None) + self.data["rpc"]["error"] = span.attributes.pop("rpc.error", None) + + elif span.name == "sqlalchemy": + self.data["sqlalchemy"]["sql"] = span.attributes.pop("sqlalchemy.sql", None) + self.data["sqlalchemy"]["eng"] = span.attributes.pop("sqlalchemy.eng", None) + self.data["sqlalchemy"]["url"] = span.attributes.pop("sqlalchemy.url", None) + self.data["sqlalchemy"]["err"] = span.attributes.pop("sqlalchemy.err", None) + + elif span.name == "mysql": + self.data["mysql"]["host"] = span.attributes.pop("host", None) + self.data["mysql"]["port"] = span.attributes.pop("port", None) + self.data["mysql"]["db"] = span.attributes.pop("db.instance", None) + self.data["mysql"]["user"] = span.attributes.pop("db.user", None) + self.data["mysql"]["stmt"] = span.attributes.pop("db.statement", None) + self.data["mysql"]["error"] = span.attributes.pop("mysql.error", None) + + elif span.name == "postgres": + self.data["pg"]["host"] = span.attributes.pop("host", None) + self.data["pg"]["port"] = span.attributes.pop("port", None) + self.data["pg"]["db"] = span.attributes.pop("db.instance", None) + self.data["pg"]["user"] = span.attributes.pop("db.user", None) + self.data["pg"]["stmt"] = span.attributes.pop("db.statement", None) + self.data["pg"]["error"] = span.attributes.pop("pg.error", None) + + elif span.name == "mongo": + service = "%s:%s" % ( + span.attributes.pop("host", None), + span.attributes.pop("port", None), + ) + namespace = "%s.%s" % ( + span.attributes.pop("db", "?"), + span.attributes.pop("collection", "?"), + ) self.data["mongo"]["service"] = service self.data["mongo"]["namespace"] = namespace - self.data["mongo"]["command"] = span.tags.pop('command', None) - self.data["mongo"]["filter"] = span.tags.pop('filter', None) - self.data["mongo"]["json"] = span.tags.pop('json', None) - self.data["mongo"]["error"] = span.tags.pop('error', None) - - elif span.operation_name == "gcs": - self.data["gcs"]["op"] = span.tags.pop('gcs.op') - self.data["gcs"]["bucket"] = span.tags.pop('gcs.bucket', None) - self.data["gcs"]["object"] = span.tags.pop('gcs.object', None) - self.data["gcs"]["entity"] = span.tags.pop('gcs.entity', None) - self.data["gcs"]["range"] = span.tags.pop('gcs.range', None) - self.data["gcs"]["sourceBucket"] = span.tags.pop('gcs.sourceBucket', None) - self.data["gcs"]["sourceObject"] = span.tags.pop('gcs.sourceObject', None) - self.data["gcs"]["sourceObjects"] = span.tags.pop('gcs.sourceObjects', None) - self.data["gcs"]["destinationBucket"] = span.tags.pop('gcs.destinationBucket', None) - self.data["gcs"]["destinationObject"] = span.tags.pop('gcs.destinationObject', None) - self.data["gcs"]["numberOfOperations"] = span.tags.pop('gcs.numberOfOperations', None) - self.data["gcs"]["projectId"] = span.tags.pop('gcs.projectId', None) - self.data["gcs"]["accessId"] = span.tags.pop('gcs.accessId', None) - - elif span.operation_name == "gcps-producer": - self.data["gcps"]["op"] = span.tags.pop('gcps.op', None) - self.data["gcps"]["projid"] = span.tags.pop('gcps.projid', None) - self.data["gcps"]["top"] = span.tags.pop('gcps.top', None) - - elif span.operation_name == "log": + self.data["mongo"]["command"] = span.attributes.pop("command", None) + self.data["mongo"]["filter"] = span.attributes.pop("filter", None) + self.data["mongo"]["json"] = span.attributes.pop("json", None) + self.data["mongo"]["error"] = span.attributes.pop("error", None) + + elif span.name == "gcs": + self.data["gcs"]["op"] = span.attributes.pop("gcs.op", None) + self.data["gcs"]["bucket"] = span.attributes.pop("gcs.bucket", None) + self.data["gcs"]["object"] = span.attributes.pop("gcs.object", None) + self.data["gcs"]["entity"] = span.attributes.pop("gcs.entity", None) + self.data["gcs"]["range"] = span.attributes.pop("gcs.range", None) + self.data["gcs"]["sourceBucket"] = span.attributes.pop( + "gcs.sourceBucket", None + ) + self.data["gcs"]["sourceObject"] = span.attributes.pop( + "gcs.sourceObject", None + ) + self.data["gcs"]["sourceObjects"] = span.attributes.pop( + "gcs.sourceObjects", None + ) + self.data["gcs"]["destinationBucket"] = span.attributes.pop( + "gcs.destinationBucket", None + ) + self.data["gcs"]["destinationObject"] = span.attributes.pop( + "gcs.destinationObject", None + ) + self.data["gcs"]["numberOfOperations"] = span.attributes.pop( + "gcs.numberOfOperations", None + ) + self.data["gcs"]["projectId"] = span.attributes.pop("gcs.projectId", None) + self.data["gcs"]["accessId"] = span.attributes.pop("gcs.accessId", None) + + elif span.name == "gcps-producer": + self.data["gcps"]["op"] = span.attributes.pop("gcps.op", None) + self.data["gcps"]["projid"] = span.attributes.pop("gcps.projid", None) + self.data["gcps"]["top"] = span.attributes.pop("gcps.top", None) + + elif span.name == "log": # use last special key values - for l in span.logs: - if "message" in l.key_values: - self.data["log"]["message"] = l.key_values.pop("message", None) - if "parameters" in l.key_values: - self.data["log"]["parameters"] = l.key_values.pop("parameters", None) + for event in span.events: + if "message" in event.attributes: + self.data["event"]["message"] = event.attributes.pop( + "message", None + ) + if "parameters" in event.attributes: + self.data["event"]["parameters"] = event.attributes.pop( + "parameters", None + ) else: - logger.debug("SpanRecorder: Unknown exit span: %s" % span.operation_name) - - def _collect_http_tags(self, span): - self.data["http"]["host"] = span.tags.pop("http.host", None) - self.data["http"]["url"] = span.tags.pop(ot_tags.HTTP_URL, None) - self.data["http"]["path"] = span.tags.pop("http.path", None) - self.data["http"]["params"] = span.tags.pop('http.params', None) - self.data["http"]["method"] = span.tags.pop(ot_tags.HTTP_METHOD, None) - self.data["http"]["status"] = span.tags.pop(ot_tags.HTTP_STATUS_CODE, None) - self.data["http"]["path_tpl"] = span.tags.pop("http.path_tpl", None) - self.data["http"]["error"] = span.tags.pop('http.error', None) - - if len(span.tags) > 0: + logger.debug("SpanRecorder: Unknown exit span: %s" % span.name) + + def _collect_http_attributes(self, span) -> None: + self.data["http"]["host"] = span.attributes.pop("http.host", None) + self.data["http"]["url"] = span.attributes.pop("http.url", None) + self.data["http"]["path"] = span.attributes.pop("http.path", None) + self.data["http"]["params"] = span.attributes.pop("http.params", None) + self.data["http"]["method"] = span.attributes.pop("http.method", None) + self.data["http"]["status"] = span.attributes.pop("http.status_code", None) + self.data["http"]["path_tpl"] = span.attributes.pop("http.path_tpl", None) + self.data["http"]["error"] = span.attributes.pop("http.error", None) + + if len(span.attributes) > 0: custom_headers = [] - for key in span.tags: + for key in span.attributes: if key[0:12] == "http.header.": custom_headers.append(key) for key in custom_headers: trimmed_key = key[12:] - self.data["http"]["header"][trimmed_key] = span.tags.pop(key) + self.data["http"]["header"][trimmed_key] = span.attributes.pop(key) diff --git a/src/instana/span_context.py b/src/instana/span_context.py index 1c874a35..ac14e61d 100644 --- a/src/instana/span_context.py +++ b/src/instana/span_context.py @@ -2,102 +2,129 @@ # (c) Copyright Instana Inc. 2019 -class SpanContext(): - def __init__( - self, - trace_id=None, - span_id=None, - baggage=None, - sampled=True, - level=1, - synthetic=False - ): - - self.level = level - self.trace_id = trace_id - self.span_id = span_id - self.sampled = sampled - self.synthetic = synthetic - self._baggage = baggage or {} - - self.trace_parent = None # true/false flag - self.instana_ancestor = None - self.long_trace_id = None - self.correlation_type = None - self.correlation_id = None - self.traceparent = None # temporary storage of the validated traceparent header of the incoming request - self.tracestate = None # temporary storage of the tracestate header +import typing + +from opentelemetry.trace import SpanContext as OtelSpanContext +from opentelemetry.trace.span import ( + DEFAULT_TRACE_OPTIONS, + DEFAULT_TRACE_STATE, + TraceFlags, + TraceState, + format_span_id, +) + + +class SpanContext(OtelSpanContext): + """The state of a Span to propagate between processes. + + This class includes the immutable attributes of a :class:`.Span` that must + be propagated to a span's children and across process boundaries. + + Required Args: + trace_id: The ID of the trace that this span belongs to. + span_id: This span's ID. + is_remote: True if propagated from a remote parent. + """ + + def __new__( + cls, + trace_id: int, + span_id: int, + is_remote: bool, + trace_flags: typing.Optional[TraceFlags] = DEFAULT_TRACE_OPTIONS, + trace_state: typing.Optional[TraceState] = DEFAULT_TRACE_STATE, + level=1, + synthetic=False, + trace_parent=None, # true/false flag, + instana_ancestor=None, + long_trace_id=None, + correlation_type=None, + correlation_id=None, + traceparent=None, # temporary storage of the validated traceparent header of the incoming request + tracestate=None, # temporary storage of the tracestate header + **kwargs, + ) -> "SpanContext": + instance = super().__new__(cls, trace_id, span_id, is_remote, trace_flags, trace_state) + return tuple.__new__( + cls, + ( + instance.trace_id, + instance.span_id, + instance.is_remote, + instance.trace_flags, + instance.trace_state, + instance.is_valid, + level, + synthetic, + trace_parent, # true/false flag, + instana_ancestor, + long_trace_id, + correlation_type, + correlation_id, + traceparent, # temporary storage of the validated traceparent header of the incoming request + tracestate, # temporary storage of the tracestate header + ), + ) + + def __getnewargs__( + self, + ): # -> typing.Tuple[int, int, bool, "TraceFlags", "TraceState", int, bool, bool]: + return ( + self.trace_id, + self.span_id, + self.is_remote, + self.trace_flags, + self.trace_state, + self.level, + self.synthetic, + self.trace_parent, + self.instana_ancestor, + self.long_trace_id, + self.correlation_type, + self.correlation_id, + self.traceparent, + self.tracestate, + ) @property - def traceparent(self): - return self._traceparent - - @traceparent.setter - def traceparent(self, value): - self._traceparent = value + def level(self) -> int: + return self[6] @property - def tracestate(self): - return self._tracestate - - @tracestate.setter - def tracestate(self, value): - self._tracestate = value + def synthetic(self) -> bool: + return self[7] @property - def trace_parent(self): - return self._trace_parent - - @trace_parent.setter - def trace_parent(self, value): - self._trace_parent = value + def trace_parent(self) -> bool: + return self[8] @property def instana_ancestor(self): - return self._instana_ancestor - - @instana_ancestor.setter - def instana_ancestor(self, value): - self._instana_ancestor = value + return self[9] @property def long_trace_id(self): - return self._long_trace_id - - @long_trace_id.setter - def long_trace_id(self, value): - self._long_trace_id = value + return self[10] @property def correlation_type(self): - return self._correlation_type - - @correlation_type.setter - def correlation_type(self, value): - self._correlation_type = value + return self[11] @property def correlation_id(self): - return self._correlation_id + return self[12] - @correlation_id.setter - def correlation_id(self, value): - self._correlation_id = value + @property + def traceparent(self): + return self[13] @property - def baggage(self): - return self._baggage + def tracestate(self): + return self[14] @property - def suppression(self): + def suppression(self) -> bool: return self.level == 0 - def with_baggage_item(self, key, value): - new_baggage = self._baggage.copy() - new_baggage[key] = value - return SpanContext( - trace_id=self.trace_id, - span_id=self.span_id, - sampled=self.sampled, - level=self.level, - baggage=new_baggage) + def __repr__(self) -> str: + return f"{type(self).__name__}(trace_id=0x{format_span_id(self.trace_id)}, span_id=0x{format_span_id(self.span_id)}, trace_flags=0x{self.trace_flags:02x}, trace_state={self.trace_state!r}, is_remote={self.is_remote}, synthetic={self.synthetic})" diff --git a/src/instana/tracer.py b/src/instana/tracer.py index 2d1d2def..b8454021 100644 --- a/src/instana/tracer.py +++ b/src/instana/tracer.py @@ -6,135 +6,176 @@ import re import time import traceback +from contextlib import contextmanager +from typing import Iterator, Mapping, Optional, Union + +from opentelemetry.context.context import Context +from opentelemetry.trace import ( + SpanKind, + TraceFlags, + Tracer, + TracerProvider, + _Links, + use_span, +) +from opentelemetry.util import types + +from instana.agent.host import HostAgent +from instana.agent.test import TestAgent +from instana.log import logger +from instana.propagators.base_propagator import CarrierT +from instana.propagators.binary_propagator import BinaryPropagator +from instana.propagators.exceptions import UnsupportedFormatException +from instana.propagators.format import Format +from instana.propagators.http_propagator import HTTPPropagator +from instana.propagators.text_propagator import TextPropagator +from instana.recorder import StanRecorder +from instana.sampling import InstanaSampler, Sampler +from instana.span import InstanaSpan, RegisteredSpan, get_current_span +from instana.span_context import SpanContext +from instana.util.ids import generate_id + + +class InstanaTracerProvider(TracerProvider): + def __init__( + self, + sampler: Optional[Sampler] = None, + recorder: Optional[StanRecorder] = None, + span_processor: Optional[Union[HostAgent, TestAgent]] = None, + ) -> None: + self.sampler = sampler or InstanaSampler() + self.recorder = recorder or StanRecorder() + self._span_processor = span_processor or HostAgent() + self._propagators = {} + self._propagators[Format.HTTP_HEADERS] = HTTPPropagator() + self._propagators[Format.TEXT_MAP] = TextPropagator() + self._propagators[Format.BINARY] = BinaryPropagator() + + def get_tracer( + self, + instrumenting_module_name: str, + instrumenting_library_version: Optional[str] = None, + schema_url: Optional[str] = None, + ) -> Tracer: + if not instrumenting_module_name: # Reject empty strings too. + instrumenting_module_name = "" + logger.error("get_tracer called with missing module name.") + + return InstanaTracer( + self.sampler, + self.recorder, + self._span_processor, + self._propagators, + ) -import opentracing as ot -from basictracer import BasicTracer - -from .util.ids import generate_id -from .span_context import SpanContext -from .span import InstanaSpan, RegisteredSpan -from .recorder import StanRecorder, InstanaSampler -from .propagators.http_propagator import HTTPPropagator -from .propagators.text_propagator import TextPropagator -from .propagators.binary_propagator import BinaryPropagator - - -class InstanaTracer(BasicTracer): - def __init__(self, scope_manager=None, recorder=None): - - if recorder is None: - recorder = StanRecorder() + def add_span_processor( + self, + span_processor: Union[HostAgent, TestAgent], + ) -> None: + """Registers a new SpanProcessor for the TracerProvider.""" + self._span_processor = span_processor + + +class InstanaTracer(Tracer): + """Handles :class:`InstanaSpan` creation and in-process context propagation. + + This class provides methods for manipulating the context, creating spans, + and controlling spans' lifecycles. + """ + + def __init__( + self, + sampler: Sampler, + recorder: StanRecorder, + span_processor: Union[HostAgent, TestAgent], + propagators: Mapping[ + str, Union[BinaryPropagator, HTTPPropagator, TextPropagator] + ], + ) -> None: + self._tracer_id = generate_id() + self._sampler = sampler + self._recorder = recorder + self._span_processor = span_processor + self._propagators = propagators + + @property + def tracer_id(self) -> str: + return self._tracer_id + + @property + def recorder(self) -> Optional[StanRecorder]: + return self._recorder + + def start_span( + self, + name: str, + context: Optional[Context] = None, + kind: SpanKind = SpanKind.INTERNAL, + attributes: types.Attributes = None, + links: _Links = None, + start_time: Optional[int] = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + ) -> InstanaSpan: + parent_context = get_current_span(context).get_span_context() + + if parent_context is not None and not isinstance(parent_context, SpanContext): + raise TypeError("parent_context must be an Instana SpanContext or None.") + + if parent_context is not None and not parent_context.is_valid: + # We probably have a INVALID_SPAN_CONTEXT. + parent_context = None + + span_context = self._create_span_context(parent_context) + span = InstanaSpan( + name, + span_context, + parent_id=(None if parent_context is None else parent_context.span_id), + start_time=(time.time_ns() if start_time is None else start_time), + attributes=attributes, + # events: Sequence[Event] = None, + ) - super(InstanaTracer, self).__init__( - recorder, InstanaSampler(), scope_manager) + if parent_context is not None: + span.synthetic = parent_context.synthetic - self._propagators[ot.Format.HTTP_HEADERS] = HTTPPropagator() - self._propagators[ot.Format.TEXT_MAP] = TextPropagator() - self._propagators[ot.Format.BINARY] = BinaryPropagator() + if name in RegisteredSpan.EXIT_SPANS: + self._add_stack(span) - def start_active_span(self, - operation_name, - child_of=None, - references=None, - tags=None, - start_time=None, - ignore_active_span=False, - finish_on_close=True): + return span - # create a new Span + @contextmanager + def start_as_current_span( + self, + name: str, + context: Optional[Context] = None, + kind: SpanKind = SpanKind.INTERNAL, + attributes: types.Attributes = None, + links: _Links = None, + start_time: Optional[int] = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + end_on_exit: bool = True, + ) -> Iterator[InstanaSpan]: span = self.start_span( - operation_name=operation_name, - child_of=child_of, - references=references, - tags=tags, + name=name, + context=context, + kind=kind, + attributes=attributes, + links=links, start_time=start_time, - ignore_active_span=ignore_active_span, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, ) - - return self.scope_manager.activate(span, finish_on_close) - - def start_span(self, - operation_name=None, - child_of=None, - references=None, - tags=None, - start_time=None, - ignore_active_span=False): - "Taken from BasicTracer so we can override generate_id calls to ours" - - start_time = time.time() if start_time is None else start_time - - # See if we have a parent_ctx in `references` - parent_ctx = None - if child_of is not None: - parent_ctx = ( - child_of if isinstance(child_of, SpanContext) - else child_of.context) - elif references is not None and len(references) > 0: - # TODO only the first reference is currently used - parent_ctx = references[0].referenced_context - - # retrieve the active SpanContext - if not ignore_active_span and parent_ctx is None: - scope = self.scope_manager.active - if scope is not None: - parent_ctx = scope.span.context - - # Assemble the child ctx - gid = generate_id() - ctx = SpanContext(span_id=gid) - if parent_ctx is not None and parent_ctx.trace_id is not None: - if hasattr(parent_ctx, '_baggage') and parent_ctx._baggage is not None: - ctx._baggage = parent_ctx._baggage.copy() - ctx.trace_id = parent_ctx.trace_id - ctx.sampled = parent_ctx.sampled - ctx.long_trace_id = parent_ctx.long_trace_id - ctx.trace_parent = parent_ctx.trace_parent - ctx.instana_ancestor = parent_ctx.instana_ancestor - ctx.level = parent_ctx.level - ctx.correlation_type = parent_ctx.correlation_type - ctx.correlation_id = parent_ctx.correlation_id - ctx.traceparent = parent_ctx.traceparent - ctx.tracestate = parent_ctx.tracestate - else: - ctx.trace_id = gid - ctx.sampled = self.sampler.sampled(ctx.trace_id) - if parent_ctx is not None: - ctx.level = parent_ctx.level - ctx.correlation_type = parent_ctx.correlation_type - ctx.correlation_id = parent_ctx.correlation_id - ctx.traceparent = parent_ctx.traceparent - ctx.tracestate = parent_ctx.tracestate - - # Tie it all together - span = InstanaSpan(self, - operation_name=operation_name, - context=ctx, - parent_id=(None if parent_ctx is None else parent_ctx.span_id), - tags=tags, - start_time=start_time) - - if parent_ctx is not None: - span.synthetic = parent_ctx.synthetic - - if operation_name in RegisteredSpan.EXIT_SPANS: - self.__add_stack(span) - - return span - - def inject(self, span_context, format, carrier, disable_w3c_trace_context=False): - if format in self._propagators: - return self._propagators[format].inject(span_context, carrier, disable_w3c_trace_context) - - raise ot.UnsupportedFormatException() - - def extract(self, format, carrier, disable_w3c_trace_context=False): - if format in self._propagators: - return self._propagators[format].extract(carrier, disable_w3c_trace_context) - - raise ot.UnsupportedFormatException() - - def __add_stack(self, span, limit=30): + with use_span( + span, + end_on_exit=end_on_exit, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + ) as span: + yield span + + def _add_stack(self, span: InstanaSpan, limit: Optional[int] = 30) -> None: """ Adds a backtrace to . The default length limit for stack traces is 30 frames. A hard limit of 40 frames is enforced. @@ -155,23 +196,78 @@ def __add_stack(self, span, limit=30): if re_with_stan_frame.search(frame[2]) is not None: continue - sanitized_stack.append({ - "c": frame[0], - "n": frame[1], - "m": frame[2] - }) + sanitized_stack.append({"c": frame[0], "n": frame[1], "m": frame[2]}) if len(sanitized_stack) > limit: # (limit * -1) gives us negative form of used for # slicing from the end of the list. e.g. stack[-30:] - span.stack = sanitized_stack[(limit*-1):] + span.stack = sanitized_stack[(limit * -1) :] else: span.stack = sanitized_stack except Exception: # No fail pass + def _create_span_context(self, parent_context: SpanContext) -> SpanContext: + """Creates a new SpanContext based on the given parent context.""" + + if parent_context is not None and parent_context.trace_id is not None: + trace_id = parent_context.trace_id + span_id = generate_id() + trace_flags = parent_context.trace_flags + is_remote = parent_context.is_remote + else: + trace_id = self.tracer_id + span_id = self.tracer_id + trace_flags = TraceFlags(self._sampler.sampled()) + is_remote = False + + span_context = SpanContext( + trace_id=trace_id, + span_id=span_id, + trace_flags=trace_flags, + is_remote=is_remote, + level=(parent_context.level if parent_context is not None else 1), + synthetic=False, + ) + + if parent_context is not None: + span_context.long_trace_id = parent_context.long_trace_id + span_context.trace_parent = parent_context.trace_parent + span_context.instana_ancestor = parent_context.instana_ancestor + span_context.correlation_type = parent_context.correlation_type + span_context.correlation_id = parent_context.correlation_id + span_context.traceparent = parent_context.traceparent + span_context.tracestate = parent_context.tracestate + + return span_context + + def inject( + self, + span_context: SpanContext, + format: Union[Format.BINARY, Format.HTTP_HEADERS, Format.TEXT_MAP], + carrier: CarrierT, + disable_w3c_trace_context: bool = False, + ) -> Optional[CarrierT]: + if format in self._propagators: + return self._propagators[format].inject( + span_context, carrier, disable_w3c_trace_context + ) + + raise UnsupportedFormatException() + + def extract( + self, + format: Union[Format.BINARY, Format.HTTP_HEADERS, Format.TEXT_MAP], + carrier: CarrierT, + disable_w3c_trace_context: bool = False, + ) -> Optional[Context]: + if format in self._propagators: + return self._propagators[format].extract(carrier, disable_w3c_trace_context) + + raise UnsupportedFormatException() + # Used by __add_stack re_tracer_frame = re.compile(r"/instana/.*\.py$") -re_with_stan_frame = re.compile('with_instana') +re_with_stan_frame = re.compile("with_instana") diff --git a/src/instana/util/ids.py b/src/instana/util/ids.py index 3d6e8d01..81792027 100644 --- a/src/instana/util/ids.py +++ b/src/instana/util/ids.py @@ -4,30 +4,32 @@ import os import time import random +from typing import Union + +from opentelemetry.trace.span import _SPAN_ID_MAX_VALUE, INVALID_SPAN_ID _rnd = random.Random() _current_pid = 0 -BAD_ID = "BADCAFFE" # Bad Caffe +def generate_id() -> int: + """Get a new ID. -def generate_id(): - """ Generate a 64bit base 16 ID for use as a Span or Trace ID """ + Returns: + A 64-bit int for use as a Span or Trace ID. + """ global _current_pid pid = os.getpid() if _current_pid != pid: _current_pid = pid _rnd.seed(int(1000000 * time.time()) ^ pid) - new_id = format(_rnd.randint(0, 18446744073709551615), '02x') - - if len(new_id) < 16: - new_id = new_id.zfill(16) + new_id = _rnd.randint(0, _SPAN_ID_MAX_VALUE) return new_id -def header_to_long_id(header): +def header_to_long_id(header: Union[bytes, str]) -> int: """ We can receive headers in the following formats: 1. unsigned base 16 hex string (or bytes) of variable length @@ -40,23 +42,19 @@ def header_to_long_id(header): header = header.decode('utf-8') if not isinstance(header, str): - return BAD_ID + return INVALID_SPAN_ID try: - # Test that header is truly a hexadecimal value before we try to convert - int(header, 16) - - length = len(header) - if length < 16: + if len(header) < 16: # Left pad ID with zeros header = header.zfill(16) - return header + return int(header, 16) except ValueError: - return BAD_ID + return INVALID_SPAN_ID -def header_to_id(header): +def header_to_id(header: Union[bytes, str]) -> int: """ We can receive headers in the following formats: 1. unsigned base 16 hex string (or bytes) of variable length @@ -69,12 +67,9 @@ def header_to_id(header): header = header.decode('utf-8') if not isinstance(header, str): - return BAD_ID + return INVALID_SPAN_ID try: - # Test that header is truly a hexadecimal value before we try to convert - int(header, 16) - length = len(header) if length < 16: # Left pad ID with zeros @@ -82,6 +77,7 @@ def header_to_id(header): elif length > 16: # Phase 0: Discard everything but the last 16byte header = header[-16:] - return header + + return int(header, 16) except ValueError: - return BAD_ID + return INVALID_SPAN_ID diff --git a/src/instana/util/traceutils.py b/src/instana/util/traceutils.py index a5b33304..89d3c3be 100644 --- a/src/instana/util/traceutils.py +++ b/src/instana/util/traceutils.py @@ -1,8 +1,12 @@ # (c) Copyright IBM Corp. 2021 # (c) Copyright Instana Inc. 2021 -from ..singletons import agent, tracer, async_tracer, tornado_tracer -from ..log import logger +from typing import Optional, Tuple + +from instana.log import logger +from instana.singletons import agent, tracer, async_tracer, tornado_tracer +from instana.span import InstanaSpan, get_current_span +from instana.tracer import InstanaTracer def extract_custom_headers(tracing_span, headers): @@ -16,14 +20,12 @@ def extract_custom_headers(tracing_span, headers): logger.debug("extract_custom_headers: ", exc_info=True) -def get_active_tracer(): +def get_active_tracer() -> Optional[InstanaTracer]: try: - if tracer.active_span: + # ToDo: Might have to add additional stuff when testing with async and tornado tracer + current_span = get_current_span() + if current_span and current_span.is_recording(): return tracer - elif async_tracer.active_span: - return async_tracer - elif tornado_tracer.active_span: - return tornado_tracer else: return None except Exception: @@ -32,10 +34,13 @@ def get_active_tracer(): return None -def get_tracer_tuple(): +def get_tracer_tuple() -> ( + Tuple[Optional[InstanaTracer], Optional[InstanaSpan], Optional[str]] +): active_tracer = get_active_tracer() + current_span = get_current_span() if active_tracer: - return (active_tracer, active_tracer.active_span, active_tracer.active_span.operation_name) + return (active_tracer, current_span, current_span.name) elif agent.options.allow_exit_as_root: return (tracer, None, None) return (None, None, None) diff --git a/src/instana/version.py b/src/instana/version.py index 83bfbd1c..ee874de0 100644 --- a/src/instana/version.py +++ b/src/instana/version.py @@ -3,4 +3,4 @@ # Module version file. Used by setup.py and snapshot reporting. -VERSION = "2.5.1" +VERSION = "3.0.0.dev0" diff --git a/src/instana/w3c_trace_context/traceparent.py b/src/instana/w3c_trace_context/traceparent.py index 3175c6cf..bc39fd3a 100644 --- a/src/instana/w3c_trace_context/traceparent.py +++ b/src/instana/w3c_trace_context/traceparent.py @@ -3,6 +3,7 @@ from ..log import logger import re +from typing import Optional # See https://www.w3.org/TR/trace-context-2/#trace-flags for details on the bitmasks. SAMPLED_BITMASK = 0b1; @@ -45,7 +46,13 @@ def get_traceparent_fields(traceparent): logger.debug("Parsing the traceparent failed: {}".format(err)) return None, None, None, None - def update_traceparent(self, traceparent, in_trace_id, in_span_id, level): + def update_traceparent( + self, + traceparent: Optional[str], + in_trace_id: int, + in_span_id: int, + level: int, + ) -> str: """ This method updates the traceparent header or generates one if there was no traceparent incoming header or it was invalid @@ -56,7 +63,11 @@ def update_traceparent(self, traceparent, in_trace_id, in_span_id, level): :return: the updated traceparent header """ if traceparent is None: # modify the trace_id part only when it was not present at all - trace_id = in_trace_id.zfill(32) + trace_id = ( + in_trace_id.zfill(32) + if not isinstance(in_trace_id, int) + else in_trace_id + ) else: # - We do not need the incoming upstream parent span ID for the header we sent downstream. # - We also do not care about the incoming version: The version field we sent downstream needs to match the @@ -67,12 +78,11 @@ def update_traceparent(self, traceparent, in_trace_id, in_span_id, level): # downstream. _, trace_id, _, _ = self.get_traceparent_fields(traceparent) - parent_id = in_span_id.zfill(16) + parent_id = ( + in_span_id.zfill(16) if not isinstance(in_span_id, int) else in_span_id + ) flags = level & SAMPLED_BITMASK flags = format(flags, '0>2x') - traceparent = "{version}-{traceid}-{parentid}-{flags}".format(version=self.SPECIFICATION_VERSION, - traceid=trace_id, - parentid=parent_id, - flags=flags) + traceparent = f"{self.SPECIFICATION_VERSION}-{trace_id}-{parent_id}-{flags}" return traceparent diff --git a/src/instana/w3c_trace_context/tracestate.py b/src/instana/w3c_trace_context/tracestate.py index b1d066ea..f6eb32cb 100644 --- a/src/instana/w3c_trace_context/tracestate.py +++ b/src/instana/w3c_trace_context/tracestate.py @@ -42,8 +42,10 @@ def update_tracestate(self, tracestate, in_trace_id, in_span_id): :return: tracestate updated """ try: - span_id = in_span_id.zfill(16) # if span_id is shorter than 16 characters we prepend zeros - instana_tracestate = "in={};{}".format(in_trace_id, span_id) + span_id = ( + in_span_id.zfill(16) if not isinstance(in_span_id, int) else in_span_id + ) + instana_tracestate = f"in={in_trace_id};{span_id}" if tracestate is None or tracestate == "": tracestate = instana_tracestate else: diff --git a/tests/conftest.py b/tests/conftest.py index 5f5fd99b..f70d6de5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,23 +4,36 @@ import importlib.util import os import sys + import pytest +from opentelemetry.context.context import Context +from opentelemetry.trace import set_span_in_context if importlib.util.find_spec('celery'): pytest_plugins = ("celery.contrib.pytest", ) # Set our testing flags os.environ["INSTANA_TEST"] = "true" -# os.environ["INSTANA_DEBUG"] = "true" +os.environ["INSTANA_DISABLE_AUTO_INSTR"] = "true" -# Make sure the instana package is fully loaded -import instana +# TODO: remove all "noqa: E402" from instana package imports and move the +# block of env variables setting to below the imports after finishing the +# migration of instrumentation codes. +from instana.span import BaseSpan, InstanaSpan # noqa: E402 +from instana.span_context import SpanContext # noqa: E402 -collect_ignore_glob = [] +collect_ignore_glob = [ + "*autoprofile*", + "*clients*", + "*frameworks*", + "*platforms*", + "*propagators*", + "*w3c_trace_context*", +] # Cassandra and gevent tests are run in dedicated jobs on CircleCI and will # be run explicitly. (So always exclude them here) -if not os.environ.get("CASSANDRA_TEST" ): +if not os.environ.get("CASSANDRA_TEST"): collect_ignore_glob.append("*test_cassandra*") if not os.environ.get("COUCHBASE_TEST"): @@ -38,30 +51,81 @@ # tests/opentracing/test_ot_span.py::TestOTSpan::test_stacks # TODO: Remove that once we find a workaround or DROP opentracing! -if sys.version_info >= (3, 12): - # Currently the dependencies of sanic and aiohttp are not installable on 3.12 - # PyLongObject’ {aka ‘struct _longobject’} has no member named ‘ob_digit’ +if sys.version_info >= (3, 13): + # TODO: Test Case failures for unknown reason: + collect_ignore_glob.append("*test_aiohttp_server*") + collect_ignore_glob.append("*test_celery*") + + # Currently there is a runtime incompatibility caused by the library: + # `undefined symbol: _PyErr_WriteUnraisableMsg` + collect_ignore_glob.append("*boto3*") + + # Currently there is a runtime incompatibility caused by the library: + # `undefined symbol: _PyInterpreterState_Get` + collect_ignore_glob.append("*test_psycopg2*") + collect_ignore_glob.append("*test_sqlalchemy*") + + # Currently the latest version of pyramid depends on the `cgi` module + # which has been deprecated since Python 3.11 and finally removed in 3.13 + # `ModuleNotFoundError: No module named 'cgi'` + collect_ignore_glob.append("*test_pyramid*") + + # Currently not installable dependencies because of 3.13 incompatibilities + collect_ignore_glob.append("*test_fastapi*") + collect_ignore_glob.append("*test_google-cloud-pubsub*") + collect_ignore_glob.append("*test_google-cloud-storage*") + collect_ignore_glob.append("*test_grpcio*") collect_ignore_glob.append("*test_sanic*") - collect_ignore_glob.append("*test_aiohttp*") - # The asyncio also depends on aiohttp - collect_ignore_glob.append("*test_asyncio*") -@pytest.fixture(scope='session') + +@pytest.fixture(scope="session") def celery_config(): return { - 'broker_connection_retry_on_startup': True, - 'broker_url': 'redis://localhost:6379', - 'result_backend': 'redis://localhost:6379' + "broker_url": "redis://localhost:6379", + "result_backend": "redis://localhost:6379", } -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def celery_enable_logging(): return True -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def celery_includes(): - return { - 'tests.frameworks.test_celery' - } + return {"tests.frameworks.test_celery"} + + +@pytest.fixture +def trace_id() -> int: + return 1812338823475918251 + + +@pytest.fixture +def span_id() -> int: + return 6895521157646639861 + + +@pytest.fixture +def span_context(trace_id: int, span_id: int) -> SpanContext: + return SpanContext( + trace_id=trace_id, + span_id=span_id, + is_remote=False, + ) + + +@pytest.fixture +def span(span_context: SpanContext) -> InstanaSpan: + span_name = "test-span" + return InstanaSpan(span_name, span_context) + + +@pytest.fixture +def base_span(span: InstanaSpan) -> BaseSpan: + return BaseSpan(span, None, "test") + + +@pytest.fixture +def context(span: InstanaSpan) -> Context: + return set_span_in_context(span) diff --git a/tests/helpers.py b/tests/helpers.py index 95fe3e61..30caf5ac 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -9,51 +9,51 @@ """ Cassandra Environment """ -testenv['cassandra_host'] = os.environ.get('CASSANDRA_HOST', '127.0.0.1') -testenv['cassandra_username'] = os.environ.get('CASSANDRA_USERNAME', 'Administrator') -testenv['cassandra_password'] = os.environ.get('CASSANDRA_PASSWORD', 'password') +testenv["cassandra_host"] = os.environ.get("CASSANDRA_HOST", "127.0.0.1") +testenv["cassandra_username"] = os.environ.get("CASSANDRA_USERNAME", "Administrator") +testenv["cassandra_password"] = os.environ.get("CASSANDRA_PASSWORD", "password") """ CouchDB Environment """ -testenv['couchdb_host'] = os.environ.get('COUCHDB_HOST', '127.0.0.1') -testenv['couchdb_username'] = os.environ.get('COUCHDB_USERNAME', 'Administrator') -testenv['couchdb_password'] = os.environ.get('COUCHDB_PASSWORD', 'password') +testenv["couchdb_host"] = os.environ.get("COUCHDB_HOST", "127.0.0.1") +testenv["couchdb_username"] = os.environ.get("COUCHDB_USERNAME", "Administrator") +testenv["couchdb_password"] = os.environ.get("COUCHDB_PASSWORD", "password") """ MySQL Environment """ -if 'MYSQL_HOST' in os.environ: - testenv['mysql_host'] = os.environ['MYSQL_HOST'] +if "MYSQL_HOST" in os.environ: + testenv["mysql_host"] = os.environ["MYSQL_HOST"] else: - testenv['mysql_host'] = '127.0.0.1' + testenv["mysql_host"] = "127.0.0.1" -testenv['mysql_port'] = int(os.environ.get('MYSQL_PORT', '3306')) -testenv['mysql_db'] = os.environ.get('MYSQL_DATABASE', 'instana_test_db') -testenv['mysql_user'] = os.environ.get('MYSQL_USER', 'root') -testenv['mysql_pw'] = os.environ.get('MYSQL_ROOT_PASSWORD', 'passw0rd') +testenv["mysql_port"] = int(os.environ.get("MYSQL_PORT", "3306")) +testenv["mysql_db"] = os.environ.get("MYSQL_DATABASE", "instana_test_db") +testenv["mysql_user"] = os.environ.get("MYSQL_USER", "root") +testenv["mysql_pw"] = os.environ.get("MYSQL_ROOT_PASSWORD", "passw0rd") """ PostgreSQL Environment """ -testenv['postgresql_host'] = os.environ.get('POSTGRES_HOST', '127.0.0.1') -testenv['postgresql_port'] = int(os.environ.get('POSTGRES_PORT', '5432')) -testenv['postgresql_db'] = os.environ.get('POSTGRES_DB', 'instana_test_db') -testenv['postgresql_user'] = os.environ.get('POSTGRES_USER', 'root') -testenv['postgresql_pw'] = os.environ.get('POSTGRES_PW', 'passw0rd') +testenv["postgresql_host"] = os.environ.get("POSTGRES_HOST", "127.0.0.1") +testenv["postgresql_port"] = int(os.environ.get("POSTGRES_PORT", "5432")) +testenv["postgresql_db"] = os.environ.get("POSTGRES_DB", "instana_test_db") +testenv["postgresql_user"] = os.environ.get("POSTGRES_USER", "root") +testenv["postgresql_pw"] = os.environ.get("POSTGRES_PW", "passw0rd") """ Redis Environment """ -testenv['redis_host'] = os.environ.get('REDIS_HOST', '127.0.0.1') +testenv["redis_host"] = os.environ.get("REDIS_HOST", "127.0.0.1") """ MongoDB Environment """ -testenv['mongodb_host'] = os.environ.get('MONGO_HOST', '127.0.0.1') -testenv['mongodb_port'] = os.environ.get('MONGO_PORT', '27017') -testenv['mongodb_user'] = os.environ.get('MONGO_USER', None) -testenv['mongodb_pw'] = os.environ.get('MONGO_PW', None) +testenv["mongodb_host"] = os.environ.get("MONGO_HOST", "127.0.0.1") +testenv["mongodb_port"] = os.environ.get("MONGO_PORT", "27017") +testenv["mongodb_user"] = os.environ.get("MONGO_USER", None) +testenv["mongodb_pw"] = os.environ.get("MONGO_PW", None) def drop_log_spans_from_list(spans): @@ -66,7 +66,7 @@ def drop_log_spans_from_list(spans): """ new_list = [] for span in spans: - if span.n != 'log': + if span.n != "log": new_list.append(span) return new_list @@ -84,8 +84,8 @@ def fail_with_message_and_span_dump(msg, spans): span_dump = "\nDumping all collected spans (%d) -->\n" % span_count if span_count > 0: for span in spans: - span.stack = '' - span_dump += repr(span) + '\n' + span.stack = "" + span_dump += repr(span) + "\n" pytest.fail(msg + span_dump, True) @@ -144,9 +144,11 @@ def launch_traced_request(url): from instana.log import logger from instana.singletons import tracer - logger.warn("Launching request with a root SDK span name of 'launch_traced_request'") + logger.warn( + "Launching request with a root SDK span name of 'launch_traced_request'" + ) - with tracer.start_active_span('launch_traced_request'): + with tracer.start_as_current_span("launch_traced_request"): response = requests.get(url) return response diff --git a/tests/opentracing/__init__.py b/tests/opentracing/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/opentracing/test_opentracing.py b/tests/opentracing/test_opentracing.py deleted file mode 100644 index 0ed9e508..00000000 --- a/tests/opentracing/test_opentracing.py +++ /dev/null @@ -1,28 +0,0 @@ -# (c) Copyright IBM Corp. 2021 -# (c) Copyright Instana Inc. 2020 - -from unittest import SkipTest -from opentracing.harness.api_check import APICompatibilityCheckMixin - -from instana.tracer import InstanaTracer - - -class TestInstanaTracer(InstanaTracer, APICompatibilityCheckMixin): - def tracer(self): - return self - - def test_binary_propagation(self): - raise SkipTest('Binary format is not supported') - - def test_mandatory_formats(self): - raise SkipTest('Binary format is not supported') - - def check_baggage_values(self): - return True - - def is_parent(self, parent, span): - # use `Span` ids to check parenting - if parent is None: - return span.parent_id is None - - return parent.context.span_id == span.parent_id diff --git a/tests/opentracing/test_ot_propagators.py b/tests/opentracing/test_ot_propagators.py deleted file mode 100644 index de26f471..00000000 --- a/tests/opentracing/test_ot_propagators.py +++ /dev/null @@ -1,309 +0,0 @@ -# (c) Copyright IBM Corp. 2021 -# (c) Copyright Instana Inc. 2020 - -import inspect -import unittest - -import opentracing as ot - -import instana.propagators.http_propagator as ihp -import instana.propagators.text_propagator as itp -import instana.propagators.binary_propagator as ibp -from instana.span_context import SpanContext -from instana.tracer import InstanaTracer - - -class TestOTSpan(unittest.TestCase): - def test_http_basics(self): - inspect.isclass(ihp.HTTPPropagator) - - inject_func = getattr(ihp.HTTPPropagator, "inject", None) - self.assertTrue(inject_func) - self.assertTrue(callable(inject_func)) - - extract_func = getattr(ihp.HTTPPropagator, "extract", None) - self.assertTrue(extract_func) - self.assertTrue(callable(extract_func)) - - - def test_http_inject_with_dict(self): - ot.tracer = InstanaTracer() - - carrier = {} - span = ot.tracer.start_span("unittest") - ot.tracer.inject(span.context, ot.Format.HTTP_HEADERS, carrier) - - self.assertIn('X-INSTANA-T', carrier) - self.assertEqual(carrier['X-INSTANA-T'], span.context.trace_id) - self.assertIn('X-INSTANA-S', carrier) - self.assertEqual(carrier['X-INSTANA-S'], span.context.span_id) - self.assertIn('X-INSTANA-L', carrier) - self.assertEqual(carrier['X-INSTANA-L'], "1") - - - def test_http_inject_with_list(self): - ot.tracer = InstanaTracer() - - carrier = [] - span = ot.tracer.start_span("unittest") - ot.tracer.inject(span.context, ot.Format.HTTP_HEADERS, carrier) - - self.assertIn(('X-INSTANA-T', span.context.trace_id), carrier) - self.assertIn(('X-INSTANA-S', span.context.span_id), carrier) - self.assertIn(('X-INSTANA-L', "1"), carrier) - - - def test_http_basic_extract(self): - ot.tracer = InstanaTracer() - - carrier = {'X-INSTANA-T': '1', 'X-INSTANA-S': '1', 'X-INSTANA-L': '1', 'X-INSTANA-SYNTHETIC': '1'} - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - self.assertTrue(ctx.synthetic) - - - def test_http_extract_with_byte_keys(self): - ot.tracer = InstanaTracer() - - carrier = {b'X-INSTANA-T': '1', b'X-INSTANA-S': '1', b'X-INSTANA-L': '1', b'X-INSTANA-SYNTHETIC': '1'} - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - self.assertTrue(ctx.synthetic) - - - def test_http_extract_from_list_of_tuples(self): - ot.tracer = InstanaTracer() - - carrier = [(b'user-agent', b'python-requests/2.23.0'), (b'accept-encoding', b'gzip, deflate'), - (b'accept', b'*/*'), (b'connection', b'keep-alive'), - (b'x-instana-t', b'1'), (b'x-instana-s', b'1'), (b'x-instana-l', b'1'), (b'X-INSTANA-SYNTHETIC', '1')] - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - self.assertTrue(ctx.synthetic) - - - def test_http_mixed_case_extract(self): - ot.tracer = InstanaTracer() - - carrier = {'x-insTana-T': '1', 'X-inSTANa-S': '1', 'X-INstana-l': '1'} - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - self.assertFalse(ctx.synthetic) - - - def test_http_extract_synthetic_only(self): - ot.tracer = InstanaTracer() - - carrier = {'X-INSTANA-SYNTHETIC': '1'} - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertIsNone(ctx.trace_id) - self.assertIsNone(ctx.span_id) - self.assertTrue(ctx.synthetic) - - - def test_http_default_context_extract(self): - ot.tracer = InstanaTracer() - - carrier = {} - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertIsNone(ctx.trace_id) - self.assertIsNone(ctx.span_id) - self.assertFalse(ctx.synthetic) - - def test_http_128bit_headers(self): - ot.tracer = InstanaTracer() - - carrier = {'X-INSTANA-T': '0000000000000000b0789916ff8f319f', - 'X-INSTANA-S': '0000000000000000b0789916ff8f319f', 'X-INSTANA-L': '1'} - ctx = ot.tracer.extract(ot.Format.HTTP_HEADERS, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, 'b0789916ff8f319f') - self.assertEqual(ctx.span_id, 'b0789916ff8f319f') - - - def test_text_basics(self): - inspect.isclass(itp.TextPropagator) - - inject_func = getattr(itp.TextPropagator, "inject", None) - self.assertTrue(inject_func) - self.assertTrue(callable(inject_func)) - - extract_func = getattr(itp.TextPropagator, "extract", None) - self.assertTrue(extract_func) - self.assertTrue(callable(extract_func)) - - - def test_text_inject_with_dict(self): - ot.tracer = InstanaTracer() - - carrier = {} - span = ot.tracer.start_span("unittest") - ot.tracer.inject(span.context, ot.Format.TEXT_MAP, carrier) - - self.assertIn('x-instana-t', carrier) - self.assertEqual(carrier['x-instana-t'], span.context.trace_id) - self.assertIn('x-instana-s', carrier) - self.assertEqual(carrier['x-instana-s'], span.context.span_id) - self.assertIn('x-instana-l', carrier) - self.assertEqual(carrier['x-instana-l'], "1") - - - def test_text_inject_with_list(self): - ot.tracer = InstanaTracer() - - carrier = [] - span = ot.tracer.start_span("unittest") - ot.tracer.inject(span.context, ot.Format.TEXT_MAP, carrier) - - self.assertIn(('x-instana-t', span.context.trace_id), carrier) - self.assertIn(('x-instana-s', span.context.span_id), carrier) - self.assertIn(('x-instana-l', "1"), carrier) - - - def test_text_basic_extract(self): - ot.tracer = InstanaTracer() - - carrier = {'x-instana-t': '1', 'x-instana-s': '1', 'x-instana-l': '1'} - ctx = ot.tracer.extract(ot.Format.TEXT_MAP, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - - - def test_text_mixed_case_extract(self): - ot.tracer = InstanaTracer() - - carrier = {'x-insTana-T': '1', 'X-inSTANa-S': '1', 'X-INstana-l': '1'} - ctx = ot.tracer.extract(ot.Format.TEXT_MAP, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - - - def test_text_default_context_extract(self): - ot.tracer = InstanaTracer() - - carrier = {} - ctx = ot.tracer.extract(ot.Format.TEXT_MAP, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertIsNone(ctx.trace_id) - self.assertIsNone(ctx.span_id) - self.assertFalse(ctx.synthetic) - - - def test_text_128bit_headers(self): - ot.tracer = InstanaTracer() - - carrier = {'x-instana-t': '0000000000000000b0789916ff8f319f', - 'x-instana-s': ' 0000000000000000b0789916ff8f319f', 'X-INSTANA-L': '1'} - ctx = ot.tracer.extract(ot.Format.TEXT_MAP, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, 'b0789916ff8f319f') - self.assertEqual(ctx.span_id, 'b0789916ff8f319f') - - def test_binary_basics(self): - inspect.isclass(ibp.BinaryPropagator) - - inject_func = getattr(ibp.BinaryPropagator, "inject", None) - self.assertTrue(inject_func) - self.assertTrue(callable(inject_func)) - - extract_func = getattr(ibp.BinaryPropagator, "extract", None) - self.assertTrue(extract_func) - self.assertTrue(callable(extract_func)) - - - def test_binary_inject_with_dict(self): - ot.tracer = InstanaTracer() - - carrier = {} - span = ot.tracer.start_span("unittest") - ot.tracer.inject(span.context, ot.Format.BINARY, carrier) - - self.assertIn(b'x-instana-t', carrier) - self.assertEqual(carrier[b'x-instana-t'], str.encode(span.context.trace_id)) - self.assertIn(b'x-instana-s', carrier) - self.assertEqual(carrier[b'x-instana-s'], str.encode(span.context.span_id)) - self.assertIn(b'x-instana-l', carrier) - self.assertEqual(carrier[b'x-instana-l'], b'1') - - - def test_binary_inject_with_list(self): - ot.tracer = InstanaTracer() - - carrier = [] - span = ot.tracer.start_span("unittest") - ot.tracer.inject(span.context, ot.Format.BINARY, carrier) - - self.assertIn((b'x-instana-t', str.encode(span.context.trace_id)), carrier) - self.assertIn((b'x-instana-s', str.encode(span.context.span_id)), carrier) - self.assertIn((b'x-instana-l', b'1'), carrier) - - - def test_binary_basic_extract(self): - ot.tracer = InstanaTracer() - - carrier = {b'X-INSTANA-T': b'1', b'X-INSTANA-S': b'1', b'X-INSTANA-L': b'1', b'X-INSTANA-SYNTHETIC': b'1'} - ctx = ot.tracer.extract(ot.Format.BINARY, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - self.assertTrue(ctx.synthetic) - - - def test_binary_mixed_case_extract(self): - ot.tracer = InstanaTracer() - - carrier = {'x-insTana-T': '1', 'X-inSTANa-S': '1', 'X-INstana-l': '1', b'X-inStaNa-SYNtheTIC': b'1'} - ctx = ot.tracer.extract(ot.Format.BINARY, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, '0000000000000001') - self.assertEqual(ctx.span_id, '0000000000000001') - self.assertTrue(ctx.synthetic) - - - def test_binary_default_context_extract(self): - ot.tracer = InstanaTracer() - - carrier = {} - ctx = ot.tracer.extract(ot.Format.BINARY, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertIsNone(ctx.trace_id) - self.assertIsNone(ctx.span_id) - self.assertFalse(ctx.synthetic) - - - def test_binary_128bit_headers(self): - ot.tracer = InstanaTracer() - - carrier = {'X-INSTANA-T': '0000000000000000b0789916ff8f319f', - 'X-INSTANA-S': ' 0000000000000000b0789916ff8f319f', 'X-INSTANA-L': '1'} - ctx = ot.tracer.extract(ot.Format.BINARY, carrier) - - self.assertIsInstance(ctx, SpanContext) - self.assertEqual(ctx.trace_id, 'b0789916ff8f319f') - self.assertEqual(ctx.span_id, 'b0789916ff8f319f') diff --git a/tests/opentracing/test_ot_span.py b/tests/opentracing/test_ot_span.py deleted file mode 100644 index 9280df76..00000000 --- a/tests/opentracing/test_ot_span.py +++ /dev/null @@ -1,290 +0,0 @@ -# (c) Copyright IBM Corp. 2021 -# (c) Copyright Instana Inc. 2020 - -import re -import sys -import json -import time -import unittest -from uuid import UUID - -import opentracing - -from instana.util import to_json -from instana.singletons import agent, tracer -from ..helpers import get_first_span_by_filter - - -class TestOTSpan(unittest.TestCase): - def setUp(self): - """ Clear all spans before a test run """ - agent.options.service_name = None - opentracing.tracer = tracer - recorder = opentracing.tracer.recorder - recorder.clear_spans() - - def tearDown(self): - """ Do nothing for now """ - return None - - def test_span_interface(self): - span = opentracing.tracer.start_span("blah") - self.assertTrue(hasattr(span, "finish")) - self.assertTrue(hasattr(span, "set_tag")) - self.assertTrue(hasattr(span, "tags")) - self.assertTrue(hasattr(span, "operation_name")) - self.assertTrue(hasattr(span, "set_baggage_item")) - self.assertTrue(hasattr(span, "get_baggage_item")) - self.assertTrue(hasattr(span, "context")) - self.assertTrue(hasattr(span, "log")) - - def test_span_ids(self): - count = 0 - while count <= 1000: - count += 1 - span = opentracing.tracer.start_span("test_span_ids") - context = span.context - self.assertTrue(0 <= int(context.span_id, 16) <= 18446744073709551615) - self.assertTrue(0 <= int(context.trace_id, 16) <= 18446744073709551615) - - # Python 3.11 support is incomplete yet - # TODO: Remove this once we find a workaround or DROP opentracing! - @unittest.skipIf(sys.version_info >= (3, 11), reason="Raises not Implemented exception in OSX") - def test_stacks(self): - # Entry spans have no stack attached by default - wsgi_span = opentracing.tracer.start_span("wsgi") - self.assertIsNone(wsgi_span.stack) - - # SDK spans have no stack attached by default - sdk_span = opentracing.tracer.start_span("unregistered_span_type") - self.assertIsNone(sdk_span.stack) - - # Exit spans are no longer than 30 frames - exit_span = opentracing.tracer.start_span("urllib3") - self.assertLessEqual(len(exit_span.stack), 30) - - def test_span_fields(self): - span = opentracing.tracer.start_span("mycustom") - self.assertEqual("mycustom", span.operation_name) - self.assertTrue(span.context) - - span.set_tag("tagone", "string") - span.set_tag("tagtwo", 150) - - self.assertEqual("string", span.tags['tagone']) - self.assertEqual(150, span.tags['tagtwo']) - - @unittest.skipIf(sys.platform == "darwin", reason="Raises not Implemented exception in OSX") - def test_span_queueing(self): - recorder = opentracing.tracer.recorder - - count = 1 - while count <= 20: - count += 1 - span = opentracing.tracer.start_span("queuethisplz") - span.set_tag("tagone", "string") - span.set_tag("tagtwo", 150) - span.finish() - - self.assertEqual(20, recorder.queue_size()) - - def test_sdk_spans(self): - recorder = opentracing.tracer.recorder - - span = opentracing.tracer.start_span("custom_sdk_span") - span.set_tag("tagone", "string") - span.set_tag("tagtwo", 150) - span.set_tag('span.kind', "entry") - time.sleep(0.5) - span.finish() - - spans = recorder.queued_spans() - self.assertEqual(1, len(spans)) - - sdk_span = spans[0] - self.assertEqual('sdk', sdk_span.n) - self.assertEqual(None, sdk_span.p) - self.assertEqual(sdk_span.s, sdk_span.t) - self.assertTrue(sdk_span.ts) - self.assertGreater(sdk_span.ts, 0) - self.assertTrue(sdk_span.d) - self.assertGreater(sdk_span.d, 0) - - self.assertTrue(sdk_span.data) - self.assertTrue(sdk_span.data["sdk"]) - self.assertEqual('entry', sdk_span.data["sdk"]["type"]) - self.assertEqual('custom_sdk_span', sdk_span.data["sdk"]["name"]) - self.assertTrue(sdk_span.data["sdk"]["custom"]) - self.assertTrue(sdk_span.data["sdk"]["custom"]["tags"]) - - def test_span_kind(self): - recorder = opentracing.tracer.recorder - - span = opentracing.tracer.start_span("custom_sdk_span") - span.set_tag('span.kind', "consumer") - span.finish() - - span = opentracing.tracer.start_span("custom_sdk_span") - span.set_tag('span.kind', "server") - span.finish() - - span = opentracing.tracer.start_span("custom_sdk_span") - span.set_tag('span.kind', "producer") - span.finish() - - span = opentracing.tracer.start_span("custom_sdk_span") - span.set_tag('span.kind', "client") - span.finish() - - span = opentracing.tracer.start_span("custom_sdk_span") - span.set_tag('span.kind', "blah") - span.finish() - - spans = recorder.queued_spans() - self.assertEqual(5, len(spans)) - - span = spans[0] - self.assertEqual('entry', span.data["sdk"]["type"]) - - span = spans[1] - self.assertEqual('entry', span.data["sdk"]["type"]) - - span = spans[2] - self.assertEqual('exit', span.data["sdk"]["type"]) - - span = spans[3] - self.assertEqual('exit', span.data["sdk"]["type"]) - - span = spans[4] - self.assertEqual('intermediate', span.data["sdk"]["type"]) - - span = spans[0] - self.assertEqual(1, span.k) - - span = spans[1] - self.assertEqual(1, span.k) - - span = spans[2] - self.assertEqual(2, span.k) - - span = spans[3] - self.assertEqual(2, span.k) - - span = spans[4] - self.assertEqual(3, span.k) - - def test_tag_values(self): - with tracer.start_active_span('test') as scope: - # Set a UUID class as a tag - # If unchecked, this causes a json.dumps error: "ValueError: Circular reference detected" - scope.span.set_tag('uuid', UUID(bytes=b'\x12\x34\x56\x78'*4)) - # Arbitrarily setting an instance of some class - scope.span.set_tag('tracer', tracer) - scope.span.set_tag('none', None) - scope.span.set_tag('mylist', [1, 2, 3]) - scope.span.set_tag('myset', {"one", 2}) - - spans = tracer.recorder.queued_spans() - self.assertEqual(1, len(spans)) - - test_span = spans[0] - self.assertTrue(test_span) - self.assertEqual(len(test_span.data['sdk']['custom']['tags']), 5) - self.assertEqual(test_span.data['sdk']['custom']['tags']['uuid'], "UUID('12345678-1234-5678-1234-567812345678')") - self.assertTrue(test_span.data['sdk']['custom']['tags']['tracer']) - self.assertEqual(test_span.data['sdk']['custom']['tags']['none'], 'None') - self.assertListEqual(test_span.data['sdk']['custom']['tags']['mylist'], [1, 2, 3]) - self.assertRegex(test_span.data['sdk']['custom']['tags']['myset'], r"\{.*,.*\}") - - # Convert to JSON - json_data = to_json(test_span) - self.assertTrue(json_data) - - # And back - span_dict = json.loads(json_data) - self.assertEqual(len(span_dict['data']['sdk']['custom']['tags']), 5) - self.assertEqual(span_dict['data']['sdk']['custom']['tags']['uuid'], "UUID('12345678-1234-5678-1234-567812345678')") - self.assertTrue(span_dict['data']['sdk']['custom']['tags']['tracer']) - self.assertEqual(span_dict['data']['sdk']['custom']['tags']['none'], 'None') - self.assertListEqual(span_dict['data']['sdk']['custom']['tags']['mylist'], [1, 2, 3]) - self.assertRegex(test_span.data['sdk']['custom']['tags']['myset'], r"{.*,.*}") - - def test_tag_names(self): - with tracer.start_active_span('test') as scope: - # Tag names (keys) must be strings - scope.span.set_tag(1234567890, 'This should not get set') - # Unicode key name - scope.span.set_tag(u'asdf', 'This should be ok') - - spans = tracer.recorder.queued_spans() - self.assertEqual(len(spans), 1) - - test_span = spans[0] - self.assertTrue(test_span) - self.assertEqual(len(test_span.data['sdk']['custom']['tags']), 1) - self.assertEqual(test_span.data['sdk']['custom']['tags']['asdf'], 'This should be ok') - - json_data = to_json(test_span) - self.assertTrue(json_data) - - def test_custom_service_name(self): - # Set a custom service name - agent.options.service_name = "custom_service_name" - - with tracer.start_active_span('entry_span') as scope: - scope.span.set_tag('span.kind', 'server') - scope.span.set_tag(u'type', 'entry_span') - - with tracer.start_active_span('intermediate_span', child_of=scope.span) as exit_scope: - exit_scope.span.set_tag(u'type', 'intermediate_span') - - with tracer.start_active_span('exit_span', child_of=scope.span) as exit_scope: - exit_scope.span.set_tag('span.kind', 'client') - exit_scope.span.set_tag(u'type', 'exit_span') - - spans = tracer.recorder.queued_spans() - self.assertEqual(len(spans), 3) - - filter = lambda span: span.n == "sdk" and span.data['sdk']['name'] == "entry_span" - entry_span = get_first_span_by_filter(spans, filter) - self.assertTrue(entry_span) - - filter = lambda span: span.n == "sdk" and span.data['sdk']['name'] == "intermediate_span" - intermediate_span = get_first_span_by_filter(spans, filter) - self.assertTrue(intermediate_span) - - filter = lambda span: span.n == "sdk" and span.data['sdk']['name'] == "exit_span" - exit_span = get_first_span_by_filter(spans, filter) - self.assertTrue(exit_span) - - self.assertTrue(entry_span) - self.assertEqual(len(entry_span.data['sdk']['custom']['tags']), 2) - self.assertEqual(entry_span.data['sdk']['custom']['tags']['type'], 'entry_span') - self.assertEqual(entry_span.data['service'], 'custom_service_name') - self.assertEqual(entry_span.k, 1) - - self.assertTrue(intermediate_span) - self.assertEqual(len(intermediate_span.data['sdk']['custom']['tags']), 1) - self.assertEqual(intermediate_span.data['sdk']['custom']['tags']['type'], 'intermediate_span') - self.assertEqual(intermediate_span.data['service'], 'custom_service_name') - self.assertEqual(intermediate_span.k, 3) - - self.assertTrue(exit_span) - self.assertEqual(len(exit_span.data['sdk']['custom']['tags']), 2) - self.assertEqual(exit_span.data['sdk']['custom']['tags']['type'], 'exit_span') - self.assertEqual(exit_span.data['service'], 'custom_service_name') - self.assertEqual(exit_span.k, 2) - - def test_span_log(self): - with tracer.start_active_span('mylogspan') as scope: - scope.span.log_kv({'Don McLean': 'American Pie'}) - scope.span.log_kv({'Elton John': 'Your Song'}) - - spans = tracer.recorder.queued_spans() - self.assertEqual(len(spans), 1) - - my_log_span = spans[0] - self.assertEqual(my_log_span.n, 'sdk') - - log_data = my_log_span.data['sdk']['custom']['logs'] - self.assertEqual(len(log_data), 2) diff --git a/tests/opentracing/test_ot_tracer.py b/tests/opentracing/test_ot_tracer.py deleted file mode 100644 index c73037f4..00000000 --- a/tests/opentracing/test_ot_tracer.py +++ /dev/null @@ -1,10 +0,0 @@ -# (c) Copyright IBM Corp. 2021 -# (c) Copyright Instana Inc. 2020 - -import opentracing - - -def test_tracer_basics(): - assert hasattr(opentracing.tracer, "start_span") - assert hasattr(opentracing.tracer, "inject") - assert hasattr(opentracing.tracer, "extract") diff --git a/tests/platforms/test_host_collector.py b/tests/platforms/test_host_collector.py index a53c801e..667e7afd 100644 --- a/tests/platforms/test_host_collector.py +++ b/tests/platforms/test_host_collector.py @@ -10,13 +10,16 @@ from instana.tracer import InstanaTracer from instana.recorder import StanRecorder from instana.agent.host import HostAgent -from instana.collector.helpers.runtime import PATH_OF_AUTOTRACE_WEBHOOK_SITEDIR +from instana.collector.helpers.runtime import ( + PATH_OF_DEPRECATED_INSTALLATION_VIA_HOST_AGENT, +) from instana.collector.host import HostCollector from instana.singletons import get_agent, set_agent, get_tracer, set_tracer from instana.version import VERSION + class TestHostCollector(unittest.TestCase): - def __init__(self, methodName='runTest'): + def __init__(self, methodName="runTest"): super(TestHostCollector, self).__init__(methodName) self.agent = None self.span_recorder = None @@ -29,14 +32,18 @@ def setUp(self): self.webhook_sitedir_path = PATH_OF_AUTOTRACE_WEBHOOK_SITEDIR + '3.8.0' def tearDown(self): - """ Reset all environment variables of consequence """ + """Reset all environment variables of consequence""" variable_names = ( - "AWS_EXECUTION_ENV", "INSTANA_EXTRA_HTTP_HEADERS", - "INSTANA_ENDPOINT_URL", "INSTANA_AGENT_KEY", "INSTANA_ZONE", - "INSTANA_TAGS", "INSTANA_DISABLE_METRICS_COLLECTION", - "INSTANA_DISABLE_PYTHON_PACKAGE_COLLECTION", - "AUTOWRAPT_BOOTSTRAP" - ) + "AWS_EXECUTION_ENV", + "INSTANA_EXTRA_HTTP_HEADERS", + "INSTANA_ENDPOINT_URL", + "INSTANA_AGENT_KEY", + "INSTANA_ZONE", + "INSTANA_TAGS", + "INSTANA_DISABLE_METRICS_COLLECTION", + "INSTANA_DISABLE_PYTHON_PACKAGE_COLLECTION", + "AUTOWRAPT_BOOTSTRAP", + ) for variable_name in variable_names: if variable_name in os.environ: @@ -61,78 +68,102 @@ def test_prepare_payload_basics(self): self.assertTrue(payload) self.assertEqual(len(payload.keys()), 3) - self.assertIn('spans', payload) - self.assertIsInstance(payload['spans'], list) - self.assertEqual(len(payload['spans']), 0) - self.assertIn('metrics', payload) - self.assertEqual(len(payload['metrics'].keys()), 1) - self.assertIn('plugins', payload['metrics']) - self.assertIsInstance(payload['metrics']['plugins'], list) - self.assertEqual(len(payload['metrics']['plugins']), 1) - - python_plugin = payload['metrics']['plugins'][0] - self.assertEqual(python_plugin['name'], 'com.instana.plugin.python') - self.assertEqual(python_plugin['entityId'], str(os.getpid())) - self.assertIn('data', python_plugin) - self.assertIn('snapshot', python_plugin['data']) - self.assertIn('m', python_plugin['data']['snapshot']) - self.assertEqual('Manual', python_plugin['data']['snapshot']['m']) - self.assertIn('metrics', python_plugin['data']) + self.assertIn("spans", payload) + self.assertIsInstance(payload["spans"], list) + self.assertEqual(len(payload["spans"]), 0) + self.assertIn("metrics", payload) + self.assertEqual(len(payload["metrics"].keys()), 1) + self.assertIn("plugins", payload["metrics"]) + self.assertIsInstance(payload["metrics"]["plugins"], list) + self.assertEqual(len(payload["metrics"]["plugins"]), 1) + + python_plugin = payload["metrics"]["plugins"][0] + self.assertEqual(python_plugin["name"], "com.instana.plugin.python") + self.assertEqual(python_plugin["entityId"], str(os.getpid())) + self.assertIn("data", python_plugin) + self.assertIn("snapshot", python_plugin["data"]) + self.assertIn("m", python_plugin["data"]["snapshot"]) + self.assertEqual("Manual", python_plugin["data"]["snapshot"]["m"]) + self.assertIn("metrics", python_plugin["data"]) # Validate that all metrics are reported on the first run - self.assertIn('ru_utime', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_utime']), [float, int]) - self.assertIn('ru_stime', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_stime']), [float, int]) - self.assertIn('ru_maxrss', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_maxrss']), [float, int]) - self.assertIn('ru_ixrss', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_ixrss']), [float, int]) - self.assertIn('ru_idrss', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_idrss']), [float, int]) - self.assertIn('ru_isrss', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_isrss']), [float, int]) - self.assertIn('ru_minflt', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_minflt']), [float, int]) - self.assertIn('ru_majflt', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_majflt']), [float, int]) - self.assertIn('ru_nswap', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_nswap']), [float, int]) - self.assertIn('ru_inblock', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_inblock']), [float, int]) - self.assertIn('ru_oublock', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_oublock']), [float, int]) - self.assertIn('ru_msgsnd', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_msgsnd']), [float, int]) - self.assertIn('ru_msgrcv', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_msgrcv']), [float, int]) - self.assertIn('ru_nsignals', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_nsignals']), [float, int]) - self.assertIn('ru_nvcsw', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_nvcsw']), [float, int]) - self.assertIn('ru_nivcsw', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['ru_nivcsw']), [float, int]) - self.assertIn('alive_threads', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['alive_threads']), [float, int]) - self.assertIn('dummy_threads', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['dummy_threads']), [float, int]) - self.assertIn('daemon_threads', python_plugin['data']['metrics']) - self.assertIn(type(python_plugin['data']['metrics']['daemon_threads']), [float, int]) - - self.assertIn('gc', python_plugin['data']['metrics']) - self.assertIsInstance(python_plugin['data']['metrics']['gc'], dict) - self.assertIn('collect0', python_plugin['data']['metrics']['gc']) - self.assertIn(type(python_plugin['data']['metrics']['gc']['collect0']), [float, int]) - self.assertIn('collect1', python_plugin['data']['metrics']['gc']) - self.assertIn(type(python_plugin['data']['metrics']['gc']['collect1']), [float, int]) - self.assertIn('collect2', python_plugin['data']['metrics']['gc']) - self.assertIn(type(python_plugin['data']['metrics']['gc']['collect2']), [float, int]) - self.assertIn('threshold0', python_plugin['data']['metrics']['gc']) - self.assertIn(type(python_plugin['data']['metrics']['gc']['threshold0']), [float, int]) - self.assertIn('threshold1', python_plugin['data']['metrics']['gc']) - self.assertIn(type(python_plugin['data']['metrics']['gc']['threshold1']), [float, int]) - self.assertIn('threshold2', python_plugin['data']['metrics']['gc']) - self.assertIn(type(python_plugin['data']['metrics']['gc']['threshold2']), [float, int]) + self.assertIn("ru_utime", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_utime"]), [float, int]) + self.assertIn("ru_stime", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_stime"]), [float, int]) + self.assertIn("ru_maxrss", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_maxrss"]), [float, int]) + self.assertIn("ru_ixrss", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_ixrss"]), [float, int]) + self.assertIn("ru_idrss", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_idrss"]), [float, int]) + self.assertIn("ru_isrss", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_isrss"]), [float, int]) + self.assertIn("ru_minflt", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_minflt"]), [float, int]) + self.assertIn("ru_majflt", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_majflt"]), [float, int]) + self.assertIn("ru_nswap", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_nswap"]), [float, int]) + self.assertIn("ru_inblock", python_plugin["data"]["metrics"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["ru_inblock"]), [float, int] + ) + self.assertIn("ru_oublock", python_plugin["data"]["metrics"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["ru_oublock"]), [float, int] + ) + self.assertIn("ru_msgsnd", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_msgsnd"]), [float, int]) + self.assertIn("ru_msgrcv", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_msgrcv"]), [float, int]) + self.assertIn("ru_nsignals", python_plugin["data"]["metrics"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["ru_nsignals"]), [float, int] + ) + self.assertIn("ru_nvcsw", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_nvcsw"]), [float, int]) + self.assertIn("ru_nivcsw", python_plugin["data"]["metrics"]) + self.assertIn(type(python_plugin["data"]["metrics"]["ru_nivcsw"]), [float, int]) + self.assertIn("alive_threads", python_plugin["data"]["metrics"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["alive_threads"]), [float, int] + ) + self.assertIn("dummy_threads", python_plugin["data"]["metrics"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["dummy_threads"]), [float, int] + ) + self.assertIn("daemon_threads", python_plugin["data"]["metrics"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["daemon_threads"]), [float, int] + ) + + self.assertIn("gc", python_plugin["data"]["metrics"]) + self.assertIsInstance(python_plugin["data"]["metrics"]["gc"], dict) + self.assertIn("collect0", python_plugin["data"]["metrics"]["gc"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["gc"]["collect0"]), [float, int] + ) + self.assertIn("collect1", python_plugin["data"]["metrics"]["gc"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["gc"]["collect1"]), [float, int] + ) + self.assertIn("collect2", python_plugin["data"]["metrics"]["gc"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["gc"]["collect2"]), [float, int] + ) + self.assertIn("threshold0", python_plugin["data"]["metrics"]["gc"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["gc"]["threshold0"]), [float, int] + ) + self.assertIn("threshold1", python_plugin["data"]["metrics"]["gc"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["gc"]["threshold1"]), [float, int] + ) + self.assertIn("threshold2", python_plugin["data"]["metrics"]["gc"]) + self.assertIn( + type(python_plugin["data"]["metrics"]["gc"]["threshold2"]), [float, int] + ) def test_prepare_payload_basics_disable_runtime_metrics(self): os.environ["INSTANA_DISABLE_METRICS_COLLECTION"] = "TRUE" @@ -142,61 +173,62 @@ def test_prepare_payload_basics_disable_runtime_metrics(self): self.assertTrue(payload) self.assertEqual(len(payload.keys()), 3) - self.assertIn('spans', payload) - self.assertIsInstance(payload['spans'], list) - self.assertEqual(len(payload['spans']), 0) - self.assertIn('metrics', payload) - self.assertEqual(len(payload['metrics'].keys()), 1) - self.assertIn('plugins', payload['metrics']) - self.assertIsInstance(payload['metrics']['plugins'], list) - self.assertEqual(len(payload['metrics']['plugins']), 1) - - python_plugin = payload['metrics']['plugins'][0] - self.assertEqual(python_plugin['name'], 'com.instana.plugin.python') - self.assertEqual(python_plugin['entityId'], str(os.getpid())) - self.assertIn('data', python_plugin) - self.assertIn('snapshot', python_plugin['data']) - self.assertIn('m', python_plugin['data']['snapshot']) - self.assertEqual('Manual', python_plugin['data']['snapshot']['m']) - self.assertNotIn('metrics', python_plugin['data']) + self.assertIn("spans", payload) + self.assertIsInstance(payload["spans"], list) + self.assertEqual(len(payload["spans"]), 0) + self.assertIn("metrics", payload) + self.assertEqual(len(payload["metrics"].keys()), 1) + self.assertIn("plugins", payload["metrics"]) + self.assertIsInstance(payload["metrics"]["plugins"], list) + self.assertEqual(len(payload["metrics"]["plugins"]), 1) + + python_plugin = payload["metrics"]["plugins"][0] + self.assertEqual(python_plugin["name"], "com.instana.plugin.python") + self.assertEqual(python_plugin["entityId"], str(os.getpid())) + self.assertIn("data", python_plugin) + self.assertIn("snapshot", python_plugin["data"]) + self.assertIn("m", python_plugin["data"]["snapshot"]) + self.assertEqual("Manual", python_plugin["data"]["snapshot"]["m"]) + self.assertNotIn("metrics", python_plugin["data"]) @patch.object(HostCollector, "should_send_snapshot_data") - def test_prepare_payload_with_snapshot_with_python_packages(self, mock_should_send_snapshot_data): + def test_prepare_payload_with_snapshot_with_python_packages( + self, mock_should_send_snapshot_data + ): mock_should_send_snapshot_data.return_value = True self.create_agent_and_setup_tracer() payload = self.agent.collector.prepare_payload() self.assertTrue(payload) - self.assertIn('snapshot', payload['metrics']['plugins'][0]['data']) - snapshot = payload['metrics']['plugins'][0]['data']['snapshot'] + self.assertIn("snapshot", payload["metrics"]["plugins"][0]["data"]) + snapshot = payload["metrics"]["plugins"][0]["data"]["snapshot"] self.assertTrue(snapshot) - self.assertIn('m', snapshot) - self.assertEqual('Manual', snapshot['m']) - self.assertIn('version', snapshot) - self.assertGreater(len(snapshot['versions']), 5) - self.assertEqual(snapshot['versions']['instana'], VERSION) - self.assertIn('wrapt', snapshot['versions']) - self.assertIn('fysom', snapshot['versions']) - self.assertIn('opentracing', snapshot['versions']) - self.assertIn('basictracer', snapshot['versions']) + self.assertIn("m", snapshot) + self.assertEqual("Manual", snapshot["m"]) + self.assertIn("version", snapshot) + self.assertGreater(len(snapshot["versions"]), 5) + self.assertEqual(snapshot["versions"]["instana"], VERSION) + self.assertIn("wrapt", snapshot["versions"]) + self.assertIn("fysom", snapshot["versions"]) @patch.object(HostCollector, "should_send_snapshot_data") - def test_prepare_payload_with_snapshot_disabled_python_packages(self, mock_should_send_snapshot_data): + def test_prepare_payload_with_snapshot_disabled_python_packages( + self, mock_should_send_snapshot_data + ): mock_should_send_snapshot_data.return_value = True os.environ["INSTANA_DISABLE_PYTHON_PACKAGE_COLLECTION"] = "TRUE" self.create_agent_and_setup_tracer() payload = self.agent.collector.prepare_payload() self.assertTrue(payload) - self.assertIn('snapshot', payload['metrics']['plugins'][0]['data']) - snapshot = payload['metrics']['plugins'][0]['data']['snapshot'] + self.assertIn("snapshot", payload["metrics"]["plugins"][0]["data"]) + snapshot = payload["metrics"]["plugins"][0]["data"]["snapshot"] self.assertTrue(snapshot) - self.assertIn('m', snapshot) - self.assertEqual('Manual', snapshot['m']) - self.assertIn('version', snapshot) - self.assertEqual(len(snapshot['versions']), 1) - self.assertEqual(snapshot['versions']['instana'], VERSION) - + self.assertIn("m", snapshot) + self.assertEqual("Manual", snapshot["m"]) + self.assertIn("version", snapshot) + self.assertEqual(len(snapshot["versions"]), 1) + self.assertEqual(snapshot["versions"]["instana"], VERSION) @patch.object(HostCollector, "should_send_snapshot_data") def test_prepare_payload_with_autowrapt(self, mock_should_send_snapshot_data): @@ -206,18 +238,21 @@ def test_prepare_payload_with_autowrapt(self, mock_should_send_snapshot_data): payload = self.agent.collector.prepare_payload() self.assertTrue(payload) - self.assertIn('snapshot', payload['metrics']['plugins'][0]['data']) - snapshot = payload['metrics']['plugins'][0]['data']['snapshot'] + self.assertIn("snapshot", payload["metrics"]["plugins"][0]["data"]) + snapshot = payload["metrics"]["plugins"][0]["data"]["snapshot"] self.assertTrue(snapshot) - self.assertIn('m', snapshot) - self.assertEqual('Autowrapt', snapshot['m']) - self.assertIn('version', snapshot) - self.assertGreater(len(snapshot['versions']), 5) - expected_packages = ('instana', 'wrapt', 'fysom', 'opentracing', 'basictracer') + self.assertIn("m", snapshot) + self.assertEqual("Autowrapt", snapshot["m"]) + self.assertIn("version", snapshot) + self.assertGreater(len(snapshot["versions"]), 5) + expected_packages = ("instana", "wrapt", "fysom") for package in expected_packages: - self.assertIn(package, snapshot['versions'], f"{package} not found in snapshot['versions']") - self.assertEqual(snapshot['versions']['instana'], VERSION) - + self.assertIn( + package, + snapshot["versions"], + f"{package} not found in snapshot['versions']", + ) + self.assertEqual(snapshot["versions"]["instana"], VERSION) @patch.object(HostCollector, "should_send_snapshot_data") def test_prepare_payload_with_autotrace(self, mock_should_send_snapshot_data): @@ -229,14 +264,18 @@ def test_prepare_payload_with_autotrace(self, mock_should_send_snapshot_data): payload = self.agent.collector.prepare_payload() self.assertTrue(payload) - self.assertIn('snapshot', payload['metrics']['plugins'][0]['data']) - snapshot = payload['metrics']['plugins'][0]['data']['snapshot'] + self.assertIn("snapshot", payload["metrics"]["plugins"][0]["data"]) + snapshot = payload["metrics"]["plugins"][0]["data"]["snapshot"] self.assertTrue(snapshot) - self.assertIn('m', snapshot) - self.assertEqual('AutoTrace', snapshot['m']) - self.assertIn('version', snapshot) - self.assertGreater(len(snapshot['versions']), 5) - expected_packages = ('instana', 'wrapt', 'fysom', 'opentracing', 'basictracer') + self.assertIn("m", snapshot) + self.assertEqual("AutoTrace", snapshot["m"]) + self.assertIn("version", snapshot) + self.assertGreater(len(snapshot["versions"]), 5) + expected_packages = ("instana", "wrapt", "fysom") for package in expected_packages: - self.assertIn(package, snapshot['versions'], f"{package} not found in snapshot['versions']") - self.assertEqual(snapshot['versions']['instana'], VERSION) + self.assertIn( + package, + snapshot["versions"], + f"{package} not found in snapshot['versions']", + ) + self.assertEqual(snapshot["versions"]["instana"], VERSION) diff --git a/tests/recorder/test_stan_recorder.py b/tests/recorder/test_stan_recorder.py index adb08e78..5f9940f6 100644 --- a/tests/recorder/test_stan_recorder.py +++ b/tests/recorder/test_stan_recorder.py @@ -1,9 +1,17 @@ -from instana.recorder import StanRecorder - from multiprocessing import Queue +import sys from unittest import TestCase from unittest.mock import NonCallableMagicMock, PropertyMock +import pytest + +from instana.recorder import StanRecorder + + +@pytest.mark.skipif( + sys.platform == "darwin", + reason="Avoiding NotImplementedError when calling multiprocessing.Queue.qsize()", +) class TestStanRecorderTC(TestCase): def setUp(self): mock_agent = NonCallableMagicMock() @@ -13,7 +21,9 @@ def setUp(self): self.mock_suppressed_span = NonCallableMagicMock() self.mock_suppressed_span.context = NonCallableMagicMock() self.mock_suppressed_property = PropertyMock(return_value=True) - type(self.mock_suppressed_span.context).suppression = self.mock_suppressed_property + type( + self.mock_suppressed_span.context + ).suppression = self.mock_suppressed_property def test_record_span_with_suppression(self): # Ensure that the queue is empty diff --git a/tests/requirements-310.txt b/tests/requirements-310.txt index 89c0817c..cbd8fe4e 100644 --- a/tests/requirements-310.txt +++ b/tests/requirements-310.txt @@ -29,6 +29,7 @@ protobuf<4.0.0 pymongo>=3.11.4 pyramid>=2.0.1 pytest>=6.2.4 +pytest-mock>=3.12.0 redis>=3.5.3 requests-mock responses<=0.17.0 diff --git a/tests/requirements-312.txt b/tests/requirements-312.txt index 530823b0..bf34cff5 100644 --- a/tests/requirements-312.txt +++ b/tests/requirements-312.txt @@ -1,7 +1,5 @@ aiofiles>=0.5.0 -#aiohttp currently depends on yarl which can't be installed: -#PyLongObject’ {aka ‘struct _longobject’} has no member named ‘ob_digit’ -#aiohttp>=3.8.3 +aiohttp>=3.8.3 boto3>=1.17.74 celery>=5.2.7 coverage>=5.5 @@ -31,12 +29,11 @@ protobuf<4.0.0 pymongo>=3.11.4 pyramid>=2.0.1 pytest>=6.2.4 +pytest-mock>=3.12.0 redis>=3.5.3 requests-mock responses<=0.17.0 -#Sanic depends on uvloop which can't be installed: -#PyLongObject’ {aka ‘struct _longobject’} has no member named ‘ob_digit’ -#sanic==21.6.2 +sanic==21.6.2 sqlalchemy>=2.0.0 spyne>=2.14.0 diff --git a/tests/requirements-313.txt b/tests/requirements-313.txt new file mode 100644 index 00000000..ed419682 --- /dev/null +++ b/tests/requirements-313.txt @@ -0,0 +1,48 @@ +aiofiles>=0.5.0 +aiohttp>=3.8.3 +boto3>=1.17.74 +celery>=5.2.7 +coverage>=5.5 +Django>=5.0a1 --pre +# Dependency orjson has no 3.13 support yet: +# https://github.com/matyasrichter/fastapi-injector/pull/31 +#fastapi>=0.92.0 +flask>=2.3.2 +markupsafe>=2.1.0 +# grpc is not supported on 3.13 yet: +# https://github.com/grpc/grpc/issues/34922 +#grpcio>=1.37.1 +# Depends on grpcio +#google-cloud-pubsub<=2.1.0 +#google-cloud-storage>=1.24.0 +lxml>=4.9.2 +mock>=4.0.3 +moto>=4.1.2 +mysqlclient>=2.0.3 +PyMySQL[rsa]>=1.0.2 +psycopg2-binary>=2.8.6 +pika>=1.2.0 + +# protobuf is pulled in and also `basictracer`, a core instana dependency +# and also by google-cloud-storage +# but also directly needed by tests/apps/grpc_server/stan_pb2.py +# On 4.0.0 we currently get: +# AttributeError: module 'google._upb._message' has no attribute 'Message' +# TODO: Remove this when support for 4.0.0 is done +protobuf<4.0.0 + +pymongo>=3.11.4 +pyramid>=2.0.1 +pytest>=6.2.4 +redis>=3.5.3 +requests-mock +responses<=0.17.0 +# Newer versions of sanic are not supported +# And this old version is not installable on 3.13 because of the `httptools` dependency fails to compile: +# `too few arguments to function ‘_PyLong_AsByteArray’` +#sanic==21.6.2 +sqlalchemy>=2.0.0 +spyne>=2.14.0 + +uvicorn>=0.13.4 +urllib3>=1.26.5 diff --git a/tests/requirements.txt b/tests/requirements.txt index 9b977dc6..065bd553 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -28,6 +28,7 @@ protobuf<4.0.0 pymongo>=3.11.4 pyramid>=2.0.1 pytest>=6.2.4 +pytest-mock>=3.12.0 redis>=3.5.3 requests-mock responses<=0.17.0 diff --git a/tests/test_id_management.py b/tests/test_id_management.py index fb3badbb..c10d2b51 100644 --- a/tests/test_id_management.py +++ b/tests/test_id_management.py @@ -1,56 +1,63 @@ # (c) Copyright IBM Corp. 2021 # (c) Copyright Instana Inc. 2017 -import unittest -import instana - - -class TestIdManagement(unittest.TestCase): - def test_id_generation(self): - count = 0 - while count <= 10000: - id = instana.util.ids.generate_id() - base10_id = int(id, 16) - self.assertGreaterEqual(base10_id, 0) - self.assertLessEqual(base10_id, 18446744073709551615) - count += 1 - - - def test_various_header_to_id_conversion(self): - # Get a hex string to test against & convert - header_id = instana.util.ids.generate_id() - converted_id = instana.util.ids.header_to_long_id(header_id) - self.assertEqual(header_id, converted_id) - - # Hex value - result should be left padded - result = instana.util.ids.header_to_long_id('abcdef') - self.assertEqual('0000000000abcdef', result) +import pytest +from opentelemetry.trace.span import _SPAN_ID_MAX_VALUE, INVALID_SPAN_ID - # Hex value - result = instana.util.ids.header_to_long_id('0123456789abcdef') - self.assertEqual('0123456789abcdef', result) - - # Very long incoming header should just return the rightmost 16 bytes - result = instana.util.ids.header_to_long_id('0x0123456789abcdef0123456789abcdef') - self.assertEqual('0x0123456789abcdef0123456789abcdef', result) - - - def test_header_to_id_conversion_with_bogus_header(self): - # Bogus nil arg - bogus_result = instana.util.ids.header_to_long_id(None) - self.assertEqual(instana.util.ids.BAD_ID, bogus_result) - - # Bogus Integer arg - bogus_result = instana.util.ids.header_to_long_id(1234) - self.assertEqual(instana.util.ids.BAD_ID, bogus_result) - - # Bogus Array arg - bogus_result = instana.util.ids.header_to_long_id([1234]) - self.assertEqual(instana.util.ids.BAD_ID, bogus_result) +import instana - # Bogus Hex Values in String - bogus_result = instana.util.ids.header_to_long_id('0xZZZZZZ') - self.assertEqual(instana.util.ids.BAD_ID, bogus_result) - bogus_result = instana.util.ids.header_to_long_id('ZZZZZZ') - self.assertEqual(instana.util.ids.BAD_ID, bogus_result) +def test_id_generation(): + count = 0 + while count <= 10000: + id = instana.util.ids.generate_id() + assert id >= 0 + assert id > INVALID_SPAN_ID + assert id <= _SPAN_ID_MAX_VALUE + count += 1 + + +@pytest.mark.parametrize( + "str_id, id", + [ + ("BADCAFFE", 3135025150), + ("abcdef", 11259375), + ("0123456789abcdef", 81985529216486895), + ("0x0123456789abcdef0123456789abcdef", 1512366075204170929049582354406559215), + (None, INVALID_SPAN_ID), + (1234, INVALID_SPAN_ID), + ([1234], INVALID_SPAN_ID), + ("0xZZZZZZ", INVALID_SPAN_ID), + ("ZZZZZZ", INVALID_SPAN_ID), + (b"BADCAFFE", 3135025150), + (b"abcdef", 11259375), + (b"0123456789abcdef", 81985529216486895), + (b"0x0123456789abcdef0123456789abcdef", 1512366075204170929049582354406559215), + ], +) +def test_header_to_long_id(str_id, id): + result = instana.util.ids.header_to_long_id(str_id) + assert result == id + + +@pytest.mark.parametrize( + "str_id, id", + [ + ("BADCAFFE", 3135025150), + ("abcdef", 11259375), + ("0123456789abcdef", 81985529216486895), + ("0x0123456789abcdef0123456789abcdef", 81985529216486895), + (None, INVALID_SPAN_ID), + (1234, INVALID_SPAN_ID), + ([1234], INVALID_SPAN_ID), + ("0xZZZZZZ", INVALID_SPAN_ID), + ("ZZZZZZ", INVALID_SPAN_ID), + (b"BADCAFFE", 3135025150), + (b"abcdef", 11259375), + (b"0123456789abcdef", 81985529216486895), + (b"0x0123456789abcdef0123456789abcdef", 81985529216486895), + ], +) +def test_header_to_id(str_id, id): + result = instana.util.ids.header_to_id(str_id) + assert result == id diff --git a/tests/test_span.py b/tests/test_span.py new file mode 100644 index 00000000..bda0333b --- /dev/null +++ b/tests/test_span.py @@ -0,0 +1,727 @@ +# (c) Copyright IBM Corp. 2024 + +import time +from unittest.mock import patch + +import pytest +from opentelemetry.trace.status import Status, StatusCode + +from instana.span import INVALID_SPAN, Event, InstanaSpan, get_current_span +from instana.span_context import SpanContext + + +def test_span_default( + span_context: SpanContext, + trace_id: int, + span_id: int, +) -> None: + span_name = "test-span" + timestamp = time.time_ns() + span = InstanaSpan(span_name, span_context) + + assert span is not None + assert isinstance(span, InstanaSpan) + assert span.name == span_name + + context = span.context + assert isinstance(context, SpanContext) + assert context.trace_id == trace_id + assert context.span_id == span_id + + assert span.start_time + assert isinstance(span.start_time, int) + assert span.start_time > timestamp + assert not span.end_time + assert not span.attributes + assert not span.events + assert span.is_recording() + assert span.status + assert span.status.is_unset + + +def test_span_get_span_context( + span_context: SpanContext, + trace_id: int, + span_id: int, +) -> None: + + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + context = span.get_span_context() + assert isinstance(context, SpanContext) + assert context.trace_id == trace_id + assert context.span_id == span_id + assert context == span.context + + +def test_span_set_attributes_default(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert not span.attributes + + attributes = { + "field1": 1, + "field2": "two", + } + span.set_attributes(attributes) + + assert span.attributes + assert len(span.attributes) == 2 + assert "field1" in span.attributes.keys() + assert "two" == span.attributes.get("field2") + + +def test_span_set_attributes(span_context: SpanContext) -> None: + span_name = "test-span" + attributes = { + "field1": 1, + "field2": "two", + } + span = InstanaSpan(span_name, span_context, attributes=attributes) + + assert span.attributes + assert len(span.attributes) == 2 + assert "field1" in span.attributes.keys() + assert "two" == span.attributes.get("field2") + + attributes = { + "field3": True, + "field4": ["four", "vier", "quatro"], + } + span.set_attributes(attributes) + + assert len(span.attributes) == 4 + assert "field3" in span.attributes.keys() + assert "vier" in span.attributes.get("field4") + + +def test_span_set_attribute_default(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert not span.attributes + + attributes = { + "field1": 1, + "field2": "two", + } + for key, value in attributes.items(): + span.set_attribute(key, value) + + assert span.attributes + assert len(span.attributes) == 2 + assert "field1" in span.attributes.keys() + assert "two" == span.attributes.get("field2") + + +def test_span_set_attribute(span_context: SpanContext) -> None: + span_name = "test-span" + attributes = { + "field1": 1, + "field2": "two", + } + span = InstanaSpan(span_name, span_context, attributes=attributes) + + assert span.attributes + assert len(span.attributes) == 2 + assert "field1" in span.attributes.keys() + assert "two" == span.attributes.get("field2") + + attributes = { + "field3": True, + "field4": ["four", "vier", "quatro"], + } + for key, value in attributes.items(): + span.set_attribute(key, value) + + assert len(span.attributes) == 4 + assert "field3" in span.attributes.keys() + assert "vier" in span.attributes.get("field4") + + +def test_span_update_name(span_context: SpanContext) -> None: + span_name = "test-span-1" + span = InstanaSpan(span_name, span_context) + + assert span is not None + assert isinstance(span, InstanaSpan) + assert span.name == span_name + + new_span_name = "test-span-2" + span.update_name(new_span_name) + assert span is not None + assert isinstance(span, InstanaSpan) + assert span.name == new_span_name + + +def test_span_set_status_with_Status_default(span_context, caplog) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert span.status + assert span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code == StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Status is OK." + span_status = Status(status_code=StatusCode.OK, description=status_desc) + + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + span.set_status(span_status) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + +def test_span_set_status_with_Status_and_desc(span_context, caplog) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert span.status + assert span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code == StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Status is OK." + span_status = Status(status_code=StatusCode.OK, description=status_desc) + + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + set_status_desc = "Test" + span.set_status(span_status, set_status_desc) + excepted_log = f"Description {set_status_desc} ignored. Use either `Status` or `(StatusCode, Description)`" + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert excepted_log == caplog.record_tuples[1][2] + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + +def test_span_set_status_with_StatusUNSET_to_StatusERROR(span_context, caplog) -> None: + span_name = "test-span" + status_desc = "Status is UNSET." + span_status = Status(status_code=StatusCode.UNSET, description=status_desc) + + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + span = InstanaSpan(span_name, span_context, status=span_status) + + assert span.status + assert span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code == StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Houston we have a problem!" + span_status = Status(StatusCode.ERROR, status_desc) + span.set_status(span_status) + + assert span.status + assert not span.status.is_unset + assert not span.status.is_ok + assert span.status.description == status_desc + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code == StatusCode.ERROR + + +def test_span_set_status_with_StatusOK_to_StatusERROR(span_context, caplog) -> None: + span_name = "test-span" + status_desc = "Status is OK." + span_status = Status(status_code=StatusCode.OK, description=status_desc) + + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + span = InstanaSpan(span_name, span_context, status=span_status) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Houston we have a problem!" + span_status = Status(StatusCode.ERROR, status_desc) + span.set_status(span_status) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + +def test_span_set_status_with_StatusCode_default(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert span.status + assert span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code == StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + span_status_code = StatusCode(StatusCode.OK) + + span.set_status(span_status_code) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + +def test_span_set_status_with_StatusCode_and_desc(span_context, caplog) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert span.status + assert span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code == StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Status is OK." + span_status_code = StatusCode(StatusCode.OK) + span.set_status(span_status_code, status_desc) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + +def test_span_set_status_with_StatusCodeUNSET_to_StatusCodeERROR( + span_context, caplog +) -> None: + span_name = "test-span" + status_desc = "Status is UNSET." + span_status = Status(status_code=StatusCode.UNSET, description=status_desc) + + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + span = InstanaSpan(span_name, span_context, status=span_status) + + assert span.status + assert span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code == StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Houston we have a problem!" + span_status_code = StatusCode(StatusCode.ERROR) + span.set_status(span_status_code, status_desc) + + assert span.status + assert not span.status.is_unset + assert not span.status.is_ok + assert span.status.description == status_desc + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code != StatusCode.OK + assert span.status.status_code == StatusCode.ERROR + + +def test_span_set_status_with_StatusCodeOK_to_StatusCodeERROR( + span_context, caplog +) -> None: + span_name = "test-span" + status_desc = "Status is OK." + span_status = Status(status_code=StatusCode.OK, description=status_desc) + + assert ( + "description should only be set when status_code is set to StatusCode.ERROR" + == caplog.record_tuples[0][2] + ) + + span = InstanaSpan(span_name, span_context, status=span_status) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + status_desc = "Houston we have a problem!" + span_status_code = StatusCode(StatusCode.ERROR) + span.set_status(span_status_code, status_desc) + + assert span.status + assert not span.status.is_unset + assert span.status.is_ok + assert not span.status.description + assert span.status.status_code != StatusCode.UNSET + assert span.status.status_code == StatusCode.OK + assert span.status.status_code != StatusCode.ERROR + + +def test_span_add_event_default(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert not span.events + + event_name = "event1" + attributes = { + "field1": 1, + "field2": "two", + } + timestamp = time.time_ns() + span.add_event(event_name, attributes, timestamp) + + assert span.events + assert len(span.events) == 1 + for event in span.events: + assert isinstance(event, Event) + assert event.name == event_name + assert event.timestamp == timestamp + assert len(event.attributes) == 2 + + +def test_span_add_event(span_context: SpanContext) -> None: + span_name = "test-span" + event_name1 = "event1" + attributes = { + "field1": 1, + "field2": "two", + } + timestamp1 = time.time_ns() + event = Event(event_name1, attributes, timestamp1) + span = InstanaSpan(span_name, span_context, events=[event]) + + assert span.events + assert len(span.events) == 1 + for event in span.events: + assert isinstance(event, Event) + assert event.name == event_name1 + assert event.timestamp == timestamp1 + assert len(event.attributes) == 2 + + event_name2 = "event2" + attributes = { + "field3": True, + "field4": ["four", "vier", "quatro"], + } + timestamp2 = time.time_ns() + span.add_event(event_name2, attributes, timestamp2) + + assert len(span.events) == 2 + for event in span.events: + assert isinstance(event, Event) + assert event.name in [event_name1, event_name2] + assert event.timestamp in [timestamp1, timestamp2] + assert len(event.attributes) == 2 + + +@pytest.mark.parametrize( + "span_name, span_attribute", + [ + ("test-span", None), + ("rpc-server", "rpc.error"), + ("rpc-client", "rpc.error"), + ("mysql", "mysql.error"), + ("postgres", "pg.error"), + ("django", "http.error"), + ("http", "http.error"), + ("urllib3", "http.error"), + ("wsgi", "http.error"), + ("asgi", "http.error"), + ("celery-client", "error"), + ("celery-worker", "error"), + ("sqlalchemy", "sqlalchemy.err"), + ("aws.lambda.entry", "lambda.error"), + ], +) +def test_span_record_exception_default( + span_context: SpanContext, + span_name: str, + span_attribute: str, +) -> None: + exception_msg = "Test Exception" + + exception = Exception(exception_msg) + span = InstanaSpan(span_name, span_context) + + span.record_exception(exception) + + assert span_name == span.name + assert 1 == span.attributes.get("ec", 0) + if span_attribute: + assert span_attribute in span.attributes.keys() + assert exception_msg == span.attributes.get(span_attribute, None) + else: + event = span.events[-1] # always get the latest event + assert isinstance(event, Event) + assert "exception" == event.name + assert exception_msg == event.attributes.get("message", None) + + +def test_span_record_exception_with_attribute(span_context: SpanContext) -> None: + span_name = "test-span" + exception_msg = "Test Exception" + attributes = { + "custom_attr": 0, + } + + exception = Exception(exception_msg) + span = InstanaSpan(span_name, span_context) + + span.record_exception(exception, attributes) + + assert span_name == span.name + assert 1 == span.attributes.get("ec", 0) + + event = span.events[-1] # always get the latest event + assert isinstance(event, Event) + assert 2 == len(event.attributes) + assert exception_msg == event.attributes.get("message", None) + assert 0 == event.attributes.get("custom_attr", None) + + +def test_span_record_exception_with_Exception_msg(span_context: SpanContext) -> None: + span_name = "wsgi" + span_attribute = "http.error" + exception_msg = "Test Exception" + + exception = Exception() + exception.message = exception_msg + span = InstanaSpan(span_name, span_context) + + span.record_exception(exception) + + assert span_name == span.name + assert 1 == span.attributes.get("ec", 0) + assert span_attribute in span.attributes.keys() + assert exception_msg == span.attributes.get(span_attribute, None) + + +def test_span_record_exception_with_Exception_none_msg( + span_context: SpanContext, +) -> None: + span_name = "wsgi" + span_attribute = "http.error" + + exception = Exception() + exception.message = None + span = InstanaSpan(span_name, span_context) + + span.record_exception(exception) + + assert span_name == span.name + assert 1 == span.attributes.get("ec", 0) + assert span_attribute in span.attributes.keys() + assert "Exception()" == span.attributes.get(span_attribute, None) + + +def test_span_record_exception_with_Exception_raised(span_context: SpanContext) -> None: + span_name = "test-span" + + exception = None + span = InstanaSpan(span_name, span_context) + + with patch( + "instana.span.InstanaSpan.add_event", side_effect=Exception("mocked error") + ): + with pytest.raises(Exception): + span.record_exception(exception) + + +def test_span_end_default(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert not span.end_time + + span.end() + + assert span.end_time + assert isinstance(span.end_time, int) + assert span.duration + assert isinstance(span.duration, int) + assert span.duration > 0 + + +def test_span_end(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + assert not span.end_time + + timestamp_end = time.time_ns() + span.end(timestamp_end) + + assert span.end_time + assert span.end_time == timestamp_end + assert span.duration + assert isinstance(span.duration, int) + assert span.duration > 0 + assert span.duration == (timestamp_end - span.start_time) + + +def test_span_mark_as_errored_default(span_context: SpanContext) -> None: + span_name = "test-span" + attributes = { + "ec": 0, + } + span = InstanaSpan(span_name, span_context, attributes=attributes) + + assert span.attributes + assert len(span.attributes) == 1 + assert span.attributes.get("ec") == 0 + + span.mark_as_errored() + + assert span.attributes + assert len(span.attributes) == 1 + assert span.attributes.get("ec") == 1 + + +def test_span_mark_as_errored(span_context: SpanContext) -> None: + span_name = "test-span" + attributes = { + "ec": 0, + } + span = InstanaSpan(span_name, span_context, attributes=attributes) + + assert span.attributes + assert len(span.attributes) == 1 + assert span.attributes.get("ec") == 0 + + attributes = { + "field1": 1, + "field2": "two", + } + span.mark_as_errored(attributes) + + assert span.attributes + assert len(span.attributes) == 3 + assert span.attributes.get("ec") == 1 + assert "field1" in span.attributes.keys() + assert span.attributes.get("field2") == "two" + + span.mark_as_errored() + + assert span.attributes + assert len(span.attributes) == 3 + assert span.attributes.get("ec") == 2 + assert "field1" in span.attributes.keys() + assert span.attributes.get("field2") == "two" + + +def test_span_mark_as_errored_exception(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + with patch( + "instana.span.InstanaSpan.set_attribute", side_effect=Exception("mocked error") + ): + span.mark_as_errored() + assert not span.attributes + + +def test_span_assure_errored_default(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + span.assure_errored() + + assert span.attributes + assert len(span.attributes) == 1 + assert span.attributes.get("ec") == 1 + + +def test_span_assure_errored(span_context: SpanContext) -> None: + span_name = "test-span" + attributes = { + "ec": 0, + } + span = InstanaSpan(span_name, span_context, attributes=attributes) + + assert span.attributes + assert len(span.attributes) == 1 + assert span.attributes.get("ec") == 0 + + span.assure_errored() + + assert span.attributes + assert len(span.attributes) == 1 + assert span.attributes.get("ec") == 1 + + +def test_span_assure_errored_exception(span_context: SpanContext) -> None: + span_name = "test-span" + span = InstanaSpan(span_name, span_context) + + with patch( + "instana.span.InstanaSpan.set_attribute", side_effect=Exception("mocked error") + ): + span.assure_errored() + assert not span.attributes + + +def test_get_current_span(context) -> None: + span = get_current_span(context) + assert isinstance(span, InstanaSpan) + + +def test_get_current_span_INVALID_SPAN() -> None: + span = get_current_span() + + assert span + assert span == INVALID_SPAN diff --git a/tests/test_span_base.py b/tests/test_span_base.py new file mode 100644 index 00000000..f4f8a66c --- /dev/null +++ b/tests/test_span_base.py @@ -0,0 +1,157 @@ +# (c) Copyright IBM Corp. 2024 + +from unittest.mock import Mock, patch + +from instana.span import BaseSpan, InstanaSpan +from instana.span_context import SpanContext +from instana.util import DictionaryOfStan + + +def test_basespan( + span: InstanaSpan, + trace_id: int, + span_id: int, +) -> None: + base_span = BaseSpan(span, None, "test") + + expected_dict = { + "t": trace_id, + "p": None, + "s": span_id, + "l": 1, + "ts": round(span.start_time / 10**6), + "d": round(span.duration / 10**6), + "f": None, + "ec": None, + "data": DictionaryOfStan(), + "stack": None, + } + + assert expected_dict["t"] == base_span.t + assert expected_dict["s"] == base_span.s + assert expected_dict["p"] == base_span.p + assert expected_dict["l"] == base_span.l + assert expected_dict["ts"] == base_span.ts + assert expected_dict["d"] == base_span.d + assert not base_span.f + assert expected_dict["ec"] == base_span.ec + assert isinstance(base_span.data, dict) + assert expected_dict["stack"] == base_span.stack + assert not base_span.sy + + expected_dict_str = str(expected_dict) + assert expected_dict_str == repr(base_span) + assert f"BaseSpan({expected_dict_str})" == str(base_span) + + +def test_basespan_with_synthetic_source_and_kwargs( + span: InstanaSpan, + trace_id: int, + span_id: int, +) -> None: + span.synthetic = True + source = "source test" + _kwarg1 = "value1" + base_span = BaseSpan(span, source, "test", arg1=_kwarg1) + + assert trace_id == base_span.t + assert span_id == base_span.s + assert base_span.sy + assert source == base_span.f + assert _kwarg1 == base_span.arg1 + + +def test_populate_extra_span_attributes(span: InstanaSpan) -> None: + base_span = BaseSpan(span, None, "test") + base_span._populate_extra_span_attributes(span) + + assert not hasattr(base_span, "tp") + assert not hasattr(base_span, "tp") + assert not hasattr(base_span, "ia") + assert not hasattr(base_span, "lt") + assert not hasattr(base_span, "crtp") + assert not hasattr(base_span, "crid") + + +def test_populate_extra_span_attributes_with_values( + trace_id: int, + span_id: int, +) -> None: + long_id = 1512366075204170929049582354406559215 + span_context = SpanContext( + trace_id=trace_id, + span_id=span_id, + is_remote=False, + synthetic=True, + trace_parent=True, + instana_ancestor="IDK", + long_trace_id=long_id, + correlation_type="IDK", + correlation_id=long_id, + ) + span = InstanaSpan("test-base-span", span_context) + base_span = BaseSpan(span, None, "test") + base_span._populate_extra_span_attributes(span) + + assert trace_id == base_span.t + assert span_id == base_span.s + assert base_span.sy + assert base_span.tp + assert "IDK" == base_span.ia + assert long_id == base_span.lt + assert "IDK" == base_span.crtp + assert long_id == base_span.crid + + +def test_validate_attributes(base_span: BaseSpan) -> None: + attributes = { + "field1": 1, + "field2": "two", + } + filtered_attributes = base_span._validate_attributes(attributes) + + assert isinstance(filtered_attributes, dict) + assert len(attributes) == len(filtered_attributes) + for key, value in attributes.items(): + assert key in filtered_attributes.keys() + assert value in filtered_attributes.values() + + +def test_validate_attribute_with_invalid_key_type(base_span: BaseSpan) -> None: + key = 1 + value = "one" + + (validated_key, validated_value) = base_span._validate_attribute(key, value) + + assert not validated_key + assert not validated_value + + +def test_validate_attribute_exception(span: InstanaSpan) -> None: + base_span = BaseSpan(span, None, "test") + key = "field1" + value = span + + with patch( + "instana.span.BaseSpan._convert_attribute_value", + side_effect=Exception("mocked error"), + ): + (validated_key, validated_value) = base_span._validate_attribute(key, value) + assert key == validated_key + assert not validated_value + + +def test_convert_attribute_value(span: InstanaSpan) -> None: + base_span = BaseSpan(span, None, "test") + value = span + + converted_value = base_span._convert_attribute_value(value) + assert " None: + mock = Mock() + mock.__repr__ = Mock(side_effect=Exception("mocked error")) + + converted_value = base_span._convert_attribute_value(mock) + assert not converted_value diff --git a/tests/test_span_context.py b/tests/test_span_context.py new file mode 100644 index 00000000..517007f8 --- /dev/null +++ b/tests/test_span_context.py @@ -0,0 +1,66 @@ +# (c) Copyright IBM Corp. 2024 + +import pickle +from opentelemetry.trace.span import ( + DEFAULT_TRACE_OPTIONS, + DEFAULT_TRACE_STATE, + format_span_id, +) + +from instana.span_context import SpanContext +from instana.util.ids import generate_id + + +def test_span_context_defaults(): + trace_id = generate_id() + span_id = generate_id() + span_context = SpanContext( + trace_id=trace_id, + span_id=span_id, + is_remote=False, + ) + + assert isinstance(span_context, SpanContext) + assert span_context.trace_id == trace_id + assert span_context.span_id == span_id + assert span_context.trace_id != span_context.span_id + assert not span_context.is_remote + assert span_context.trace_flags == DEFAULT_TRACE_OPTIONS + assert span_context.trace_state == DEFAULT_TRACE_STATE + assert span_context.is_valid + assert span_context.level == 1 + assert not span_context.synthetic + assert span_context.trace_parent is None + assert span_context.instana_ancestor is None + assert span_context.long_trace_id is None + assert span_context.correlation_type is None + assert span_context.correlation_id is None + assert span_context.traceparent is None + assert span_context.tracestate is None + assert not span_context.suppression + assert repr(span_context) == f"SpanContext(trace_id=0x{format_span_id(trace_id)}, span_id=0x{format_span_id(span_id)}, trace_flags=0x{DEFAULT_TRACE_OPTIONS:02x}, trace_state={DEFAULT_TRACE_STATE!r}, is_remote=False, synthetic=False)" + + +def test_span_context_invalid(): + span_context = SpanContext( + trace_id=9999999999999999999999999999999999999999999999999999999999999999999999999999, + span_id=9, + is_remote=False, + ) + assert not span_context.is_valid + + +def test_span_context_pickle(): + trace_id = generate_id() + span_id = generate_id() + span_context = SpanContext( + trace_id=trace_id, + span_id=span_id, + is_remote=False, + ) + + span_context_binary = pickle.dumps(span_context) + span_context_pickle = pickle.loads(span_context_binary) + assert trace_id == span_context_pickle.trace_id + assert span_id == span_context_pickle.span_id + diff --git a/tests/test_span_event.py b/tests/test_span_event.py new file mode 100644 index 00000000..cdfc724a --- /dev/null +++ b/tests/test_span_event.py @@ -0,0 +1,35 @@ +# (c) Copyright IBM Corp. 2024 + +import time +from instana.span import Event + + +def test_span_event_defaults(): + event_name = "test-span-event" + event = Event(event_name) + + assert event + assert isinstance(event, Event) + assert event.name == event_name + assert not event.attributes + assert isinstance(event.timestamp, int) + + +def test_span_event(): + event_name = "test-span-event" + attributes = { + "field1": 1, + "field2": "two", + } + timestamp = time.time_ns() + + event = Event(event_name, attributes, timestamp) + + assert event + assert isinstance(event, Event) + assert event.name == event_name + assert event.attributes + assert len(event.attributes) == 2 + assert "field1" in event.attributes.keys() + assert "two" == event.attributes.get("field2") + assert event.timestamp == timestamp diff --git a/tests/test_span_registered.py b/tests/test_span_registered.py new file mode 100644 index 00000000..c184d940 --- /dev/null +++ b/tests/test_span_registered.py @@ -0,0 +1,408 @@ +# (c) Copyright IBM Corp. 2024 + +import time +from typing import Any, Dict, Tuple + +import pytest + +from instana.span import InstanaSpan, RegisteredSpan +from instana.span_context import SpanContext + + +@pytest.mark.parametrize( + "span_name, expected_result, attributes", + [ + ("wsgi", ("wsgi", 1, "http"), {}), + ("rabbitmq", ("rabbitmq", 1, "rabbitmq"), {}), + ("gcps-producer", ("gcps", 2, "gcps"), {}), + ("urllib3", ("urllib3", 2, "http"), {}), + ("rabbitmq", ("rabbitmq", 2, "rabbitmq"), {"sort": "publish"}), + ("render", ("render", 3, "render"), {"arguments": "--quiet"}), + ], +) +def test_registered_span( + span_context: SpanContext, + span_name: str, + expected_result: Tuple[str, int, str], + attributes: Dict[str, Any] +) -> None: + service_name = "test-registered-service" + span = InstanaSpan(span_name, span_context, attributes=attributes) + reg_span = RegisteredSpan(span, None, service_name) + + assert expected_result[0] == reg_span.n + assert expected_result[1] == reg_span.k + assert service_name == reg_span.data["service"] + assert expected_result[2] in reg_span.data.keys() + + +def test_collect_http_attributes_with_attributes(span_context: SpanContext) -> None: + span_name = "test-registered-span" + attributes = { + "span.kind": "entry", + "http.host": "localhost", + "http.url": "https://www.instana.com", + "http.header.test": "one more test", + } + service_name = "test-registered-service" + span = InstanaSpan(span_name, span_context, attributes=attributes) + reg_span = RegisteredSpan(span, None, service_name) + + excepted_result = { + "http.host": attributes["http.host"], + "http.url": attributes["http.url"], + "http.header.test": attributes["http.header.test"], + } + + reg_span._collect_http_attributes(span) + + assert excepted_result["http.host"] == reg_span.data["http"]["host"] + assert excepted_result["http.url"] == reg_span.data["http"]["url"] + assert excepted_result["http.header.test"] == reg_span.data["http"]["header"]["test"] + + +def test_populate_local_span_data_with_other_name(span_context: SpanContext, caplog) -> None: + # span_name = "test-registered-span" + # service_name = "test-registered-service" + # span = InstanaSpan(span_name, span_context) + # reg_span = RegisteredSpan(span, None, service_name) + + # expected_msg = f"SpanRecorder: Unknown local span: {span_name}" + + # reg_span._populate_local_span_data(span) + + # assert expected_msg == caplog.record_tuples[0][2] + pass + + +@pytest.mark.parametrize( + "span_name, service_name, attributes", + [ + ( + "aws.lambda.entry", + "lambda", + { + "lambda.arn": "test", + "lambda.trigger": None, + }, + ), + ( + "celery-worker", + "celery", + { + "host": "localhost", + "port": 1234, + }, + ), + ( + "gcps-consumer", + "gcps", + { + "gcps.op": "consume", + "gcps.projid": "MY_PROJECT", + "gcps.sub": "MY_SUBSCRIPTION_NAME", + }, + ), + ( + "rpc-server", + "rpc", + { + "rpc.flavor": "Vanilla", + "rpc.host": "localhost", + "rpc.port": 1234, + }, + ), + ], +) +def test_populate_entry_span_data( + span_context: SpanContext, + span_name: str, + service_name: str, + attributes: Dict[str, Any] +) -> None: + span = InstanaSpan(span_name, span_context) + reg_span = RegisteredSpan(span, None, service_name) + + expected_result = {} + for attr, value in attributes.items(): + attrl = attr.split(".") + attrl = attrl[1] if len(attrl) > 1 else attrl[0] + expected_result[attrl] = value + + span.set_attributes(attributes) + reg_span._populate_entry_span_data(span) + + for attr, value in expected_result.items(): + assert value == reg_span.data[service_name][attr] + + +@pytest.mark.parametrize( + "attributes", + [ + { + "lambda.arn": "test", + "lambda.trigger": "aws:api.gateway", + "http.host": "localhost", + "http.url": "https://www.instana.com", + + }, + { + "lambda.arn": "test", + "lambda.trigger": "aws:cloudwatch.events", + "lambda.cw.events.resources": "Resource 1", + }, + { + "lambda.arn": "test", + "lambda.trigger": "aws:cloudwatch.logs", + "lambda.cw.logs.group": "My Group", + }, + { + "lambda.arn": "test", + "lambda.trigger": "aws:s3", + "lambda.s3.events": "Event 1", + }, + { + "lambda.arn": "test", + "lambda.trigger": "aws:sqs", + "lambda.sqs.messages": "Message 1", + }, + ], +) +def test_populate_entry_span_data_AWSlambda( + span_context: SpanContext, + attributes: Dict[str, Any] +) -> None: + span_name = "aws.lambda.entry" + service_name = "lambda" + expected_result = attributes.copy() + + span = InstanaSpan(span_name, span_context) + reg_span = RegisteredSpan(span, None, service_name) + + span.set_attributes(attributes) + reg_span._populate_entry_span_data(span) + + assert "python" == reg_span.data["lambda"]["runtime"] + assert "Unknown" == reg_span.data["lambda"]["functionName"] + assert "test" == reg_span.data["lambda"]["arn"] + assert expected_result["lambda.trigger"] == reg_span.data["lambda"]["trigger"] + + if expected_result["lambda.trigger"] == "aws:api.gateway": + assert expected_result["http.host"] == reg_span.data["http"]["host"] + assert expected_result["http.url"] == reg_span.data["http"]["url"] + + elif expected_result["lambda.trigger"] == "aws:cloudwatch.events": + assert expected_result["lambda.cw.events.resources"] == reg_span.data["lambda"]["cw"]["events"]["resources"] + elif expected_result["lambda.trigger"] == "aws:cloudwatch.logs": + assert expected_result["lambda.cw.logs.group"] == reg_span.data["lambda"]["cw"]["logs"]["group"] + elif expected_result["lambda.trigger"] == "aws:s3": + assert expected_result["lambda.s3.events"] == reg_span.data["lambda"]["s3"]["events"] + elif expected_result["lambda.trigger"] == "aws:sqs": + assert expected_result["lambda.sqs.messages"] == reg_span.data["lambda"]["sqs"]["messages"] + +@pytest.mark.parametrize( + "span_name, service_name, attributes", + [ + ( + "cassandra", + "cassandra", + { + "cassandra.cluster": "my_cluster", + "cassandra.error": "minor error", + }, + ), + ( + "celery-client", + "celery", + { + "host": "localhost", + "port": 1234, + }, + ), + ( + "couchbase", + "couchbase", + { + "couchbase.hostname": "localhost", + "couchbase.error_type": 1234, + }, + ), + ( + "rabbitmq", + "rabbitmq", + { + "address": "localhost", + "key": 1234, + }, + ), + ( + "redis", + "redis", + { + "command": "ls -l", + "redis.error": "minor error", + }, + ), + ( + "rpc-client", + "rpc", + { + "rpc.flavor": "Vanilla", + "rpc.host": "localhost", + "rpc.port": 1234, + }, + ), + ( + "sqlalchemy", + "sqlalchemy", + { + "sqlalchemy.sql": "SELECT * FROM everything;", + "sqlalchemy.err": "Impossible select everything from everything!", + }, + ), + ( + "mysql", + "mysql", + { + "host": "localhost", + "port": 1234, + }, + ), + ( + "postgres", + "pg", + { + "host": "localhost", + "port": 1234, + }, + ), + ( + "mongo", + "mongo", + { + "command": "IDK", + "error": "minor error", + }, + ), + ( + "gcs", + "gcs", + { + "gcs.op": "produce", + "gcs.projectId": "MY_PROJECT", + "gcs.accessId": "Can not tell you!", + }, + ), + ( + "gcps-producer", + "gcps", + { + "gcps.op": "produce", + "gcps.projid": "MY_PROJECT", + "gcps.top": "MY_SUBSCRIPTION_NAME", + }, + ), + ], +) +def test_populate_exit_span_data( + span_context: SpanContext, + span_name: str, + service_name: str, + attributes: Dict[str, Any] +) -> None: + span = InstanaSpan(span_name, span_context) + reg_span = RegisteredSpan(span, None, service_name) + + expected_result = {} + for attr, value in attributes.items(): + attrl = attr.split(".") + attrl = attrl[1] if len(attrl) > 1 else attrl[0] + expected_result[attrl] = value + + span.set_attributes(attributes) + reg_span._populate_exit_span_data(span) + + for attr, value in expected_result.items(): + assert value == reg_span.data[service_name][attr] + + +@pytest.mark.parametrize( + "attributes", + [ + { + "op": "test", + "http.host": "localhost", + "http.url": "https://www.instana.com", + }, + { + "payload": { + "blah": "bleh", + "blih": "bloh", + }, + "http.host": "localhost", + "http.url": "https://www.instana.com", + }, + ], +) +def test_populate_exit_span_data_boto3( + span_context: SpanContext, + attributes: Dict[str, Any] +) -> None: + span_name = service_name = "boto3" + expected_result = attributes.copy() + + + span = InstanaSpan(span_name, span_context) + reg_span = RegisteredSpan(span, None, service_name) + + # expected_result = {} + # for attr, value in attributes.items(): + # attrl = attr.split(".") + # attrl = attrl[1] if len(attrl) > 1 else attrl[0] + # expected_result[attrl] = value + + span.set_attributes(attributes) + reg_span._populate_exit_span_data(span) + + assert expected_result.pop("http.host", None) == reg_span.data["http"]["host"] + assert expected_result.pop("http.url", None) == reg_span.data["http"]["url"] + + for attr, value in expected_result.items(): + assert value == reg_span.data[service_name][attr] + + + +def test_populate_exit_span_data_log(span_context: SpanContext) -> None: + span_name = service_name = "log" + span = InstanaSpan(span_name, span_context) + reg_span = RegisteredSpan(span, None, service_name) + + excepted_text = "Houston, we have a problem!" + events = [ + ( + "test_populate_exit_span_data_log_event_with_message", + { + "field1": 1, + "field2": "two", + "message": excepted_text, + }, + time.time_ns(), + ), + ( + "test_populate_exit_span_data_log_event_with_parameters", + { + "field1": 1, + "field2": "two", + "parameters": excepted_text, + }, + time.time_ns(), + ), + ] + + for (event_name, attributes, timestamp) in events: + span.add_event(event_name, attributes, timestamp) + + reg_span._populate_exit_span_data(span) + + assert excepted_text == reg_span.data["event"]["message"] + assert excepted_text == reg_span.data["event"]["parameters"] diff --git a/tests/test_span_sdk.py b/tests/test_span_sdk.py new file mode 100644 index 00000000..c7fc8d98 --- /dev/null +++ b/tests/test_span_sdk.py @@ -0,0 +1,83 @@ +# (c) Copyright IBM Corp. 2024 + +from typing import Tuple + +import pytest + +from instana.span import InstanaSpan, SDKSpan +from instana.span_context import SpanContext + + +def test_sdkspan(span_context: SpanContext) -> None: + span_name = "test-sdk-span" + service_name = "test-sdk" + attributes = { + "span.kind": "entry", + "arguments": "--quiet", + "return": "True", + } + span = InstanaSpan(span_name, span_context, attributes=attributes) + sdk_span = SDKSpan(span, None, service_name) + + expected_result = { + "n": "sdk", + "k": 1, + "data": { + "service": service_name, + "sdk": { + "name": span_name, + "type": attributes["span.kind"], + "custom": { + "attributes": attributes, + }, + "arguments": attributes["arguments"], + "return": attributes["return"], + }, + }, + } + + assert expected_result["n"] == sdk_span.n + assert expected_result["k"] == sdk_span.k + assert len(expected_result["data"]) == len(sdk_span.data) + assert expected_result["data"]["service"] == sdk_span.data["service"] + assert len(expected_result["data"]["sdk"]) == len(sdk_span.data["sdk"]) + assert expected_result["data"]["sdk"]["name"] == sdk_span.data["sdk"]["name"] + assert expected_result["data"]["sdk"]["type"] == sdk_span.data["sdk"]["type"] + assert len(attributes) == len(sdk_span.data["sdk"]["custom"]["attributes"]) + assert attributes == sdk_span.data["sdk"]["custom"]["attributes"] + assert attributes["arguments"] == sdk_span.data["sdk"]["arguments"] + assert attributes["return"] == sdk_span.data["sdk"]["return"] + + +@pytest.mark.parametrize( + "span_kind, expected_result", + [ + (None, ("intermediate", 3)), + ("entry", ("entry", 1)), + ("server", ("entry", 1)), + ("consumer", ("entry", 1)), + ("exit", ("exit", 2)), + ("client", ("exit", 2)), + ("producer", ("exit", 2)), + ], +) +def test_sdkspan_get_span_kind( + span_context: SpanContext, + span_kind: str, + expected_result: Tuple[str, int], +) -> None: + attributes = { + "span.kind": span_kind, + } + span = InstanaSpan("test-sdk-span", span_context, attributes=attributes) + sdk_span = SDKSpan(span, None, "test") + + kind = sdk_span.get_span_kind(span) + + assert expected_result == kind + + +def test_sdkspan_get_span_kind_with_no_attributes(span: InstanaSpan) -> None: + sdk_span = SDKSpan(span, None, "test") + kind = sdk_span.get_span_kind(span) + assert ("intermediate", 3) == kind diff --git a/tests/test_tracer.py b/tests/test_tracer.py new file mode 100644 index 00000000..feae19cc --- /dev/null +++ b/tests/test_tracer.py @@ -0,0 +1,153 @@ +# (c) Copyright IBM Corp. 2024 + +from opentelemetry.trace import set_span_in_context +from opentelemetry.trace.span import _SPAN_ID_MAX_VALUE, INVALID_SPAN_ID +import pytest + +from instana.span import InstanaSpan +from instana.span_context import SpanContext +from instana.tracer import InstanaTracer, InstanaTracerProvider + + +def test_tracer_defaults() -> None: + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + + assert tracer.tracer_id > INVALID_SPAN_ID + assert tracer.tracer_id <= _SPAN_ID_MAX_VALUE + assert tracer.recorder == provider.recorder + assert tracer._sampler == provider.sampler + assert tracer._span_processor == provider._span_processor + assert tracer._propagators == provider._propagators + +def test_tracer_start_span(span) -> None: + span_name = "test-span" + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + parent_context = set_span_in_context(span) + span = tracer.start_span(name=span_name, context=parent_context) + + assert span + assert isinstance(span, InstanaSpan) + assert span.name == span_name + assert not span.stack + + +def test_tracer_start_span_with_stack(span: InstanaSpan) -> None: + span_name = "log" + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + span = tracer.start_span(name=span_name) + + assert span + assert isinstance(span, InstanaSpan) + assert span.name == span_name + assert span.stack + + stack_0 = span.stack[0] + assert 3 == len(stack_0) + assert "c" in stack_0.keys() + assert "n" in stack_0.keys() + assert "m" in stack_0.keys() + + +def test_tracer_start_span_Exception(mocker, span) -> None: + span_name = "test-span" + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + + parent_context = set_span_in_context(span) + + mocker.patch("instana.span.InstanaSpan.get_span_context", return_value={"key": "value"}) + with pytest.raises(TypeError): + tracer.start_span(name=span_name, context=parent_context) + + +def test_tracer_start_as_current_span() -> None: + span_name = "test-span" + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + with tracer.start_as_current_span(name=span_name) as span: + assert span is not None + assert isinstance(span, InstanaSpan) + assert span.name == span_name + + +def test_tracer_create_span_context(span_context: SpanContext) -> None: + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + new_span_context = tracer._create_span_context(span_context) + + assert span_context.trace_id == new_span_context.trace_id + assert span_context.span_id != new_span_context.span_id + assert span_context.long_trace_id == new_span_context.long_trace_id + + +def test_tracer_add_stack_high_limit(span: InstanaSpan) -> None: + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + tracer._add_stack(span, 50) + + assert span.stack + assert 40 >= len(span.stack) + + stack_0 = span.stack[0] + assert 3 == len(stack_0) + assert "c" in stack_0.keys() + assert "n" in stack_0.keys() + assert "m" in stack_0.keys() + + +def test_tracer_add_stack_low_limit(span: InstanaSpan) -> None: + provider = InstanaTracerProvider() + tracer = InstanaTracer( + provider.sampler, + provider.recorder, + provider._span_processor, + provider._propagators, + ) + tracer._add_stack(span, 5) + + assert span.stack + assert 5 >= len(span.stack) + + stack_0 = span.stack[0] + assert 3 == len(stack_0) + assert "c" in stack_0.keys() + assert "n" in stack_0.keys() + assert "m" in stack_0.keys() diff --git a/tests/test_tracer_provider.py b/tests/test_tracer_provider.py new file mode 100644 index 00000000..d5222fad --- /dev/null +++ b/tests/test_tracer_provider.py @@ -0,0 +1,54 @@ +# (c) Copyright IBM Corp. 2024 + +from opentelemetry.trace.span import _SPAN_ID_MAX_VALUE, INVALID_SPAN_ID +from pytest import LogCaptureFixture + +from instana.agent.host import HostAgent +from instana.agent.test import TestAgent +from instana.propagators.binary_propagator import BinaryPropagator +from instana.propagators.format import Format +from instana.propagators.http_propagator import HTTPPropagator +from instana.propagators.text_propagator import TextPropagator +from instana.recorder import StanRecorder +from instana.sampling import InstanaSampler +from instana.tracer import InstanaTracer, InstanaTracerProvider + + +def test_tracer_provider_defaults() -> None: + provider = InstanaTracerProvider() + assert isinstance(provider.sampler, InstanaSampler) + assert isinstance(provider.recorder, StanRecorder) + assert isinstance(provider._span_processor, HostAgent) + assert len(provider._propagators) == 3 + assert isinstance(provider._propagators[Format.HTTP_HEADERS], HTTPPropagator) + assert isinstance(provider._propagators[Format.TEXT_MAP], TextPropagator) + assert isinstance(provider._propagators[Format.BINARY], BinaryPropagator) + + +def test_tracer_provider_get_tracer() -> None: + provider = InstanaTracerProvider() + tracer = provider.get_tracer("instana.test.tracer") + + assert isinstance(tracer, InstanaTracer) + assert tracer.tracer_id > INVALID_SPAN_ID + assert tracer.tracer_id <= _SPAN_ID_MAX_VALUE + + +def test_tracer_provider_get_tracer_empty_instrumenting_module_name( + caplog: LogCaptureFixture, +) -> None: + provider = InstanaTracerProvider() + tracer = provider.get_tracer("") + + assert "get_tracer called with missing module name." == caplog.record_tuples[0][2] + assert isinstance(tracer, InstanaTracer) + assert tracer.tracer_id > INVALID_SPAN_ID + assert tracer.tracer_id <= _SPAN_ID_MAX_VALUE + + +def test_tracer_provider_add_span_processor() -> None: + provider = InstanaTracerProvider() + assert isinstance(provider._span_processor, HostAgent) + + provider.add_span_processor(TestAgent()) + assert isinstance(provider._span_processor, TestAgent)