From e373d2d7bcc66d6702104871c9cb8f9acd8a7508 Mon Sep 17 00:00:00 2001 From: Jinsong Wang <64242752+jinsongo@users.noreply.github.com> Date: Fri, 11 Oct 2024 02:11:53 -0700 Subject: [PATCH] feat(sdk): add OpenTelemetry logging support (#2112) --- packages/traceloop-sdk/poetry.lock | 69 +++++++++++------- packages/traceloop-sdk/pyproject.toml | 1 + .../traceloop-sdk/traceloop/sdk/__init__.py | 39 ++++++---- .../traceloop/sdk/config/__init__.py | 4 ++ .../traceloop/sdk/logging/__init__.py | 0 .../traceloop/sdk/logging/logging.py | 71 +++++++++++++++++++ 6 files changed, 144 insertions(+), 40 deletions(-) create mode 100644 packages/traceloop-sdk/traceloop/sdk/logging/__init__.py create mode 100644 packages/traceloop-sdk/traceloop/sdk/logging/logging.py diff --git a/packages/traceloop-sdk/poetry.lock b/packages/traceloop-sdk/poetry.lock index ca3e86c2a..3171055c1 100644 --- a/packages/traceloop-sdk/poetry.lock +++ b/packages/traceloop-sdk/poetry.lock @@ -1417,7 +1417,7 @@ wrapt = ">=1.0.0,<2.0.0" [[package]] name = "opentelemetry-instrumentation-alephalpha" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Aleph Alpha instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1439,7 +1439,7 @@ url = "../opentelemetry-instrumentation-alephalpha" [[package]] name = "opentelemetry-instrumentation-anthropic" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Anthropic instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1461,7 +1461,7 @@ url = "../opentelemetry-instrumentation-anthropic" [[package]] name = "opentelemetry-instrumentation-bedrock" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Bedrock instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1481,7 +1481,7 @@ url = "../opentelemetry-instrumentation-bedrock" [[package]] name = "opentelemetry-instrumentation-chromadb" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Chroma DB instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1503,7 +1503,7 @@ url = "../opentelemetry-instrumentation-chromadb" [[package]] name = "opentelemetry-instrumentation-cohere" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Cohere instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1525,7 +1525,7 @@ url = "../opentelemetry-instrumentation-cohere" [[package]] name = "opentelemetry-instrumentation-google-generativeai" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Google Generative AI instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1547,7 +1547,7 @@ url = "../opentelemetry-instrumentation-google-generativeai" [[package]] name = "opentelemetry-instrumentation-groq" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Groq instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1569,7 +1569,7 @@ url = "../opentelemetry-instrumentation-groq" [[package]] name = "opentelemetry-instrumentation-haystack" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Haystack instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1591,7 +1591,7 @@ url = "../opentelemetry-instrumentation-haystack" [[package]] name = "opentelemetry-instrumentation-lancedb" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Lancedb instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1613,7 +1613,7 @@ url = "../opentelemetry-instrumentation-lancedb" [[package]] name = "opentelemetry-instrumentation-langchain" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Langchain instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1635,7 +1635,7 @@ url = "../opentelemetry-instrumentation-langchain" [[package]] name = "opentelemetry-instrumentation-llamaindex" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry LlamaIndex instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1656,9 +1656,24 @@ instruments = [] type = "directory" url = "../opentelemetry-instrumentation-llamaindex" +[[package]] +name = "opentelemetry-instrumentation-logging" +version = "0.48b0" +description = "OpenTelemetry Logging instrumentation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_logging-0.48b0-py3-none-any.whl", hash = "sha256:75e5357d9b8c12071a19e1fef664dc1f430ef45874445c324ba4439a00972dc0"}, + {file = "opentelemetry_instrumentation_logging-0.48b0.tar.gz", hash = "sha256:529eb13eedf57d6b2f94e20e996271db2957b817b9457fe4796365d6d4238dec"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.48b0" + [[package]] name = "opentelemetry-instrumentation-marqo" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Marqo instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1680,7 +1695,7 @@ url = "../opentelemetry-instrumentation-marqo" [[package]] name = "opentelemetry-instrumentation-milvus" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Milvus instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1702,7 +1717,7 @@ url = "../opentelemetry-instrumentation-milvus" [[package]] name = "opentelemetry-instrumentation-mistralai" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Mistral AI instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1724,7 +1739,7 @@ url = "../opentelemetry-instrumentation-mistralai" [[package]] name = "opentelemetry-instrumentation-ollama" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Ollama instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1738,7 +1753,7 @@ opentelemetry-semantic-conventions = "^0.48b0" opentelemetry-semantic-conventions-ai = "0.4.1" [package.extras] -instruments = ["ollama (>=0.2.0,<0.3.0)"] +instruments = ["ollama (>=0.3.2,<0.4.0)"] [package.source] type = "directory" @@ -1746,7 +1761,7 @@ url = "../opentelemetry-instrumentation-ollama" [[package]] name = "opentelemetry-instrumentation-openai" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry OpenAI instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1769,7 +1784,7 @@ url = "../opentelemetry-instrumentation-openai" [[package]] name = "opentelemetry-instrumentation-pinecone" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Pinecone instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1791,7 +1806,7 @@ url = "../opentelemetry-instrumentation-pinecone" [[package]] name = "opentelemetry-instrumentation-qdrant" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Qdrant instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1813,7 +1828,7 @@ url = "../opentelemetry-instrumentation-qdrant" [[package]] name = "opentelemetry-instrumentation-replicate" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Replicate instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1855,7 +1870,7 @@ instruments = ["requests (>=2.0,<3.0)"] [[package]] name = "opentelemetry-instrumentation-sagemaker" -version = "0.25.6" +version = "0.32.2" description = "OpenTelemetry SageMaker instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1911,7 +1926,7 @@ wrapt = ">=1.0.0,<2.0.0" [[package]] name = "opentelemetry-instrumentation-together" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Together AI instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1933,7 +1948,7 @@ url = "../opentelemetry-instrumentation-together" [[package]] name = "opentelemetry-instrumentation-transformers" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry transformers instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1973,7 +1988,7 @@ instruments = ["urllib3 (>=1.0.0,<3.0.0)"] [[package]] name = "opentelemetry-instrumentation-vertexai" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Vertex AI instrumentation" optional = false python-versions = ">=3.9,<4" @@ -1995,7 +2010,7 @@ url = "../opentelemetry-instrumentation-vertexai" [[package]] name = "opentelemetry-instrumentation-watsonx" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry IBM Watsonx Instrumentation" optional = false python-versions = ">=3.9,<4" @@ -2017,7 +2032,7 @@ url = "../opentelemetry-instrumentation-watsonx" [[package]] name = "opentelemetry-instrumentation-weaviate" -version = "0.31.0" +version = "0.32.2" description = "OpenTelemetry Weaviate instrumentation" optional = false python-versions = ">=3.9,<4" @@ -3275,4 +3290,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<4" -content-hash = "d42935daa01b14ea3f8b44a647ab73819a17fded081b362d67d6f91617c7fea5" +content-hash = "38ca90d48eb8fefebc864dcb62ad31b7f018bb3439beeeed1ccafe06ba1d1336" diff --git a/packages/traceloop-sdk/pyproject.toml b/packages/traceloop-sdk/pyproject.toml index 31fecc7f0..52d411838 100644 --- a/packages/traceloop-sdk/pyproject.toml +++ b/packages/traceloop-sdk/pyproject.toml @@ -29,6 +29,7 @@ opentelemetry-api = "^1.27.0" opentelemetry-sdk = "^1.27.0" opentelemetry-exporter-otlp-proto-http = "^1.26.0" opentelemetry-exporter-otlp-proto-grpc = "^1.26.0" +opentelemetry-instrumentation-logging = "^0.48b0" opentelemetry-instrumentation-requests = "^0.48b0" opentelemetry-instrumentation-sqlalchemy = "^0.48b0" opentelemetry-instrumentation-urllib3 = "^0.48b0" diff --git a/packages/traceloop-sdk/traceloop/sdk/__init__.py b/packages/traceloop-sdk/traceloop/sdk/__init__.py index d31f9bb1e..79a3a482a 100644 --- a/packages/traceloop-sdk/traceloop/sdk/__init__.py +++ b/packages/traceloop-sdk/traceloop/sdk/__init__.py @@ -7,18 +7,21 @@ from opentelemetry.sdk.trace import SpanProcessor from opentelemetry.sdk.trace.export import SpanExporter from opentelemetry.sdk.metrics.export import MetricExporter +from opentelemetry.sdk._logs.export import LogExporter from opentelemetry.sdk.resources import SERVICE_NAME from opentelemetry.propagators.textmap import TextMapPropagator from opentelemetry.util.re import parse_env_headers from traceloop.sdk.images.image_uploader import ImageUploader from traceloop.sdk.metrics.metrics import MetricsWrapper +from traceloop.sdk.logging.logging import LoggerWrapper from traceloop.sdk.telemetry import Telemetry from traceloop.sdk.instruments import Instruments from traceloop.sdk.config import ( is_content_tracing_enabled, is_tracing_enabled, is_metrics_enabled, + is_logging_enabled, ) from traceloop.sdk.fetcher import Fetcher from traceloop.sdk.tracing.tracing import ( @@ -48,6 +51,7 @@ def init( exporter: SpanExporter = None, metrics_exporter: MetricExporter = None, metrics_headers: Dict[str, str] = None, + logging_exporter: LogExporter = None, processor: SpanProcessor = None, propagator: TextMapPropagator = None, traceloop_sync_enabled: bool = False, @@ -135,23 +139,32 @@ def init( instruments=instruments, ) - if not metrics_exporter and exporter: - return - - metrics_endpoint = os.getenv("TRACELOOP_METRICS_ENDPOINT") or api_endpoint - metrics_headers = ( - os.getenv("TRACELOOP_METRICS_HEADERS") or metrics_headers or headers - ) - if not is_metrics_enabled() or not metrics_exporter and exporter: print(Fore.YELLOW + "Metrics are disabled" + Fore.RESET) - return + else: + metrics_endpoint = os.getenv("TRACELOOP_METRICS_ENDPOINT") or api_endpoint + metrics_headers = ( + os.getenv("TRACELOOP_METRICS_HEADERS") or metrics_headers or headers + ) + if metrics_exporter or processor: + print(Fore.GREEN + "Traceloop exporting metrics to a custom exporter") - MetricsWrapper.set_static_params( - resource_attributes, metrics_endpoint, metrics_headers - ) + MetricsWrapper.set_static_params( + resource_attributes, metrics_endpoint, metrics_headers + ) + Traceloop.__metrics_wrapper = MetricsWrapper(exporter=metrics_exporter) - Traceloop.__metrics_wrapper = MetricsWrapper(exporter=metrics_exporter) + if not is_logging_enabled() or not logging_exporter and exporter: + print(Fore.YELLOW + "Logging are disabled" + Fore.RESET) + else: + logging_endpoint = os.getenv("TRACELOOP_LOGGING_ENDPOINT") or api_endpoint + if logging_exporter or processor: + print(Fore.GREEN + "Traceloop exporting logs to a custom exporter") + + LoggerWrapper.set_static_params( + resource_attributes, logging_endpoint + ) + Traceloop.__logger_wrapper = LoggerWrapper(exporter=logging_exporter) def set_association_properties(properties: dict) -> None: set_association_properties(properties) diff --git a/packages/traceloop-sdk/traceloop/sdk/config/__init__.py b/packages/traceloop-sdk/traceloop/sdk/config/__init__.py index 8a681fb86..8b57568ff 100644 --- a/packages/traceloop-sdk/traceloop/sdk/config/__init__.py +++ b/packages/traceloop-sdk/traceloop/sdk/config/__init__.py @@ -11,3 +11,7 @@ def is_content_tracing_enabled() -> bool: def is_metrics_enabled() -> bool: return (os.getenv("TRACELOOP_METRICS_ENABLED") or "true").lower() == "true" + + +def is_logging_enabled() -> bool: + return (os.getenv("TRACELOOP_LOGGING_ENABLED") or "false").lower() == "true" diff --git a/packages/traceloop-sdk/traceloop/sdk/logging/__init__.py b/packages/traceloop-sdk/traceloop/sdk/logging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/packages/traceloop-sdk/traceloop/sdk/logging/logging.py b/packages/traceloop-sdk/traceloop/sdk/logging/logging.py new file mode 100644 index 000000000..e9f6d6bd5 --- /dev/null +++ b/packages/traceloop-sdk/traceloop/sdk/logging/logging.py @@ -0,0 +1,71 @@ +import logging + +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter as GRPCExporter, +) +from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter as HTTPExporter, +) +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk._logs.export import LogExporter, BatchLogRecordProcessor +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + +from opentelemetry.instrumentation.logging import LoggingInstrumentor + + +class LoggerWrapper(object): + resource_attributes: dict = {} + endpoint: str = None + __logging_exporter: LogExporter = None + __logging_provider: LoggerProvider = None + + def __new__(cls, exporter: LogExporter = None) -> "LoggerWrapper": + if not hasattr(cls, "instance"): + obj = cls.instance = super(LoggerWrapper, cls).__new__(cls) + if not LoggerWrapper.endpoint: + return obj + + obj.__logging_exporter = ( + exporter + if exporter + else init_logging_exporter(LoggerWrapper.endpoint) + ) + obj.__logging_provider = init_logging_provider( + obj.__logging_exporter, LoggerWrapper.resource_attributes + ) + LoggingInstrumentor().instrument(set_logging_format=True) + + return cls.instance + + @staticmethod + def set_static_params( + resource_attributes: dict, + endpoint: str, + ) -> None: + LoggerWrapper.resource_attributes = resource_attributes + LoggerWrapper.endpoint = endpoint + + +def init_logging_exporter(endpoint: str) -> LogExporter: + if "http" in endpoint.lower() or "https" in endpoint.lower(): + return HTTPExporter(endpoint=f"{endpoint}/v1/logs") + else: + return GRPCExporter(endpoint=endpoint) + + +def init_logging_provider( + exporter: LogExporter, resource_attributes: dict = None +) -> LoggerProvider: + resource = ( + Resource.create(resource_attributes) + if resource_attributes + else Resource.create() + ) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + + logging_handler = LoggingHandler(level=logging.NOTSET, logger_provider=logger_provider) + logging.basicConfig(level=logging.INFO, handlers=[logging_handler]) + + return logger_provider