From d84378141898b64e7917108cd2ff63ffe8f2c9ed Mon Sep 17 00:00:00 2001 From: Roy Williams Date: Tue, 14 Jan 2020 14:32:35 -0500 Subject: [PATCH 1/3] Use TBinaryProtocolAccelerated when serializing reports. We're currently seeing our services spend a large amount of time inside of the lightstep sdk serializing reports when they instrument themselves with tracing. This is in large part due to the fact that lightstep is using the pure python implemenation of Thrift. Fixes https://github.com/lightstep/lightstep-tracer-python/issues/89 --- lightstep/thrift_connection.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lightstep/thrift_connection.py b/lightstep/thrift_connection.py index fa540ce..54d24d7 100755 --- a/lightstep/thrift_connection.py +++ b/lightstep/thrift_connection.py @@ -4,6 +4,7 @@ import threading from thrift import Thrift from thrift.transport import THttpClient +from thrift.transport import TTransport from thrift.protocol import TBinaryProtocol from .crouton import ReportingService @@ -21,6 +22,7 @@ class _ThriftConnection(object): def __init__(self, collector_url): self._collector_url = collector_url self._lock = threading.Lock() + self._http_client = None self._transport = None self._client = None self.ready = False @@ -38,9 +40,10 @@ def open(self): """ self._lock.acquire() try: - self._transport = THttpClient.THttpClient(self._collector_url) + self._http_client = THttpClient.THttpClient(self._collector_url) + self._transport = TTransport.TBufferedTransport(self._http_client) self._transport.open() - protocol = TBinaryProtocol.TBinaryProtocol(self._transport) + protocol = TBinaryProtocol.TBinaryProtocolAccelerated(self._transport, fallback=True) self._client = ReportingService.Client(protocol) except Thrift.TException: self._open_exceptions_count += 1 @@ -61,7 +64,7 @@ def report(self, *args, **kwargs): try: if self._client: headers = {"Lightstep-Access-Token": args[0].access_token} - self._transport.setCustomHeaders(headers) + self._http_client.setCustomHeaders(headers) resp = self._client.Report(*args, **kwargs) self._report_consecutive_errors = 0 except Thrift.TException: From 0f3a3e5f151f1016e0e9302a01d30bc24e15473e Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 17 Jan 2020 14:47:24 -0800 Subject: [PATCH 2/3] Removing status code check When using THttpClient.THttpClient as the transport, an invalid status code could cause memory usage to spike. This behaviour is corrected via TTransport.TBufferedTransport, so this check is no longer needed. Signed-off-by: Alex Boten --- lightstep/crouton/ReportingService.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lightstep/crouton/ReportingService.py b/lightstep/crouton/ReportingService.py index 31bb404..fa3bbab 100644 --- a/lightstep/crouton/ReportingService.py +++ b/lightstep/crouton/ReportingService.py @@ -55,8 +55,6 @@ def send_Report(self, auth, request): def recv_Report(self): iprot = self._iprot - if iprot.trans.code != 200: - raise TApplicationException(TApplicationException, "Report failed: Received a server error") (fname, mtype, rseqid) = iprot.readMessageBegin() if mtype == TMessageType.EXCEPTION: x = TApplicationException() From 019bca4d800373132ccd5c1c4e114c0c0b797223 Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 17 Jan 2020 17:02:25 -0800 Subject: [PATCH 3/3] Remove check again, lost in merge Signed-off-by: Alex Boten --- lightstep/crouton/ReportingService.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lightstep/crouton/ReportingService.py b/lightstep/crouton/ReportingService.py index 9542b9a..42ad0bd 100644 --- a/lightstep/crouton/ReportingService.py +++ b/lightstep/crouton/ReportingService.py @@ -56,8 +56,6 @@ def send_Report(self, auth, request): def recv_Report(self): iprot = self._iprot - if iprot.trans.code != 200: - raise TApplicationException(TApplicationException, "Report failed: Received a server error") (fname, mtype, rseqid) = iprot.readMessageBegin() if mtype == TMessageType.EXCEPTION: x = TApplicationException()