From 181cc10b053b37fae793e2ad1254c53047deccba Mon Sep 17 00:00:00 2001 From: Ilya Volodarsky Date: Tue, 25 Jun 2013 12:42:38 -0700 Subject: [PATCH 001/323] fixing #14, fixing #12, adding change log and bumping version to 0.4.2 --- analytics/__init__.py | 135 +++++++++++++++++++++++++++++++++++++----- analytics/client.py | 18 ++++-- analytics/version.py | 2 +- history.md | 5 ++ setup.py | 2 +- 5 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 history.md diff --git a/analytics/__init__.py b/analytics/__init__.py index 0b99be95..df4306bc 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -9,13 +9,6 @@ from stats import Statistics stats = Statistics() -methods = ['identify', 'track', 'alias', 'flush', 'on_success', 'on_failure'] - -def uninitialized(*args, **kwargs): - print >>sys.stderr, 'Please call analytics.init(secret) before calling analytics methods.' - -for method in methods: - setattr(this_module, method, uninitialized) def init(secret, **kwargs): @@ -48,12 +41,126 @@ def init(secret, **kwargs): setattr(this_module, 'default_client', default_client) - def proxy(method): - def proxy_to_default_client(*args, **kwargs): - func = getattr(default_client, method) - return func(*args, **kwargs) - setattr(this_module, method, proxy_to_default_client) +def _get_default_client(): + default_client = None + if hasattr(this_module, 'default_client'): + default_client = getattr(this_module, 'default_client') + else: + sys.stderr.write('Please call analytics.init(secret) ' + + 'before calling analytics methods.\n') + return default_client + + +def identify(user_id=None, traits={}, context={}, timestamp=None): + """Identifying a user ties all of their actions to an id, and + associates user traits to that id. + + :param str user_id: the user's id after they are logged in. It's the + same id as which you would recognize a signed-in user in your system. + + : param dict traits: a dictionary with keys like subscriptionPlan or + age. You only need to record a trait once, no need to send it again. + Accepted value types are string, boolean, ints,, longs, and + datetime.datetime. + + : param dict context: An optional dictionary with additional + information thats related to the visit. Examples are userAgent, and IP + address of the visitor. + + : param datetime.datetime timestamp: If this event happened in the + past, the timestamp can be used to designate when the identification + happened. Careful with this one, if it just happened, leave it None. + If you do choose to provide a timestamp, make sure it has a timezone. + """ + default_client = _get_default_client() + if default_client: + default_client.identify(user_id=user_id, traits=traits, + context=context, timestamp=timestamp) + + +def track(user_id=None, event=None, properties={}, context={}, + timestamp=None): + """Whenever a user triggers an event, you'll want to track it. + + :param str user_id: the user's id after they are logged in. It's the + same id as which you would recognize a signed-in user in your system. + + :param str event: The event name you are tracking. It is recommended + that it is in human readable form. For example, "Bought T-Shirt" + or "Started an exercise" + + :param dict properties: A dictionary with items that describe the + event in more detail. This argument is optional, but highly recommended + - you'll find these properties extremely useful later. Accepted value + types are string, boolean, ints, doubles, longs, and datetime.datetime. + + :param dict context: An optional dictionary with additional information + thats related to the visit. Examples are userAgent, and IP address + of the visitor. + + :param datetime.datetime timestamp: If this event happened in the past, + the timestamp can be used to designate when the identification + happened. Careful with this one, if it just happened, leave it None. + If you do choose to provide a timestamp, make sure it has a timezone. + + """ + default_client = _get_default_client() + if default_client: + default_client.track(user_id=user_id, event=event, + properties=properties, context=context, + timestamp=timestamp) + - for method in methods: - proxy(method) +def alias(from_id, to_id, context={}, timestamp=None): + """Aliases an anonymous user into an identified user + + :param str from_id: the anonymous user's id before they are logged in + + :param str to_id: the identified user's id after they're logged in + + :param dict context: An optional dictionary with additional information + thats related to the visit. Examples are userAgent, and IP address + of the visitor. + + :param datetime.datetime timestamp: If this event happened in the past, + the timestamp can be used to designate when the identification + happened. Careful with this one, if it just happened, leave it None. + If you do choose to provide a timestamp, make sure it has a timezone. + """ + default_client = _get_default_client() + if default_client: + default_client.alias(from_id=from_id, to_id=to_id, context=context, + timestamp=timestamp) + + +def flush(async=None): + """ Forces a flush from the internal queue to the server + + :param bool async: True to block until all messages have been flushed + """ + default_client = _get_default_client() + if default_client: + default_client.flush(async=async) + + +def on_success(callback): + """ + Assign a callback to fire after a successful flush + + :param func callback: A callback that will be fired on a flush success + """ + default_client = _get_default_client() + if default_client: + default_client.on_success(callback) + + +def on_failure(callback): + """ + Assign a callback to fire after a failed flush + + :param func callback: A callback that will be fired on a failed flush + """ + default_client = _get_default_client() + if default_client: + default_client.on_failure(callback) diff --git a/analytics/client.py b/analytics/client.py index 2701f88a..57a786cd 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -203,13 +203,22 @@ def _clean(self, item): return self._coerce_unicode(item) def on_success(self, callback): + """ + Assign a callback to fire after a successful flush + + :param func callback: A callback that will be fired on a flush success + """ self.success_callbacks.append(callback) def on_failure(self, callback): + """ + Assign a callback to fire after a failed flush + + :param func callback: A callback that will be fired on a failed flush + """ self.failure_callbacks.append(callback) def identify(self, user_id=None, traits={}, context={}, timestamp=None): - # TODO: reduce the complexity (mccabe) """Identifying a user ties all of their actions to an id, and associates user traits to that id. @@ -264,7 +273,6 @@ def identify(self, user_id=None, traits={}, context={}, timestamp=None): def track(self, user_id=None, event=None, properties={}, context={}, timestamp=None): - # TODO: reduce the complexity (mccabe) """Whenever a user triggers an event, you'll want to track it. :param str user_id: the user's id after they are logged in. It's the @@ -327,7 +335,6 @@ def track(self, user_id=None, event=None, properties={}, context={}, self.stats.tracks += 1 def alias(self, from_id, to_id, context={}, timestamp=None): - # TODO: reduce the complexity (mccabe) """Aliases an anonymous user into an identified user :param str from_id: the anonymous user's id before they are logged in @@ -428,7 +435,10 @@ def _flush_thread_is_free(self): or not self.flushing_thread.is_alive() def flush(self, async=None): - """ Forces a flush from the queue to the server """ + """ Forces a flush from the internal queue to the server + + :param bool async: True to block until all messages have been flushed + """ flushing = False diff --git a/analytics/version.py b/analytics/version.py index 52f089c2..1376913e 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '0.4.1' +VERSION = '0.4.2' diff --git a/history.md b/history.md new file mode 100644 index 00000000..51f78499 --- /dev/null +++ b/history.md @@ -0,0 +1,5 @@ +0.4.2 / 2013-06-25 +================= +* Added history.d change log +* Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! +* Fixing #12, adding static public API to analytics.__init__ \ No newline at end of file diff --git a/setup.py b/setup.py index 25b455d2..c03dc9b0 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ except ImportError: from distutils.core import setup -# Don't import stripe module here, since deps may not be installed +# Don't import analytics-python module here, since deps may not be installed sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) from version import VERSION From f3d686400875c9f751432ba45a338961aa74267a Mon Sep 17 00:00:00 2001 From: Alex Louden Date: Mon, 22 Jul 2013 18:14:40 +0800 Subject: [PATCH 002/323] Made test script executable (shebang and 755) --- test.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 test.py diff --git a/test.py b/test.py old mode 100644 new mode 100755 index 57e837ef..8718d19b --- a/test.py +++ b/test.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python +# encoding: utf-8 + import unittest from datetime import datetime, timedelta From c274dda622755ecadfc0de5d8c66a43ee691cc2b Mon Sep 17 00:00:00 2001 From: Alex Louden Date: Mon, 22 Jul 2013 18:15:15 +0800 Subject: [PATCH 003/323] Added datetime serialisation class and test --- analytics/client.py | 4 ++-- analytics/utils.py | 9 ++++++++- test.py | 10 ++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 57a786cd..9e791573 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -10,7 +10,7 @@ from stats import Statistics from errors import ApiError -from utils import guess_timezone +from utils import guess_timezone, DatetimeSerializer import options @@ -66,7 +66,7 @@ def request(client, url, data): try: response = requests.post(url, - data=json.dumps(data), + data=json.dumps(data, cls=DatetimeSerializer), headers={'content-type': 'application/json'}, timeout=client.timeout) diff --git a/analytics/utils.py b/analytics/utils.py index a955b7b5..77723135 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -1,4 +1,4 @@ - +import json from datetime import datetime from dateutil.tz import tzlocal, tzutc @@ -24,3 +24,10 @@ def guess_timezone(dt): return dt.replace(tzinfo=tzutc()) return dt + +class DatetimeSerializer(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime): + return obj.isoformat() + + return json.JSONEncoder.default(self, obj) diff --git a/test.py b/test.py index 8718d19b..7d56d464 100755 --- a/test.py +++ b/test.py @@ -2,6 +2,7 @@ # encoding: utf-8 import unittest +import json from datetime import datetime, timedelta @@ -78,6 +79,15 @@ def test_clean(self): analytics.default_client._clean(combined) self.assertEqual(combined.keys(), pre_clean_keys) + + def test_datetime_serialization(self): + + data = { + 'created': datetime(2012, 3, 4, 5, 6, 7, 891011), + } + result = json.dumps(data, cls=analytics.utils.DatetimeSerializer) + + self.assertEqual(result, '{"created": "2012-03-04T05:06:07.891011"}') def test_async_basic_identify(self): # flush after every message From 0cee37dbe3608acdd80f9cb68ff4d8b3d04df257 Mon Sep 17 00:00:00 2001 From: Bitdeli Chef Date: Wed, 30 Oct 2013 01:53:15 +0000 Subject: [PATCH 004/323] Add a Bitdeli badge to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 86824191..ad6ee7c5 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,7 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/segmentio/analytics-python/trend.png)](https://bitdeli.com/free "Bitdeli Badge") + From 536c4ef5c0376129df6da430e099016fae713d10 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Wed, 13 Nov 2013 15:19:06 -0800 Subject: [PATCH 005/323] release: 0.4.3 c274dda Added datetime serialisation class and test --- analytics/version.py | 2 +- history.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/analytics/version.py b/analytics/version.py index 1376913e..c42de8a5 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '0.4.2' +VERSION = '0.4.3' diff --git a/history.md b/history.md index 51f78499..c71f86a1 100644 --- a/history.md +++ b/history.md @@ -1,5 +1,9 @@ +0.4.3 / 2013-11-13 +------------------ +* added datetime serialization fix (alexlouden) + 0.4.2 / 2013-06-25 -================= +------------------ * Added history.d change log * Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! * Fixing #12, adding static public API to analytics.__init__ \ No newline at end of file From 4e9833513ea83db482da3a64dbda9e221d83733a Mon Sep 17 00:00:00 2001 From: Ilya Volodarsky Date: Thu, 21 Nov 2013 15:45:57 -0800 Subject: [PATCH 006/323] fixing total_seconds python 2.6 compat issue and bump version --- analytics/utils.py | 6 +++++- analytics/version.py | 2 +- history.md | 8 ++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/analytics/utils.py b/analytics/utils.py index 77723135..d136a7d4 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -7,6 +7,10 @@ def is_naive(dt): """ Determines if a given datetime.datetime is naive. """ return dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None +def total_seconds(delta): + """ Determines total seconds with python < 2.7 compat """ + # http://stackoverflow.com/questions/3694835/python-2-6-5-divide-timedelta-with-timedelta + return (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6 def guess_timezone(dt): """ Attempts to convert a naive datetime to an aware datetime """ @@ -15,7 +19,7 @@ def guess_timezone(dt): # case, and then defaults to utc delta = datetime.now() - dt - if delta.total_seconds() < 5: + if total_seconds(delta) < 5: # this was created using datetime.datetime.now() # so we are in the local timezone return dt.replace(tzinfo=tzlocal()) diff --git a/analytics/version.py b/analytics/version.py index c42de8a5..384a3e1e 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '0.4.3' +VERSION = '0.4.4' diff --git a/history.md b/history.md index c71f86a1..98e24d84 100644 --- a/history.md +++ b/history.md @@ -1,8 +1,12 @@ -0.4.3 / 2013-11-13 +0.4.4 / November 21, 2013 +------------------ +* add < python 2.7 compatibility by removing `delta.total_seconds` + +0.4.3 / November 13, 2013 ------------------ * added datetime serialization fix (alexlouden) -0.4.2 / 2013-06-25 +0.4.2 / June 26, 2013 ------------------ * Added history.d change log * Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! From c262b1e5ab183638965c9282408728180d1836f1 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Thu, 4 Sep 2014 17:40:33 -0700 Subject: [PATCH 007/323] updating to spec 1.0 This commit includes a number of significant changes: - updating the library to use the new spec - moving to analytics.write_key API - moving to a consumer in a separate thread - adding request retries - python 3 support - making analytics.flush() synchronous - adding full travis tests --- .travis.yml | 13 + Makefile | 5 + README.md | 5 +- analytics/__init__.py | 186 ++--------- analytics/client.py | 647 ++++++++++--------------------------- analytics/consumer.py | 90 ++++++ analytics/errors.py | 13 - analytics/options.py | 9 - analytics/request.py | 49 +++ analytics/stats.py | 22 -- analytics/test/__init__.py | 12 + analytics/test/client.py | 231 +++++++++++++ analytics/test/consumer.py | 53 +++ analytics/test/module.py | 45 +++ analytics/test/request.py | 31 ++ analytics/test/utils.py | 50 +++ analytics/utils.py | 53 ++- analytics/version.py | 2 +- requirements.txt | 2 + setup.py | 7 +- test.py | 308 ------------------ 21 files changed, 841 insertions(+), 992 deletions(-) create mode 100644 .travis.yml create mode 100644 Makefile create mode 100644 analytics/consumer.py delete mode 100644 analytics/errors.py delete mode 100644 analytics/options.py create mode 100644 analytics/request.py delete mode 100644 analytics/stats.py create mode 100644 analytics/test/__init__.py create mode 100644 analytics/test/client.py create mode 100644 analytics/test/consumer.py create mode 100644 analytics/test/module.py create mode 100644 analytics/test/request.py create mode 100644 analytics/test/utils.py create mode 100644 requirements.txt delete mode 100755 test.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..b65b688c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" +install: + - "pip install ." + - "pip install -r requirements.txt" +script: make test +matrix: + fast_finish: true diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..28e9c14a --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ + +test: + python setup.py test + +.PHONY: test \ No newline at end of file diff --git a/README.md b/README.md index ad6ee7c5..8ef4c87d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ analytics-python ============== +[![Build Status](https://travis-ci.org/segmentio/analytics-python.svg?branch=master)](https://travis-ci.org/segmentio/analytics-python) + analytics-python is a python client for [Segment.io](https://segment.io) ## Documentation @@ -32,6 +34,3 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/segmentio/analytics-python/trend.png)](https://bitdeli.com/free "Bitdeli Badge") - diff --git a/analytics/__init__.py b/analytics/__init__.py index df4306bc..2df084bc 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -1,166 +1,50 @@ -import version +from analytics.version import VERSION +from analytics.client import Client -VERSION = version.VERSION __version__ = VERSION -import sys -this_module = sys.modules[__name__] +"""Settings.""" +write_key = None +on_error = None +debug = False -from stats import Statistics -stats = Statistics() +default_client = None -def init(secret, **kwargs): - """Create a default instance of a analytics-python client +def track(*args, **kwargs): + """Send a track call.""" + _proxy('track', *args, **kwargs) - :param str secret: The Segment.io API Secret +def identify(*args, **kwargs): + """Send a identify call.""" + _proxy('identify', *args, **kwargs) - Kwargs: +def group(*args, **kwargs): + """Send a group call.""" + _proxy('group', *args, **kwargs) - :param logging.LOG_LEVEL log_level: The logging log level for the client - talks to. Use log_level=logging.DEBUG to troubleshoot - : param bool log: False to turn off logging completely, True by default - : param int flush_at: Specicies after how many messages the client will flush - to the server. Use flush_at=1 to disable batching - : param datetime.timedelta flush_after: Specifies after how much time - of no flushing that the server will flush. Used in conjunction with - the flush_at size policy - : param bool async: True to have the client flush to the server on another - thread, therefore not blocking code (this is the default). False to - enable blocking and making the request on the calling thread. +def alias(*args, **kwargs): + """Send a alias call.""" + _proxy('alias', *args, **kwargs) - """ - from client import Client +def page(*args, **kwargs): + """Send a page call.""" + _proxy('page', *args, **kwargs) - # if we have already initialized, no-op - if hasattr(this_module, 'default_client'): - return +def screen(*args, **kwargs): + """Send a screen call.""" + _proxy('screen', *args, **kwargs) - default_client = Client(secret=secret, stats=stats, **kwargs) +def flush(): + """Tell the client to flush.""" + _proxy('flush') - setattr(this_module, 'default_client', default_client) +def _proxy(method, *args, **kwargs): + """Create an analytics client if one doesn't exist and send to it.""" + global default_client + if not default_client: + default_client = Client(write_key, debug=debug, on_error=on_error) - -def _get_default_client(): - default_client = None - if hasattr(this_module, 'default_client'): - default_client = getattr(this_module, 'default_client') - else: - sys.stderr.write('Please call analytics.init(secret) ' + - 'before calling analytics methods.\n') - return default_client - - -def identify(user_id=None, traits={}, context={}, timestamp=None): - """Identifying a user ties all of their actions to an id, and - associates user traits to that id. - - :param str user_id: the user's id after they are logged in. It's the - same id as which you would recognize a signed-in user in your system. - - : param dict traits: a dictionary with keys like subscriptionPlan or - age. You only need to record a trait once, no need to send it again. - Accepted value types are string, boolean, ints,, longs, and - datetime.datetime. - - : param dict context: An optional dictionary with additional - information thats related to the visit. Examples are userAgent, and IP - address of the visitor. - - : param datetime.datetime timestamp: If this event happened in the - past, the timestamp can be used to designate when the identification - happened. Careful with this one, if it just happened, leave it None. - If you do choose to provide a timestamp, make sure it has a timezone. - """ - default_client = _get_default_client() - if default_client: - default_client.identify(user_id=user_id, traits=traits, - context=context, timestamp=timestamp) - - -def track(user_id=None, event=None, properties={}, context={}, - timestamp=None): - """Whenever a user triggers an event, you'll want to track it. - - :param str user_id: the user's id after they are logged in. It's the - same id as which you would recognize a signed-in user in your system. - - :param str event: The event name you are tracking. It is recommended - that it is in human readable form. For example, "Bought T-Shirt" - or "Started an exercise" - - :param dict properties: A dictionary with items that describe the - event in more detail. This argument is optional, but highly recommended - - you'll find these properties extremely useful later. Accepted value - types are string, boolean, ints, doubles, longs, and datetime.datetime. - - :param dict context: An optional dictionary with additional information - thats related to the visit. Examples are userAgent, and IP address - of the visitor. - - :param datetime.datetime timestamp: If this event happened in the past, - the timestamp can be used to designate when the identification - happened. Careful with this one, if it just happened, leave it None. - If you do choose to provide a timestamp, make sure it has a timezone. - - """ - default_client = _get_default_client() - if default_client: - default_client.track(user_id=user_id, event=event, - properties=properties, context=context, - timestamp=timestamp) - - -def alias(from_id, to_id, context={}, timestamp=None): - """Aliases an anonymous user into an identified user - - :param str from_id: the anonymous user's id before they are logged in - - :param str to_id: the identified user's id after they're logged in - - :param dict context: An optional dictionary with additional information - thats related to the visit. Examples are userAgent, and IP address - of the visitor. - - :param datetime.datetime timestamp: If this event happened in the past, - the timestamp can be used to designate when the identification - happened. Careful with this one, if it just happened, leave it None. - If you do choose to provide a timestamp, make sure it has a timezone. - """ - default_client = _get_default_client() - if default_client: - default_client.alias(from_id=from_id, to_id=to_id, context=context, - timestamp=timestamp) - - -def flush(async=None): - """ Forces a flush from the internal queue to the server - - :param bool async: True to block until all messages have been flushed - """ - default_client = _get_default_client() - if default_client: - default_client.flush(async=async) - - -def on_success(callback): - """ - Assign a callback to fire after a successful flush - - :param func callback: A callback that will be fired on a flush success - """ - default_client = _get_default_client() - if default_client: - default_client.on_success(callback) - - -def on_failure(callback): - """ - Assign a callback to fire after a failed flush - - :param func callback: A callback that will be fired on a failed flush - """ - default_client = _get_default_client() - if default_client: - default_client.on_failure(callback) + fn = getattr(default_client, method) + fn(*args, **kwargs) diff --git a/analytics/client.py b/analytics/client.py index 9e791573..de0e1220 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -1,503 +1,208 @@ -import collections -from datetime import datetime, timedelta -import json +from datetime import datetime +from uuid import uuid4 import logging import numbers -import threading +import six from dateutil.tz import tzutc -import requests -from stats import Statistics -from errors import ApiError -from utils import guess_timezone, DatetimeSerializer +from analytics.consumer import Consumer +from analytics.utils import guess_timezone, clean +from analytics.version import VERSION -import options +try: + import queue +except: + import Queue as queue -logging_enabled = True -logger = logging.getLogger('analytics') - - -def log(level, *args, **kwargs): - if logging_enabled: - method = getattr(logger, level) - method(*args, **kwargs) - - -def package_exception(client, data, e): - log('warn', 'Segment.io request error', exc_info=True) - client._on_failed_flush(data, e) - - -def package_response(client, data, response): - # TODO: reduce the complexity (mccabe) - if response.status_code == 200: - client._on_successful_flush(data, response) - elif response.status_code == 400: - content = response.text - try: - body = json.loads(content) - - code = 'bad_request' - message = 'Bad request' - - if 'error' in body: - error = body.error - - if 'code' in error: - code = error['code'] - - if 'message' in error: - message = error['message'] - - client._on_failed_flush(data, ApiError(code, message)) - - except Exception: - client._on_failed_flush(data, ApiError('Bad Request', content)) - else: - client._on_failed_flush(data, - ApiError(response.status_code, response.text)) - - -def request(client, url, data): - - log('debug', 'Sending request to Segment.io ...') - try: - - response = requests.post(url, - data=json.dumps(data, cls=DatetimeSerializer), - headers={'content-type': 'application/json'}, - timeout=client.timeout) - - log('debug', 'Finished Segment.io request.') - - package_response(client, data, response) - - return response.status_code == 200 - - except requests.ConnectionError as e: - package_exception(client, data, e) - except requests.Timeout as e: - package_exception(client, data, e) - - return False - - -class FlushThread(threading.Thread): - - def __init__(self, client): - threading.Thread.__init__(self) - self.client = client - - def run(self): - log('debug', 'Flushing thread running ...') - - self.client._sync_flush() - - log('debug', 'Flushing thread done.') +ID_TYPES = (str, six.text_type, int, six.integer_types, float, numbers.Number) class Client(object): - """The Client class is a batching asynchronous python wrapper over the - Segment.io API. - - """ - - def __init__(self, secret=None, log_level=logging.INFO, log=True, - flush_at=20, flush_after=timedelta(0, 10), - async=True, max_queue_size=10000, stats=Statistics(), - timeout=10, send=True): - """Create a new instance of a analytics-python Client - - :param str secret: The Segment.io API secret - :param logging.LOG_LEVEL log_level: The logging log level for the - client talks to. Use log_level=logging.DEBUG to troubleshoot - : param bool log: False to turn off logging completely, True by default - : param int flush_at: Specicies after how many messages the client will - flush to the server. Use flush_at=1 to disable batching - : param datetime.timedelta flush_after: Specifies after how much time - of no flushing that the server will flush. Used in conjunction with - the flush_at size policy - : param bool async: True to have the client flush to the server on - another thread, therefore not blocking code (this is the default). - False to enable blocking and making the request on the calling thread. - : param float timeout: Number of seconds before timing out request to - Segment.io - : param bool send: True to send requests, False to not send. False to - turn analytics off (for testing). - """ - - self.secret = secret - - self.queue = collections.deque() - self.last_flushed = None - - if not log: - # TODO: logging_enabled is assigned, but not used - logging_enabled = False - # effectively disables the logger - logger.setLevel(logging.CRITICAL) - else: - logger.setLevel(log_level) - - self.async = async - - self.max_queue_size = max_queue_size - self.max_flush_size = 50 - - self.flush_at = flush_at - self.flush_after = flush_after - - self.timeout = timeout - - self.stats = stats - - self.flush_lock = threading.Lock() - self.flushing_thread = None - + """Create a new Segment client.""" + log = logging.getLogger('segment') + + def __init__(self, write_key=None, debug=False, max_queue_size=10000, + send=True, on_error=None): + require('write_key', write_key, str) + + self.queue = queue.Queue(max_queue_size) + self.consumer = Consumer(self.queue, write_key, on_error=on_error) + self.write_key = write_key + self.on_error = on_error + self.debug = debug self.send = send - self.success_callbacks = [] - self.failure_callbacks = [] - - def set_log_level(self, level): - """Sets the log level for analytics-python - - :param logging.LOG_LEVEL level: The level at which analytics-python log - should talk at - """ - logger.setLevel(level) - - def _check_for_secret(self): - if not self.secret: - raise Exception('Please set analytics.secret before calling ' + - 'identify or track.') - - def _coerce_unicode(self, cmplx): - return unicode(cmplx) - - def _clean_list(self, l): - return [self._clean(item) for item in l] - - def _clean_dict(self, d): - data = {} - for k, v in d.iteritems(): - try: - data[k] = self._clean(v) - except TypeError: - log('warn', 'Dictionary values must be serializeable to ' + - 'JSON "%s" value %s of type %s is unsupported.' - % (k, v, type(v))) - return data - - def _clean(self, item): - if isinstance(item, (str, unicode, int, long, float, bool, - numbers.Number, datetime)): - return item - elif isinstance(item, (set, list, tuple)): - return self._clean_list(item) - elif isinstance(item, dict): - return self._clean_dict(item) - else: - return self._coerce_unicode(item) - - def on_success(self, callback): - """ - Assign a callback to fire after a successful flush - - :param func callback: A callback that will be fired on a flush success - """ - self.success_callbacks.append(callback) - - def on_failure(self, callback): - """ - Assign a callback to fire after a failed flush - - :param func callback: A callback that will be fired on a failed flush - """ - self.failure_callbacks.append(callback) - - def identify(self, user_id=None, traits={}, context={}, timestamp=None): - """Identifying a user ties all of their actions to an id, and - associates user traits to that id. - - :param str user_id: the user's id after they are logged in. It's the - same id as which you would recognize a signed-in user in your system. - - : param dict traits: a dictionary with keys like subscriptionPlan or - age. You only need to record a trait once, no need to send it again. - Accepted value types are string, boolean, ints,, longs, and - datetime.datetime. - - : param dict context: An optional dictionary with additional - information thats related to the visit. Examples are userAgent, and IP - address of the visitor. - - : param datetime.datetime timestamp: If this event happened in the - past, the timestamp can be used to designate when the identification - happened. Careful with this one, if it just happened, leave it None. - If you do choose to provide a timestamp, make sure it has a timezone. - """ - - self._check_for_secret() - - if not user_id: - raise Exception('Must supply a user_id.') - - if traits is not None and not isinstance(traits, dict): - raise Exception('Traits must be a dictionary.') - - if context is not None and not isinstance(context, dict): - raise Exception('Context must be a dictionary.') - - if timestamp is None: - timestamp = datetime.utcnow().replace(tzinfo=tzutc()) - elif not isinstance(timestamp, datetime): - raise Exception('Timestamp must be a datetime object.') - else: - timestamp = guess_timezone(timestamp) + if debug: + self.log.setLevel('debug') - cleaned_traits = self._clean(traits) + self.consumer.start() - action = {'userId': user_id, - 'traits': cleaned_traits, - 'context': context, - 'timestamp': timestamp.isoformat(), - 'action': 'identify'} + def identify(self, user_id=None, traits={}, context={}, timestamp=None, + anonymous_id=None, integrations={}): + require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) + require('traits', traits, dict) - context['library'] = 'analytics-python' + msg = { + 'integrations': integrations, + 'anonymousId': anonymous_id, + 'timestamp': timestamp, + 'context': context, + 'type': 'identify', + 'userId': user_id, + 'traits': traits + } - if self._enqueue(action): - self.stats.identifies += 1 + return self._enqueue(msg) def track(self, user_id=None, event=None, properties={}, context={}, - timestamp=None): - """Whenever a user triggers an event, you'll want to track it. - - :param str user_id: the user's id after they are logged in. It's the - same id as which you would recognize a signed-in user in your system. - - :param str event: The event name you are tracking. It is recommended - that it is in human readable form. For example, "Bought T-Shirt" - or "Started an exercise" - - :param dict properties: A dictionary with items that describe the - event in more detail. This argument is optional, but highly recommended - - you'll find these properties extremely useful later. Accepted value - types are string, boolean, ints, doubles, longs, and datetime.datetime. - - :param dict context: An optional dictionary with additional information - thats related to the visit. Examples are userAgent, and IP address - of the visitor. - - :param datetime.datetime timestamp: If this event happened in the past, - the timestamp can be used to designate when the identification - happened. Careful with this one, if it just happened, leave it None. - If you do choose to provide a timestamp, make sure it has a timezone. - - """ - - self._check_for_secret() - - if not user_id: - raise Exception('Must supply a user_id.') - - if not event: - raise Exception('Event is a required argument as a non-empty ' + - 'string.') - - if properties is not None and not isinstance(properties, dict): - raise Exception('Context must be a dictionary.') - - if context is not None and not isinstance(context, dict): - raise Exception('Context must be a dictionary.') - + timestamp=None, anonymous_id=None, integrations={}): + + require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) + require('properties', properties, dict) + require('event', event, str) + + msg = { + 'integrations': integrations, + 'anonymousId': anonymous_id, + 'properties': properties, + 'timestamp': timestamp, + 'context': context, + 'userId': user_id, + 'type': 'track', + 'event': event + } + + return self._enqueue(msg) + + def alias(self, previous_id=None, user_id=None, context={}, timestamp=None, + integrations={}): + require('previous_id', previous_id, ID_TYPES) + require('user_id', user_id, ID_TYPES) + + msg = { + 'integrations': integrations, + 'previousId': previous_id, + 'timestamp': timestamp, + 'context': context, + 'userId': user_id, + 'type': 'alias' + } + + return self._enqueue(msg) + + def group(self, user_id=None, group_id=None, traits={}, context={}, + timestamp=None, anonymous_id=None, integrations={}): + require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) + require('group_id', group_id, ID_TYPES) + require('traits', traits, dict) + + msg = { + 'integrations': integrations, + 'anonymousId': anonymous_id, + 'timestamp': timestamp, + 'groupId': group_id, + 'context': context, + 'userId': user_id, + 'traits': traits, + 'type': 'group' + } + + return self._enqueue(msg) + + def page(self, user_id=None, category=None, name=None, properties={}, + context={}, timestamp=None, anonymous_id=None, integrations={}): + require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) + require('properties', properties, dict) + + if name: + require('name', name, str) + if category: + require('category', category, str) + + msg = { + 'integrations': integrations, + 'anonymousId': anonymous_id, + 'properties': properties, + 'timestamp': timestamp, + 'category': category, + 'context': context, + 'userId': user_id, + 'type': 'page', + 'name': name, + } + + return self._enqueue(msg) + + def screen(self, user_id=None, category=None, name=None, properties={}, + context={}, timestamp=None, anonymous_id=None, integrations={}): + require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) + require('properties', properties, dict) + + if name: + require('name', name, str) + if category: + require('category', category, str) + + msg = { + 'integrations': integrations, + 'anonymousId': anonymous_id, + 'properties': properties, + 'timestamp': timestamp, + 'category': category, + 'context': context, + 'userId': user_id, + 'type': 'screen', + 'name': name, + } + + return self._enqueue(msg) + + def _enqueue(self, msg): + """Push a new `msg` onto the queue, return `(success, msg)`""" + timestamp = msg['timestamp'] if timestamp is None: timestamp = datetime.utcnow().replace(tzinfo=tzutc()) - elif not isinstance(timestamp, datetime): - raise Exception('Timestamp must be a datetime.datetime object.') - else: - timestamp = guess_timezone(timestamp) - - cleaned_properties = self._clean(properties) - - action = {'userId': user_id, - 'event': event, - 'context': context, - 'properties': cleaned_properties, - 'timestamp': timestamp.isoformat(), - 'action': 'track'} - - context['library'] = 'analytics-python' - - if self._enqueue(action): - self.stats.tracks += 1 - def alias(self, from_id, to_id, context={}, timestamp=None): - """Aliases an anonymous user into an identified user + require('integrations', msg['integrations'], dict) + require('timestamp', timestamp, datetime) + require('context', msg['context'], dict) + require('type', msg['type'], str) - :param str from_id: the anonymous user's id before they are logged in + # add common + timestamp = guess_timezone(timestamp) + msg['timestamp'] = timestamp.isoformat() + msg['messageId'] = str(uuid4()) + msg['context']['library'] = { + 'name': 'analytics-python', + 'version': VERSION + } - :param str to_id: the identified user's id after they're logged in - - :param dict context: An optional dictionary with additional information - thats related to the visit. Examples are userAgent, and IP address - of the visitor. - - :param datetime.datetime timestamp: If this event happened in the past, - the timestamp can be used to designate when the identification - happened. Careful with this one, if it just happened, leave it None. - If you do choose to provide a timestamp, make sure it has a timezone. - """ - - self._check_for_secret() - - if not from_id: - raise Exception('Must supply a from_id.') - - if not to_id: - raise Exception('Must supply a to_id.') - - if context is not None and not isinstance(context, dict): - raise Exception('Context must be a dictionary.') - - if timestamp is None: - timestamp = datetime.utcnow().replace(tzinfo=tzutc()) - elif not isinstance(timestamp, datetime): - raise Exception('Timestamp must be a datetime.datetime object.') - else: - timestamp = guess_timezone(timestamp) - - action = {'from': from_id, - 'to': to_id, - 'context': context, - 'timestamp': timestamp.isoformat(), - 'action': 'alias'} - - context['library'] = 'analytics-python' - - if self._enqueue(action): - self.stats.aliases += 1 - - def _should_flush(self): - """ Determine whether we should sync """ - - full = len(self.queue) >= self.flush_at - stale = self.last_flushed is None - - if not stale: - stale = datetime.now() - self.last_flushed > self.flush_after - - return full or stale - - def _enqueue(self, action): + clean(msg) # if we've disabled sending, just return False if not self.send: - return False - - submitted = False - - if len(self.queue) < self.max_queue_size: - self.queue.append(action) - - self.stats.submitted += 1 - - submitted = True - - log('debug', 'Enqueued ' + action['action'] + '.') - - else: - log('warn', 'analytics-python queue is full') - - if self._should_flush(): - self.flush() - - return submitted - - def _on_successful_flush(self, data, response): - if 'batch' in data: - for item in data['batch']: - self.stats.successful += 1 - for callback in self.success_callbacks: - callback(data, response) - - def _on_failed_flush(self, data, error): - if 'batch' in data: - for item in data['batch']: - self.stats.failed += 1 - for callback in self.failure_callbacks: - callback(data, error) - - def _flush_thread_is_free(self): - return self.flushing_thread is None \ - or not self.flushing_thread.is_alive() - - def flush(self, async=None): - """ Forces a flush from the internal queue to the server - - :param bool async: True to block until all messages have been flushed - """ - - flushing = False - - # if the async arg is provided, it overrides the client's settings - if async is None: - async = self.async - - if async: - # We should asynchronously flush on another thread - with self.flush_lock: - - if self._flush_thread_is_free(): - - log('debug', 'Initiating asynchronous flush ..') - - self.flushing_thread = FlushThread(self) - self.flushing_thread.start() - - flushing = True - - else: - log('debug', 'The flushing thread is still active.') - else: - - # Flushes on this thread - log('debug', 'Initiating synchronous flush ..') - self._sync_flush() - flushing = True - - if flushing: - self.last_flushed = datetime.now() - self.stats.flushes += 1 - - return flushing - - def _sync_flush(self): - - log('debug', 'Starting flush ..') - - successful = 0 - failed = 0 - - url = options.host + options.endpoints['batch'] - - while len(self.queue) > 0: + return False, msg - batch = [] - for i in range(self.max_flush_size): - if len(self.queue) == 0: - break + if self.queue.full(): + self.log.warn('analytics-python queue is full') + return False, msg - batch.append(self.queue.pop()) + self.queue.put(msg) + self.log.debug('enqueued ' + msg['type'] + '.') + return True, msg - payload = {'batch': batch, 'secret': self.secret} + def flush(self): + """Forces a flush from the internal queue to the server""" + queue = self.queue + size = queue.qsize() + queue.join() + self.log.debug('successfully flushed {0} items.'.format(size)) - if request(self, url, payload): - successful += len(batch) - else: - failed += len(batch) - log('debug', 'Successfully flushed {0} items [{1} failed].'. - format(str(successful), str(failed))) +def require(name, field, data_type): + """Require that the named `field` has the right `data_type`""" + if not isinstance(field, data_type): + msg = '{0} must have {1}, got: {2}'.format(name, data_type, field) + raise AssertionError(msg) diff --git a/analytics/consumer.py b/analytics/consumer.py new file mode 100644 index 00000000..ca1761f4 --- /dev/null +++ b/analytics/consumer.py @@ -0,0 +1,90 @@ +from threading import Thread +import logging + +from analytics.version import VERSION +from analytics.request import post + + +class Consumer(Thread): + """Consumes the messages from the client's queue.""" + log = logging.getLogger('segment') + + def __init__(self, queue, write_key, upload_size=100, on_error=None): + """Create a consumer thread.""" + Thread.__init__(self) + self.daemon = True # set as a daemon so the program can exit + self.upload_size = upload_size + self.write_key = write_key + self.on_error = on_error + self.queue = queue + self.retries = 3 + + def run(self): + """Runs the consumer.""" + self.log.debug('consumer is running...') + + self.running = True + while self.running: + self.upload() + + self.log.debug('consumer exited.') + + def pause(self): + """Pause the consumer.""" + self.running = False + + def upload(self): + """Upload the next batch of items, return whether successful.""" + success = False + batch = self.next() + if len(batch) == 0: + return False + + try: + self.request(batch) + success = True + except Exception as e: + self.log.error('error uploading: %s', e) + success = False + if self.on_error: + self.on_error(e, batch) + finally: + # cleanup + for item in batch: + self.queue.task_done() + + return success + + def next(self): + """Return the next batch of items to upload.""" + queue = self.queue + items = [] + item = self.next_item() + if item is None: + return items + + items.append(item) + while len(items) < self.upload_size and not queue.empty(): + item = self.next_item() + if item: + items.append(item) + + return items + + def next_item(self): + """Get a single item from the queue.""" + queue = self.queue + try: + item = queue.get(block=True, timeout=5) + return item + except Exception: + return None + + def request(self, batch, attempt=0): + """Attempt to upload the batch and retry before raising an error """ + try: + post(self.write_key, batch=batch) + except: + if attempt > self.retries: + raise + self.request(batch, attempt+1) diff --git a/analytics/errors.py b/analytics/errors.py deleted file mode 100644 index ca645edb..00000000 --- a/analytics/errors.py +++ /dev/null @@ -1,13 +0,0 @@ - - -class ApiError(Exception): - - def __init__(self, code, message): - self.code = code - self.message = message - - def __repr__(self): - return self.__str__() - - def __str__(self): - return repr(self.message) diff --git a/analytics/options.py b/analytics/options.py deleted file mode 100644 index 326b38ad..00000000 --- a/analytics/options.py +++ /dev/null @@ -1,9 +0,0 @@ - -host = 'https://api.segment.io' - -endpoints = { - 'track': '/v1/track', - 'identify': '/v1/identify', - 'alias': '/v1/alias', - 'batch': '/v1/import' -} diff --git a/analytics/request.py b/analytics/request.py new file mode 100644 index 00000000..064d81e4 --- /dev/null +++ b/analytics/request.py @@ -0,0 +1,49 @@ +from datetime import datetime +import logging +import json + +from requests.auth import HTTPBasicAuth +import requests + + +def post(write_key, **kwargs): + """Post the `kwargs` to the API""" + log = logging.getLogger('segment') + body = kwargs + url = 'https://api.segment.io/v1/batch' + auth = HTTPBasicAuth(write_key, '') + data = json.dumps(body, cls=DatetimeSerializer) + headers = { 'content-type': 'application/json' } + log.debug('making request: %s', data) + res = requests.post(url, data=data, auth=auth, headers=headers, timeout=15) + + if res.status_code == 200: + log.debug('data uploaded successfully') + return res + + try: + payload = res.json() + log.debug('received response: %s', payload) + raise APIError(res.status_code, payload['code'], payload['message']) + except ValueError: + raise APIError(res.status_code, 'unknown', res.text) + + +class APIError(Exception): + + def __init__(self, status, code, message): + self.message = message + self.status = status + self.code = code + + def __str__(self): + msg = "[Segment] {0}: {1} ({2})" + return msg.format(self.code, self.message, self.status) + + +class DatetimeSerializer(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime): + return obj.isoformat() + + return json.JSONEncoder.default(self, obj) \ No newline at end of file diff --git a/analytics/stats.py b/analytics/stats.py deleted file mode 100644 index 096e5310..00000000 --- a/analytics/stats.py +++ /dev/null @@ -1,22 +0,0 @@ - - -class Statistics(object): - - def __init__(self): - # The number of submitted identifies/tracks - self.submitted = 0 - - # The number of identifies submitted - self.identifies = 0 - # The number of tracks submitted - self.tracks = 0 - # The number of aliases - self.aliases = 0 - - # The number of actions to be successful - self.successful = 0 - # The number of actions to fail - self.failed = 0 - - # The number of flushes to happen - self.flushes = 0 diff --git a/analytics/test/__init__.py b/analytics/test/__init__.py new file mode 100644 index 00000000..877653af --- /dev/null +++ b/analytics/test/__init__.py @@ -0,0 +1,12 @@ +import unittest +import pkgutil +import logging +import sys + +def all_names(): + for _, modname, _ in pkgutil.iter_modules(__path__): + yield 'analytics.test.' + modname + +def all(): + logging.basicConfig(stream=sys.stderr) + return unittest.defaultTestLoader.loadTestsFromNames(all_names()) diff --git a/analytics/test/client.py b/analytics/test/client.py new file mode 100644 index 00000000..96ea3de5 --- /dev/null +++ b/analytics/test/client.py @@ -0,0 +1,231 @@ +from datetime import datetime +import unittest +import time + +from analytics.version import VERSION +from analytics.client import Client + + +class TestClient(unittest.TestCase): + + def fail(self, e, batch): + """Mark the failure handler""" + self.failed = True + + def setUp(self): + self.failed = False + self.client = Client('testsecret', on_error=self.fail) + + def test_requires_write_key(self): + self.assertRaises(AssertionError, Client) + + def test_empty_flush(self): + self.client.flush() + + def test_basic_track(self): + client = self.client + success, msg = client.track('userId', 'python test event') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['event'], 'python test event') + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['properties'], {}) + self.assertEqual(msg['type'], 'track') + + def test_advanced_track(self): + client = self.client + success, msg = client.track( + 'userId', 'python test event', { 'property': 'value' }, + { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', + { 'Amplitude': True }) + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['properties'], { 'property': 'value' }) + self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['event'], 'python test event') + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'track') + + def test_basic_identify(self): + client = self.client + success, msg = client.identify('userId', { 'trait': 'value' }) + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['traits'], { 'trait': 'value' }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'identify') + + def test_advanced_identify(self): + client = self.client + success, msg = client.identify( + 'userId', { 'trait': 'value' }, { 'ip': '192.168.0.1' }, + datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }) + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['traits'], { 'trait': 'value' }) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'identify') + + def test_basic_group(self): + client = self.client + success, msg = client.group('userId', 'groupId') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['groupId'], 'groupId') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'group') + + def test_advanced_group(self): + client = self.client + success, msg = client.group( + 'userId', 'groupId', { 'trait': 'value' }, { 'ip': '192.168.0.1' }, + datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }) + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['traits'], { 'trait': 'value' }) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'group') + + def test_basic_alias(self): + client = self.client + success, msg = client.alias('previousId', 'userId') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + self.assertEqual(msg['previousId'], 'previousId') + self.assertEqual(msg['userId'], 'userId') + + def test_basic_page(self): + client = self.client + success, msg = client.page('userId', name='name') + self.assertFalse(self.failed) + client.flush() + self.assertTrue(success) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'page') + self.assertEqual(msg['name'], 'name') + + def test_advanced_page(self): + client = self.client + success, msg = client.page( + 'userId', 'category', 'name', { 'property': 'value' }, + { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', + { 'Amplitude': True }) + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['properties'], { 'property': 'value' }) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertEqual(msg['category'], 'category') + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'page') + self.assertEqual(msg['name'], 'name') + + def test_basic_screen(self): + client = self.client + success, msg = client.screen('userId', name='name') + client.flush() + self.assertTrue(success) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'screen') + self.assertEqual(msg['name'], 'name') + + def test_advanced_screen(self): + client = self.client + success, msg = client.screen( + 'userId', 'category', 'name', { 'property': 'value' }, + { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', + { 'Amplitude': True }) + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['properties'], { 'property': 'value' }) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['category'], 'category') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'screen') + self.assertEqual(msg['name'], 'name') + + def test_flush(self): + client = self.client + # send a few more requests than a single batch will allow + for i in range(60): + success, msg = client.identify('userId', { 'trait': 'value' }) + + self.assertFalse(client.queue.empty()) + client.flush() + self.assertTrue(client.queue.empty()) + + def test_overflow(self): + client = Client('testsecret', max_queue_size=1) + client.consumer.pause() + time.sleep(5.1) # allow time for consumer to exit + + client.identify('userId') + success, msg = client.identify('userId') + self.assertFalse(success) + + def test_fail(self): + client = Client('bad_key', on_error=self.fail) + client.track('userId', 'event') + client.flush() + self.assertTrue(self.failed) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py new file mode 100644 index 00000000..782d1377 --- /dev/null +++ b/analytics/test/consumer.py @@ -0,0 +1,53 @@ +import unittest + +try: + from queue import Queue +except: + from Queue import Queue + +from analytics.consumer import Consumer + + +class TestConsumer(unittest.TestCase): + + def test_next(self): + q = Queue() + consumer = Consumer(q, '') + q.put(1) + next = consumer.next() + self.assertEqual(next, [1]) + + def test_next_limit(self): + q = Queue() + upload_size = 50 + consumer = Consumer(q, '', upload_size) + for i in range(10000): + q.put(i) + next = consumer.next() + self.assertEqual(next, list(range(upload_size))) + + def test_upload(self): + q = Queue() + consumer = Consumer(q, 'testsecret') + track = { + 'type': 'track', + 'event': 'python event', + 'userId': 'userId' + } + q.put(track) + success = consumer.upload() + self.assertTrue(success) + + def test_request(self): + consumer = Consumer(None, 'testsecret') + track = { + 'type': 'track', + 'event': 'python event', + 'userId': 'userId' + } + consumer.request([track]) + + def test_pause(self): + consumer = Consumer(None, 'testsecret') + consumer.pause() + self.assertFalse(consumer.running) diff --git a/analytics/test/module.py b/analytics/test/module.py new file mode 100644 index 00000000..503e9364 --- /dev/null +++ b/analytics/test/module.py @@ -0,0 +1,45 @@ +import unittest + +import analytics + + +class TestModule(unittest.TestCase): + + def failed(self): + self.failed = True + + def setUp(self): + self.failed = False + analytics.write_key = 'testsecret' + analytics.on_error = self.failed + + def test_no_write_key(self): + analytics.write_key = None + self.assertRaises(Exception, analytics.track) + + def test_track(self): + analytics.track('userId', 'python module event') + analytics.flush() + + def test_identify(self): + analytics.identify('userId', { 'email': 'user@email.com' }) + analytics.flush() + + def test_group(self): + analytics.group('userId', 'groupId') + analytics.flush() + + def test_alias(self): + analytics.alias('previousId', 'userId') + analytics.flush() + + def test_page(self): + analytics.page('userId') + analytics.flush() + + def test_screen(self): + analytics.screen('userId') + analytics.flush() + + def test_flush(self): + analytics.flush() \ No newline at end of file diff --git a/analytics/test/request.py b/analytics/test/request.py new file mode 100644 index 00000000..11c8eec2 --- /dev/null +++ b/analytics/test/request.py @@ -0,0 +1,31 @@ +from datetime import datetime +import unittest +import json + +from analytics.request import post, DatetimeSerializer + + +class TestRequests(unittest.TestCase): + + def test_valid_request(self): + res = post('testsecret', batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track' + }]) + self.assertEqual(res.status_code, 200) + + def test_invalid_write_key(self): + self.assertRaises(Exception, post, 'invalid', batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track' + }]) + + def test_invalid_request_error(self): + self.assertRaises(Exception, post, 'testsecret', batch='invalid') + + def test_datetime_serialization(self): + data = { 'created': datetime(2012, 3, 4, 5, 6, 7, 891011) } + result = json.dumps(data, cls=DatetimeSerializer) + self.assertEqual(result, '{"created": "2012-03-04T05:06:07.891011"}') diff --git a/analytics/test/utils.py b/analytics/test/utils.py new file mode 100644 index 00000000..8be40a4a --- /dev/null +++ b/analytics/test/utils.py @@ -0,0 +1,50 @@ +from datetime import datetime, timedelta +from decimal import Decimal +import unittest + +from dateutil.tz import tzutc +import six + +from analytics import utils + + +class TestUtils(unittest.TestCase): + + def test_timezone_utils(self): + now = datetime.now() + utcnow = datetime.now(tz=tzutc()) + self.assertTrue(utils.is_naive(now)) + self.assertFalse(utils.is_naive(utcnow)) + + fixed = utils.guess_timezone(now) + self.assertFalse(utils.is_naive(fixed)) + + shouldnt_be_edited = utils.guess_timezone(utcnow) + self.assertEqual(utcnow, shouldnt_be_edited) + + def test_clean(self): + simple = { + 'decimal': Decimal('0.142857'), + 'unicode': six.u('woo'), + 'date': datetime.now(), + 'long': 200000000, + 'integer': 1, + 'float': 2.0, + 'bool': True, + 'str': 'woo', + 'none': None + } + + complicated = { + 'exception': Exception('This should show up'), + 'timedelta': timedelta(microseconds=20), + 'list': [1, 2, 3] + } + + combined = dict(simple.items()) + combined.update(complicated.items()) + + pre_clean_keys = combined.keys() + + utils.clean(combined) + self.assertEqual(combined.keys(), pre_clean_keys) diff --git a/analytics/utils.py b/analytics/utils.py index d136a7d4..d7eac370 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -1,23 +1,25 @@ -import json -from datetime import datetime from dateutil.tz import tzlocal, tzutc +from datetime import datetime +import logging +import numbers + +import six def is_naive(dt): - """ Determines if a given datetime.datetime is naive. """ + """Determines if a given datetime.datetime is naive.""" return dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None def total_seconds(delta): - """ Determines total seconds with python < 2.7 compat """ + """Determines total seconds with python < 2.7 compat.""" # http://stackoverflow.com/questions/3694835/python-2-6-5-divide-timedelta-with-timedelta return (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6 def guess_timezone(dt): - """ Attempts to convert a naive datetime to an aware datetime """ + """Attempts to convert a naive datetime to an aware datetime.""" if is_naive(dt): # attempts to guess the datetime.datetime.now() local timezone # case, and then defaults to utc - delta = datetime.now() - dt if total_seconds(delta) < 5: # this was created using datetime.datetime.now() @@ -29,9 +31,38 @@ def guess_timezone(dt): return dt -class DatetimeSerializer(json.JSONEncoder): - def default(self, obj): - if isinstance(obj, datetime): - return obj.isoformat() +def clean(item): + if isinstance(item, (str, six.text_type, int, six.integer_types, float, + bool, numbers.Number, datetime, type(None))): + return item + elif isinstance(item, (set, list, tuple)): + return _clean_list(item) + elif isinstance(item, dict): + return _clean_dict(item) + else: + return _coerce_unicode(item) + +def _clean_list(list_): + return [clean(item) for item in list_] + +def _clean_dict(dict_): + data = {} + for k, v in six.iteritems(dict_): + try: + data[k] = clean(v) + except TypeError: + log = logging.getLogger('segment') + log.warning('Dictionary values must be serializeable to ' + + 'JSON "%s" value %s of type %s is unsupported.' + % (k, v, type(v))) + return data - return json.JSONEncoder.default(self, obj) +def _coerce_unicode(self, cmplx): + try: + item = cmplx.decode("utf-8", "strict") + except AttributeError as exception: + item = ":".join(exception) + item.decode("utf-8", "strict") + except: + raise + return item \ No newline at end of file diff --git a/analytics/version.py b/analytics/version.py index 384a3e1e..1e5a6058 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '0.4.4' +VERSION = '1.0.0' diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..5c7e606e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +python-dateutil \ No newline at end of file diff --git a/setup.py b/setup.py index c03dc9b0..d3ba02d6 100644 --- a/setup.py +++ b/setup.py @@ -25,11 +25,12 @@ name='analytics-python', version=VERSION, url='https://github.com/segmentio/analytics-python', - author='Ilya Volodarsky', - author_email='ilya@segment.io', + author='Segment.io', + author_email='friends@segment.io', maintainer='Segment.io', maintainer_email='friends@segment.io', - packages=['analytics'], + test_suite='analytics.test.all', + packages=['analytics', 'analytics.test'], license='MIT License', install_requires=[ 'requests', diff --git a/test.py b/test.py deleted file mode 100755 index 7d56d464..00000000 --- a/test.py +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -import unittest -import json - -from datetime import datetime, timedelta - -from random import randint -from time import sleep, time -from decimal import * - -import logging -logging.basicConfig() - -from dateutil.tz import tzutc - -import analytics -import analytics.utils - -secret = 'testsecret' - - -def on_success(data, response): - print 'Success', response - - -def on_failure(data, error): - print 'Failure', error - - -class AnalyticsBasicTests(unittest.TestCase): - - def setUp(self): - analytics.init(secret, log_level=logging.DEBUG) - - analytics.on_success(on_success) - analytics.on_failure(on_failure) - - def test_timezone_utils(self): - - now = datetime.now() - utcnow = datetime.now(tz=tzutc()) - - self.assertTrue(analytics.utils.is_naive(now)) - self.assertFalse(analytics.utils.is_naive(utcnow)) - - fixed = analytics.utils.guess_timezone(now) - - self.assertFalse(analytics.utils.is_naive(fixed)) - - shouldnt_be_edited = analytics.utils.guess_timezone(utcnow) - - self.assertEqual(utcnow, shouldnt_be_edited) - - def test_clean(self): - - simple = { - 'integer': 1, - 'float': 2.0, - 'long': 200000000, - 'bool': True, - 'str': 'woo', - 'unicode': u'woo', - 'decimal': Decimal('0.142857'), - 'date': datetime.now(), - } - - complicated = { - 'exception': Exception('This should show up'), - 'timedelta': timedelta(microseconds=20), - 'list': [1, 2, 3] - } - - combined = dict(simple.items() + complicated.items()) - - pre_clean_keys = combined.keys() - - analytics.default_client._clean(combined) - - self.assertEqual(combined.keys(), pre_clean_keys) - - def test_datetime_serialization(self): - - data = { - 'created': datetime(2012, 3, 4, 5, 6, 7, 891011), - } - result = json.dumps(data, cls=analytics.utils.DatetimeSerializer) - - self.assertEqual(result, '{"created": "2012-03-04T05:06:07.891011"}') - - def test_async_basic_identify(self): - # flush after every message - analytics.default_client.flush_at = 1 - analytics.default_client.async = True - - last_identifies = analytics.stats.identifies - last_successful = analytics.stats.successful - last_flushes = analytics.stats.flushes - - analytics.identify('ilya@analytics.io', { - "Subscription Plan": "Free", - "Friends": 30 - }) - - self.assertEqual(analytics.stats.identifies, last_identifies + 1) - - # this should flush because we set the flush_at to 1 - self.assertEqual(analytics.stats.flushes, last_flushes + 1) - - # this should do nothing, as the async thread is currently active - analytics.flush() - - # we should see no more flushes here - self.assertEqual(analytics.stats.flushes, last_flushes + 1) - - sleep(1) - - self.assertEqual(analytics.stats.successful, last_successful + 1) - - def test_async_basic_track(self): - - analytics.default_client.flush_at = 50 - analytics.default_client.async = True - - last_tracks = analytics.stats.tracks - last_successful = analytics.stats.successful - - analytics.track('ilya@analytics.io', 'Played a Song', { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - }) - - self.assertEqual(analytics.stats.tracks, last_tracks + 1) - - analytics.flush() - - sleep(2) - - self.assertEqual(analytics.stats.successful, last_successful + 1) - - def test_async_full_identify(self): - - analytics.default_client.flush_at = 1 - analytics.default_client.async = True - - last_identifies = analytics.stats.identifies - last_successful = analytics.stats.successful - - traits = { - "Subscription Plan": "Free", - "Friends": 30 - } - - context = { - "ip": "12.31.42.111", - "location": { - "countryCode": "US", - "region": "CA" - }, - "userAgent": ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) " + - "AppleWebKit/534.53.11 (KHTML, like Gecko) Version/5.1.3 " + - "Safari/534.53.10"), - "language": "en-us" - } - - analytics.identify('ilya@analytics.io', traits, - context=context, timestamp=datetime.now()) - - self.assertEqual(analytics.stats.identifies, last_identifies + 1) - - sleep(2) - - self.assertEqual(analytics.stats.successful, last_successful + 1) - - def test_async_full_track(self): - - analytics.default_client.flush_at = 1 - analytics.default_client.async = True - - last_tracks = analytics.stats.tracks - last_successful = analytics.stats.successful - - properties = { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - } - - analytics.track('ilya@analytics.io', 'Played a Song', - properties, timestamp=datetime.now()) - - self.assertEqual(analytics.stats.tracks, last_tracks + 1) - - sleep(1) - - self.assertEqual(analytics.stats.successful, last_successful + 1) - - def test_alias(self): - - session_id = str(randint(1000000, 99999999)) - user_id = 'bob+'+session_id + '@gmail.com' - - analytics.default_client.flush_at = 1 - analytics.default_client.async = False - - last_aliases = analytics.stats.aliases - last_successful = analytics.stats.successful - - analytics.identify(session_id, traits={'AnonymousTrait': 'Who am I?'}) - analytics.track(session_id, 'Anonymous Event') - - # alias the user - analytics.alias(session_id, user_id) - - analytics.identify(user_id, traits={'IdentifiedTrait': 'A Hunk'}) - analytics.track(user_id, 'Identified Event') - - self.assertEqual(analytics.stats.aliases, last_aliases + 1) - self.assertEqual(analytics.stats.successful, last_successful + 5) - - def test_blocking_flush(self): - - analytics.default_client.flush_at = 1 - analytics.default_client.async = False - - last_tracks = analytics.stats.tracks - last_successful = analytics.stats.successful - - properties = { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - } - - analytics.track('ilya@analytics.io', 'Played a Song', - properties, timestamp=datetime.today()) - - self.assertEqual(analytics.stats.tracks, last_tracks + 1) - self.assertEqual(analytics.stats.successful, last_successful + 1) - - def test_time_policy(self): - - analytics.default_client.async = False - analytics.default_client.flush_at = 1 - - # add something so we have a reason to flush - analytics.track('ilya@analytics.io', 'Played a Song', { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - }) - - # flush to reset flush count - analytics.flush() - - last_flushes = analytics.stats.flushes - - # set the flush size trigger high - analytics.default_client.flush_at = 50 - # set the time policy to 1 second from now - analytics.default_client.flush_after = timedelta(seconds=1) - - analytics.track('ilya@analytics.io', 'Played a Song', { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - }) - - # that shouldn't of triggered a flush - self.assertEqual(analytics.stats.flushes, last_flushes) - - # sleep past the time-flush policy - sleep(1.2) - - # submit another track to trigger the policy - analytics.track('ilya@analytics.io', 'Played a Song', { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - }) - - self.assertEqual(analytics.stats.flushes, last_flushes + 1) - - def test_performance(self): - - to_send = 100 - - target = analytics.stats.successful + to_send - - analytics.default_client.async = True - analytics.default_client.flush_at = 200 - analytics.default_client.max_flush_size = 50 - analytics.default_client.set_log_level(logging.DEBUG) - - for i in range(to_send): - analytics.track('ilya@analytics.io', 'Played a Song', { - "Artist": "The Beatles", - "Song": "Eleanor Rigby" - }) - - print 'Finished submitting into the queue' - - start = time() - while analytics.stats.successful < target: - print ('Successful ', analytics.stats.successful, 'Left', - (target - analytics.stats.successful), - 'Duration ', (time() - start)) - analytics.flush() - sleep(1.0) - -if __name__ == '__main__': - unittest.main() From f9aa11618fa1aa36264e985d77cbb22f9b64c813 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Fri, 5 Sep 2014 14:57:06 -0700 Subject: [PATCH 008/323] Release 1.0.0 --- history.md | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/history.md b/history.md index 98e24d84..9545ac4c 100644 --- a/history.md +++ b/history.md @@ -1,13 +1,28 @@ -0.4.4 / November 21, 2013 ------------------- -* add < python 2.7 compatibility by removing `delta.total_seconds` - -0.4.3 / November 13, 2013 ------------------- -* added datetime serialization fix (alexlouden) - -0.4.2 / June 26, 2013 ------------------- -* Added history.d change log -* Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! -* Fixing #12, adding static public API to analytics.__init__ \ No newline at end of file + +1.0.0 / 2014-09-05 +================== + + * updating to spec 1.0 + * adding python3 support + * moving to analytics.write_key API + * moving consumer to a separate thread + * adding request retries + * making analytics.flush() syncrhonous + * adding full travis tests + +0.4.4 / 2013-11-21 +================== + + * add < python 2.7 compatibility by removing `delta.total_seconds` + +0.4.3 / 2013-11-13 +================== + + * added datetime serialization fix (alexlouden) + +0.4.2 / 2013-06-26 +================== + + * Added history.d change log + * Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! + * Fixing #12, adding static public API to analytics.__init__ \ No newline at end of file From 62c1436c77dcc7e8fcbbd082f4740e956150e8b3 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Mon, 8 Sep 2014 17:50:41 -0700 Subject: [PATCH 009/323] fixing unicode handling, fixes #26 --- analytics/client.py | 20 ++++++++++---------- analytics/test/client.py | 9 +++++++++ analytics/utils.py | 4 ++-- requirements.txt | 3 ++- setup.py | 3 ++- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index de0e1220..37c091f0 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -2,12 +2,12 @@ from uuid import uuid4 import logging import numbers -import six from dateutil.tz import tzutc +from six import string_types -from analytics.consumer import Consumer from analytics.utils import guess_timezone, clean +from analytics.consumer import Consumer from analytics.version import VERSION try: @@ -16,7 +16,7 @@ import Queue as queue -ID_TYPES = (str, six.text_type, int, six.integer_types, float, numbers.Number) +ID_TYPES = (numbers.Number, string_types) class Client(object): @@ -25,7 +25,7 @@ class Client(object): def __init__(self, write_key=None, debug=False, max_queue_size=10000, send=True, on_error=None): - require('write_key', write_key, str) + require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) self.consumer = Consumer(self.queue, write_key, on_error=on_error) @@ -61,7 +61,7 @@ def track(self, user_id=None, event=None, properties={}, context={}, require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('properties', properties, dict) - require('event', event, str) + require('event', event, string_types) msg = { 'integrations': integrations, @@ -117,9 +117,9 @@ def page(self, user_id=None, category=None, name=None, properties={}, require('properties', properties, dict) if name: - require('name', name, str) + require('name', name, string_types) if category: - require('category', category, str) + require('category', category, string_types) msg = { 'integrations': integrations, @@ -141,9 +141,9 @@ def screen(self, user_id=None, category=None, name=None, properties={}, require('properties', properties, dict) if name: - require('name', name, str) + require('name', name, string_types) if category: - require('category', category, str) + require('category', category, string_types) msg = { 'integrations': integrations, @@ -166,9 +166,9 @@ def _enqueue(self, msg): timestamp = datetime.utcnow().replace(tzinfo=tzutc()) require('integrations', msg['integrations'], dict) + require('type', msg['type'], string_types) require('timestamp', timestamp, datetime) require('context', msg['context'], dict) - require('type', msg['type'], str) # add common timestamp = guess_timezone(timestamp) diff --git a/analytics/test/client.py b/analytics/test/client.py index 96ea3de5..6d488539 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -1,6 +1,7 @@ from datetime import datetime import unittest import time +import six from analytics.version import VERSION from analytics.client import Client @@ -229,3 +230,11 @@ def test_fail(self): client.track('userId', 'event') client.flush() self.assertTrue(self.failed) + + def test_unicode(self): + client = Client(six.u('unicode_key')) + + def test_numeric_user_id(self): + self.client.track(1234, 'python event') + self.client.flush() + self.assertFalse(self.failed) \ No newline at end of file diff --git a/analytics/utils.py b/analytics/utils.py index d7eac370..d3a966a9 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -32,8 +32,8 @@ def guess_timezone(dt): return dt def clean(item): - if isinstance(item, (str, six.text_type, int, six.integer_types, float, - bool, numbers.Number, datetime, type(None))): + if isinstance(item, (six.string_types, bool, numbers.Number, datetime, + type(None))): return item elif isinstance(item, (set, list, tuple)): return _clean_list(item) diff --git a/requirements.txt b/requirements.txt index 5c7e606e..7f7230f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +python-dateutil requests -python-dateutil \ No newline at end of file +six \ No newline at end of file diff --git a/setup.py b/setup.py index d3ba02d6..d4e076ee 100644 --- a/setup.py +++ b/setup.py @@ -33,8 +33,9 @@ packages=['analytics', 'analytics.test'], license='MIT License', install_requires=[ + 'python-dateutil', 'requests', - 'python-dateutil' + 'six' ], description='The hassle-free way to integrate analytics into any python application.', long_description=long_description From 390232d8376674ffe660e2f2c190e455795d5505 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Mon, 8 Sep 2014 18:04:15 -0700 Subject: [PATCH 010/323] Release 1.0.1 --- analytics/version.py | 2 +- history.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 1e5a6058..1dea037c 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.0.0' +VERSION = '1.0.1' diff --git a/history.md b/history.md index 9545ac4c..9b8784ca 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,10 @@ +1.0.1 / 2014-09-08 +================== + + * fixing unicode handling, for write_key and events + * adding six to requirements.txt and install scripts + 1.0.0 / 2014-09-05 ================== From cbd3cefb905b2e859e7f33731ad0d00f5d2d9fe2 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Wed, 17 Sep 2014 12:05:16 -0700 Subject: [PATCH 011/323] fixing debug logging levels Realized that 'DEBUG' will error while 'debug' is good. --- analytics/client.py | 2 +- analytics/test/client.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 37c091f0..5a5b3a7c 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -35,7 +35,7 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, self.send = send if debug: - self.log.setLevel('debug') + self.log.setLevel('DEBUG') self.consumer.start() diff --git a/analytics/test/client.py b/analytics/test/client.py index 6d488539..45d0f442 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -232,9 +232,12 @@ def test_fail(self): self.assertTrue(self.failed) def test_unicode(self): - client = Client(six.u('unicode_key')) + Client(six.u('unicode_key')) def test_numeric_user_id(self): self.client.track(1234, 'python event') self.client.flush() - self.assertFalse(self.failed) \ No newline at end of file + self.assertFalse(self.failed) + + def test_debug(self): + Client('bad_key', debug=True) \ No newline at end of file From 161a92b522c0e86206850166675ee491f9e045eb Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Wed, 17 Sep 2014 12:17:14 -0700 Subject: [PATCH 012/323] Release 1.0.2 --- analytics/version.py | 2 +- history.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 1dea037c..6732d5aa 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.0.1' +VERSION = '1.0.2' diff --git a/history.md b/history.md index 9b8784ca..2d9bd0fa 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,10 @@ +1.0.2 / 2014-09-17 +================== + + * fixing debug logging levels + + 1.0.1 / 2014-09-08 ================== From d465e1834c2659f16f2bc25da7ebfa1c29af594b Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Tue, 30 Sep 2014 16:10:02 -0700 Subject: [PATCH 013/323] adding top level send option --- analytics/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 2df084bc..4179dd0f 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -8,6 +8,7 @@ write_key = None on_error = None debug = False +send=True default_client = None @@ -44,7 +45,8 @@ def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client if not default_client: - default_client = Client(write_key, debug=debug, on_error=on_error) + default_client = Client(write_key, debug=debug, on_error=on_error, + send=send) fn = getattr(default_client, method) fn(*args, **kwargs) From 627637e2b5de0d423d807320790bde224854923c Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Tue, 30 Sep 2014 16:18:35 -0700 Subject: [PATCH 014/323] Release 1.0.3 --- analytics/version.py | 2 +- history.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 6732d5aa..33f9c5a9 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.0.2' +VERSION = '1.0.3' diff --git a/history.md b/history.md index 2d9bd0fa..1f2f1777 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,9 @@ +1.0.3 / 2014-09-30 +================== + + * adding top level send option + 1.0.2 / 2014-09-17 ================== From 05f502232ae0c4ff491001c12e68d18aa20bab56 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Thu, 2 Oct 2014 22:05:05 -0700 Subject: [PATCH 015/323] spacing --- analytics/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 4179dd0f..c16ef1c4 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -8,7 +8,7 @@ write_key = None on_error = None debug = False -send=True +send = True default_client = None From 10d69ac905b61172970df2d61beca466260944c3 Mon Sep 17 00:00:00 2001 From: Ilya Volodarsky Date: Thu, 4 Dec 2014 11:18:52 -0800 Subject: [PATCH 016/323] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8ef4c87d..4f3fd9ad 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,11 @@ [![Build Status](https://travis-ci.org/segmentio/analytics-python.svg?branch=master)](https://travis-ci.org/segmentio/analytics-python) -analytics-python is a python client for [Segment.io](https://segment.io) +analytics-python is a python client for [Segment](https://segment.com) ## Documentation -Documentation is available at [https://segment.io/libraries/python](https://segment.io/libraries/python). +Documentation is available at [https://segment.com/libraries/python](https://segment.com/libraries/python). ## License @@ -26,7 +26,7 @@ WWWWWW||WWWWWW (The MIT License) -Copyright (c) 2013 Segment.io Inc. +Copyright (c) 2013 Segment Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: From ee43687c188e64c409afe267bb11c1501fa614fb Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Sat, 13 Dec 2014 13:00:39 -0800 Subject: [PATCH 017/323] removing .io's --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d4e076ee..d8840ba7 100644 --- a/setup.py +++ b/setup.py @@ -25,10 +25,10 @@ name='analytics-python', version=VERSION, url='https://github.com/segmentio/analytics-python', - author='Segment.io', - author_email='friends@segment.io', - maintainer='Segment.io', - maintainer_email='friends@segment.io', + author='Segment', + author_email='friends@segment.com', + maintainer='Segment', + maintainer_email='friends@segment.com', test_suite='analytics.test.all', packages=['analytics', 'analytics.test'], license='MIT License', From 84fbe86c5ad7c7da940ffad5d461a950860aab1a Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Sat, 13 Dec 2014 13:07:07 -0800 Subject: [PATCH 018/323] fixing overflow test --- analytics/test/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 45d0f442..c3cc6c36 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -221,7 +221,9 @@ def test_overflow(self): client.consumer.pause() time.sleep(5.1) # allow time for consumer to exit - client.identify('userId') + for i in range(10): + client.identify('userId') + success, msg = client.identify('userId') self.assertFalse(success) From 8c4426a410088636414fb79cddf44f4140f9a480 Mon Sep 17 00:00:00 2001 From: Damien Cirotteau Date: Mon, 29 Dec 2014 23:15:28 +0100 Subject: [PATCH 019/323] make it really testable --- analytics/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 5a5b3a7c..c6b4b7fe 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -37,7 +37,9 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, if debug: self.log.setLevel('DEBUG') - self.consumer.start() + # if we've disabled sending, just don't start the consumer + if send: + self.consumer.start() def identify(self, user_id=None, traits={}, context={}, timestamp=None, anonymous_id=None, integrations={}): @@ -181,10 +183,6 @@ def _enqueue(self, msg): clean(msg) - # if we've disabled sending, just return False - if not self.send: - return False, msg - if self.queue.full(): self.log.warn('analytics-python queue is full') return False, msg From f904c1cce847288d31cdbd3901dbe89a5c85648f Mon Sep 17 00:00:00 2001 From: Amir Abushareb Date: Wed, 11 Feb 2015 16:29:13 +0200 Subject: [PATCH 020/323] adding .sentAt --- analytics/request.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/analytics/request.py b/analytics/request.py index 064d81e4..edaa8a25 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -1,4 +1,5 @@ from datetime import datetime +from dateutil.tz import tzutc import logging import json @@ -10,6 +11,7 @@ def post(write_key, **kwargs): """Post the `kwargs` to the API""" log = logging.getLogger('segment') body = kwargs + body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat() url = 'https://api.segment.io/v1/batch' auth = HTTPBasicAuth(write_key, '') data = json.dumps(body, cls=DatetimeSerializer) From 28ad04ae3aaeb77a381162ad93f92ed156fdfc57 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Thu, 16 Apr 2015 13:19:02 -0600 Subject: [PATCH 021/323] Update tests --- .gitignore | 3 ++- analytics/test/client.py | 4 ++-- analytics/test/request.py | 9 +-------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 7943ec53..2d736009 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ dist *.egg-info dist MANIFEST -build \ No newline at end of file +build +.eggs \ No newline at end of file diff --git a/analytics/test/client.py b/analytics/test/client.py index c3cc6c36..071e95aa 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -227,11 +227,11 @@ def test_overflow(self): success, msg = client.identify('userId') self.assertFalse(success) - def test_fail(self): + def test_success_on_invalid_write_key(self): client = Client('bad_key', on_error=self.fail) client.track('userId', 'event') client.flush() - self.assertTrue(self.failed) + self.assertFalse(self.failed) def test_unicode(self): Client(six.u('unicode_key')) diff --git a/analytics/test/request.py b/analytics/test/request.py index 11c8eec2..5d1ce5ee 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -15,15 +15,8 @@ def test_valid_request(self): }]) self.assertEqual(res.status_code, 200) - def test_invalid_write_key(self): - self.assertRaises(Exception, post, 'invalid', batch=[{ - 'userId': 'userId', - 'event': 'python event', - 'type': 'track' - }]) - def test_invalid_request_error(self): - self.assertRaises(Exception, post, 'testsecret', batch='invalid') + self.assertRaises(Exception, post, 'testsecret', '[{]') def test_datetime_serialization(self): data = { 'created': datetime(2012, 3, 4, 5, 6, 7, 891011) } From 57bfa80e0adb61a7446778964af2bf0403286d7f Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 3 Jun 2015 17:43:29 +0100 Subject: [PATCH 022/323] Suppport universal wheels You'll need to deploy using `python setup.py sdist bdist_wheel register upload` --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..2a9acf13 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 From 24b79f29c703659d876b6be593e4c60209170219 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 3 Jun 2015 18:06:49 +0100 Subject: [PATCH 023/323] Support HTTP keep-alive using a Session connection pool ``` $ python -m timeit -s "import requests" "requests.head('https://graingert.co.uk')" 10 loops, best of 3: 199 msec per loop $ python -m timeit -s "import requests; sess=requests.session()" "sess.head('https://graingert.co.uk')" 10 loops, best of 3: 136 msec per loop ``` --- analytics/request.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/analytics/request.py b/analytics/request.py index edaa8a25..59578b4b 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -4,7 +4,9 @@ import json from requests.auth import HTTPBasicAuth -import requests +from requests import sessions + +_session = sessions.Session() def post(write_key, **kwargs): @@ -17,7 +19,7 @@ def post(write_key, **kwargs): data = json.dumps(body, cls=DatetimeSerializer) headers = { 'content-type': 'application/json' } log.debug('making request: %s', data) - res = requests.post(url, data=data, auth=auth, headers=headers, timeout=15) + res = _session.post(url, data=data, auth=auth, headers=headers, timeout=15) if res.status_code == 200: log.debug('data uploaded successfully') @@ -48,4 +50,4 @@ def default(self, obj): if isinstance(obj, datetime): return obj.isoformat() - return json.JSONEncoder.default(self, obj) \ No newline at end of file + return json.JSONEncoder.default(self, obj) From 538886d5df69c999c193addb6ed3ace6b7cd64b6 Mon Sep 17 00:00:00 2001 From: Tomasz Kolinko Date: Wed, 17 Jun 2015 14:10:06 +0200 Subject: [PATCH 024/323] Added "join" functionn to the client. It blocks program until the queue is empty. Join should be called before the application finishes, when you want to make sure that all the messages are sent. It's especially useful when developing for Google App Engine which doesn't like any leftover threads after the web request is complete, causing errors. --- analytics/__init__.py | 4 ++++ analytics/client.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/analytics/__init__.py b/analytics/__init__.py index c16ef1c4..433ab61e 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -41,6 +41,10 @@ def flush(): """Tell the client to flush.""" _proxy('flush') +def join(): + """Block program until the client clears the queue""" + _proxy('join') + def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client diff --git a/analytics/client.py b/analytics/client.py index c6b4b7fe..06a140ba 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -199,6 +199,13 @@ def flush(self): self.log.debug('successfully flushed {0} items.'.format(size)) + def join(self): + """Ends the consumer thread once the queue is empty. Blocks execution until finished""" + self.consumer.pause() + self.consumer.join() + + + def require(name, field, data_type): """Require that the named `field` has the right `data_type`""" if not isinstance(field, data_type): From 7214b8e5da808a4efaae9a2be6808f08e3f2f924 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Mon, 22 Jun 2015 23:25:53 -0700 Subject: [PATCH 025/323] Adding `logging.DEBUG` fix for `setLevel` --- analytics/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/client.py b/analytics/client.py index 06a140ba..f18d3108 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -35,7 +35,7 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, self.send = send if debug: - self.log.setLevel('DEBUG') + self.log.setLevel(logging.DEBUG) # if we've disabled sending, just don't start the consumer if send: From 1e3bb52e8abb20b81c816b5cb681d0cd8ba24d67 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Mon, 22 Jun 2015 23:28:11 -0700 Subject: [PATCH 026/323] Fixing byte/bytearray handling --- analytics/test/utils.py | 8 ++++++++ analytics/utils.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/analytics/test/utils.py b/analytics/test/utils.py index 8be40a4a..8325904b 100644 --- a/analytics/test/utils.py +++ b/analytics/test/utils.py @@ -48,3 +48,11 @@ def test_clean(self): utils.clean(combined) self.assertEqual(combined.keys(), pre_clean_keys) + + def test_bytes(self): + if six.PY3: + item = bytes(10) + else: + item = bytearray(10) + + utils.clean(item) diff --git a/analytics/utils.py b/analytics/utils.py index d3a966a9..b293d547 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -26,7 +26,7 @@ def guess_timezone(dt): # so we are in the local timezone return dt.replace(tzinfo=tzlocal()) else: - # at this point, the best we can do (I htink) is guess UTC + # at this point, the best we can do is guess UTC return dt.replace(tzinfo=tzutc()) return dt @@ -57,7 +57,7 @@ def _clean_dict(dict_): % (k, v, type(v))) return data -def _coerce_unicode(self, cmplx): +def _coerce_unicode(cmplx): try: item = cmplx.decode("utf-8", "strict") except AttributeError as exception: From 804e1f87399e00ee88095124a03c5ac63a1660bc Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Tue, 23 Jun 2015 00:24:28 -0700 Subject: [PATCH 027/323] Adding fixes for handling invalid json types Previously invalid json types weren't being modified and would end up simply stringifying the field. Now the fields are actually modified and removed so that they don't pollute analytics. --- analytics/client.py | 4 ++-- analytics/test/utils.py | 7 +++++++ analytics/utils.py | 5 ++++- setup.py | 4 ++-- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index f18d3108..5555ae15 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -181,7 +181,8 @@ def _enqueue(self, msg): 'version': VERSION } - clean(msg) + msg = clean(msg) + self.log.debug('queueing: %s', msg) if self.queue.full(): self.log.warn('analytics-python queue is full') @@ -198,7 +199,6 @@ def flush(self): queue.join() self.log.debug('successfully flushed {0} items.'.format(size)) - def join(self): """Ends the consumer thread once the queue is empty. Blocks execution until finished""" self.consumer.pause() diff --git a/analytics/test/utils.py b/analytics/test/utils.py index 8325904b..30eb0bad 100644 --- a/analytics/test/utils.py +++ b/analytics/test/utils.py @@ -56,3 +56,10 @@ def test_bytes(self): item = bytearray(10) utils.clean(item) + + def test_clean_fn(self): + cleaned = utils.clean({ 'fn': lambda x: x, 'number': 4 }) + self.assertEqual(cleaned['number'], 4) + # TODO: fixme, different behavior on python 2 and 3 + if 'fn' in cleaned: + self.assertEqual(cleaned['fn'], None) diff --git a/analytics/utils.py b/analytics/utils.py index b293d547..2eb5b13d 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -5,6 +5,8 @@ import six +log = logging.getLogger('segment') + def is_naive(dt): """Determines if a given datetime.datetime is naive.""" @@ -51,7 +53,6 @@ def _clean_dict(dict_): try: data[k] = clean(v) except TypeError: - log = logging.getLogger('segment') log.warning('Dictionary values must be serializeable to ' + 'JSON "%s" value %s of type %s is unsupported.' % (k, v, type(v))) @@ -63,6 +64,8 @@ def _coerce_unicode(cmplx): except AttributeError as exception: item = ":".join(exception) item.decode("utf-8", "strict") + log.warning('Error decoding: %s', item) + return None except: raise return item \ No newline at end of file diff --git a/setup.py b/setup.py index d8840ba7..73d3ba97 100644 --- a/setup.py +++ b/setup.py @@ -12,11 +12,11 @@ from version import VERSION long_description = ''' -Segment.io is the simplest way to integrate analytics into your application. +Segment is the simplest way to integrate analytics into your application. One API allows you to turn on any other analytics service. No more learning new APIs, repeated code, and wasted development time. -This is the official python client that wraps the Segment.io REST API (https://segment.io). +This is the official python client that wraps the Segment REST API (https://segment.com). Documentation and more details at https://github.com/segmentio/analytics-python ''' From c193c77ae9f7ae810d8d5e0405ce8e0513a38541 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Tue, 23 Jun 2015 00:54:37 -0700 Subject: [PATCH 028/323] Release 1.1.0 --- Makefile | 5 ++++- analytics/version.py | 2 +- history.md | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 28e9c14a..7259d7bf 100644 --- a/Makefile +++ b/Makefile @@ -2,4 +2,7 @@ test: python setup.py test -.PHONY: test \ No newline at end of file +dist: + python setup.py sdist bdist_wheel upload + +.PHONY: test dist \ No newline at end of file diff --git a/analytics/version.py b/analytics/version.py index 33f9c5a9..9cff1bc2 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.0.3' +VERSION = '1.1.0' diff --git a/history.md b/history.md index 1f2f1777..04a4a9c3 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,19 @@ +1.1.0 / 2015-06-23 +================== + + * Adding fixes for handling invalid json types + * Fixing byte/bytearray handling + * Adding `logging.DEBUG` fix for `setLevel` + * Support HTTP keep-alive using a Session connection pool + * Suppport universal wheels + * adding .sentAt + * make it really testable + * fixing overflow test + * removing .io's + * Update README.md + * spacing + 1.0.3 / 2014-09-30 ================== @@ -42,4 +57,4 @@ * Added history.d change log * Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! - * Fixing #12, adding static public API to analytics.__init__ \ No newline at end of file + * Fixing #12, adding static public API to analytics.__init__ From 263c46f11f63f9d526d46caf49f72495d28f5dce Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 2 Sep 2015 11:51:18 +0100 Subject: [PATCH 029/323] ensure all logging is correctly templated --- analytics/client.py | 4 ++-- analytics/utils.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 5555ae15..9de932e8 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -189,7 +189,7 @@ def _enqueue(self, msg): return False, msg self.queue.put(msg) - self.log.debug('enqueued ' + msg['type'] + '.') + self.log.debug('enqueued %s.', msg['type']) return True, msg def flush(self): @@ -197,7 +197,7 @@ def flush(self): queue = self.queue size = queue.qsize() queue.join() - self.log.debug('successfully flushed {0} items.'.format(size)) + self.log.debug('successfully flushed %s items.', size) def join(self): """Ends the consumer thread once the queue is empty. Blocks execution until finished""" diff --git a/analytics/utils.py b/analytics/utils.py index 2eb5b13d..c96a4bff 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -53,9 +53,11 @@ def _clean_dict(dict_): try: data[k] = clean(v) except TypeError: - log.warning('Dictionary values must be serializeable to ' + - 'JSON "%s" value %s of type %s is unsupported.' - % (k, v, type(v))) + log.warning( + 'Dictionary values must be serializeable to ' + 'JSON "%s" value %s of type %s is unsupported.', + k, v, type(v), + ) return data def _coerce_unicode(cmplx): @@ -68,4 +70,4 @@ def _coerce_unicode(cmplx): return None except: raise - return item \ No newline at end of file + return item From 0cbefde790beeabc2738d1e7fc747632d993ef5b Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 27 Nov 2015 15:37:09 +0000 Subject: [PATCH 030/323] add 3.5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b65b688c..cb1a5dd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ python: - "3.2" - "3.3" - "3.4" + - "3.5" install: - "pip install ." - "pip install -r requirements.txt" From d128fcd97d8399acb3152871301d95ae34c4a5b4 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Fri, 27 Nov 2015 15:40:37 +0000 Subject: [PATCH 031/323] Add trove classifiers for caniusepython3 --- setup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 73d3ba97..91b99b15 100644 --- a/setup.py +++ b/setup.py @@ -38,5 +38,20 @@ 'six' ], description='The hassle-free way to integrate analytics into any python application.', - long_description=long_description + long_description=long_description, + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.2", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + ], ) From a23a2411ead7aa43b098a62f46e62fa67a3aa26c Mon Sep 17 00:00:00 2001 From: Justin Henck Date: Fri, 8 Jan 2016 21:16:08 -0500 Subject: [PATCH 032/323] Quick fix for Decimal to send as a float Send decimals as floats --- analytics/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/analytics/utils.py b/analytics/utils.py index c96a4bff..6840f359 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -1,5 +1,6 @@ from dateutil.tz import tzlocal, tzutc from datetime import datetime +from decimal import Decimal import logging import numbers @@ -34,7 +35,9 @@ def guess_timezone(dt): return dt def clean(item): - if isinstance(item, (six.string_types, bool, numbers.Number, datetime, + if isinstance(item, Decimal): + return float(item) + elif isinstance(item, (six.string_types, bool, numbers.Number, datetime, type(None))): return item elif isinstance(item, (set, list, tuple)): From 6585ef8369241d9c6ad01984554c5671a5ab01ed Mon Sep 17 00:00:00 2001 From: Joshua Kehn Date: Mon, 8 Feb 2016 18:21:45 -0500 Subject: [PATCH 033/323] Remove uses of mutable default parameters Ref #67 --- analytics/client.py | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 9de932e8..c4d9d8ac 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -41,8 +41,11 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, if send: self.consumer.start() - def identify(self, user_id=None, traits={}, context={}, timestamp=None, - anonymous_id=None, integrations={}): + def identify(self, user_id=None, traits=None, context=None, timestamp=None, + anonymous_id=None, integrations=None): + traits = traits or {} + context = context or {} + integrations = integrations or {} require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('traits', traits, dict) @@ -58,9 +61,11 @@ def identify(self, user_id=None, traits={}, context={}, timestamp=None, return self._enqueue(msg) - def track(self, user_id=None, event=None, properties={}, context={}, - timestamp=None, anonymous_id=None, integrations={}): - + def track(self, user_id=None, event=None, properties=None, context=None, + timestamp=None, anonymous_id=None, integrations=None): + properties = properties or {} + context = context or {} + integrations = integrations or {} require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('properties', properties, dict) require('event', event, string_types) @@ -78,8 +83,10 @@ def track(self, user_id=None, event=None, properties={}, context={}, return self._enqueue(msg) - def alias(self, previous_id=None, user_id=None, context={}, timestamp=None, - integrations={}): + def alias(self, previous_id=None, user_id=None, context=None, + timestamp=None, integrations=None): + context = context or {} + integrations = integrations or {} require('previous_id', previous_id, ID_TYPES) require('user_id', user_id, ID_TYPES) @@ -94,8 +101,11 @@ def alias(self, previous_id=None, user_id=None, context={}, timestamp=None, return self._enqueue(msg) - def group(self, user_id=None, group_id=None, traits={}, context={}, - timestamp=None, anonymous_id=None, integrations={}): + def group(self, user_id=None, group_id=None, traits=None, context=None, + timestamp=None, anonymous_id=None, integrations=None): + traits = traits or {} + context = context or {} + integrations = integrations or {} require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('group_id', group_id, ID_TYPES) require('traits', traits, dict) @@ -113,8 +123,12 @@ def group(self, user_id=None, group_id=None, traits={}, context={}, return self._enqueue(msg) - def page(self, user_id=None, category=None, name=None, properties={}, - context={}, timestamp=None, anonymous_id=None, integrations={}): + def page(self, user_id=None, category=None, name=None, properties=None, + context=None, timestamp=None, anonymous_id=None, + integrations=None): + properties = properties or {} + context = context or {} + integrations = integrations or {} require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('properties', properties, dict) @@ -137,8 +151,12 @@ def page(self, user_id=None, category=None, name=None, properties={}, return self._enqueue(msg) - def screen(self, user_id=None, category=None, name=None, properties={}, - context={}, timestamp=None, anonymous_id=None, integrations={}): + def screen(self, user_id=None, category=None, name=None, properties=None, + context=None, timestamp=None, anonymous_id=None, + integrations=None): + properties = properties or {} + context = context or {} + integrations = integrations or {} require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('properties', properties, dict) @@ -205,7 +223,6 @@ def join(self): self.consumer.join() - def require(name, field, data_type): """Require that the named `field` has the right `data_type`""" if not isinstance(field, data_type): From 12b8a1a2d39c44e7cd593274be448af0d4fa4840 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Fri, 11 Mar 2016 12:14:19 -0800 Subject: [PATCH 034/323] adding versioned requirements.txt file Adds versioning to our requirements so that folks make sure they get the right version. We can relax these over time via PRs --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7f7230f9..9516dcee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -python-dateutil -requests -six \ No newline at end of file +python-dateutil==1.5 +requests==2.7.0 +six==1.4.1 \ No newline at end of file From 2cce718e89f1971c84b9a7dad0d01ceb863d49d8 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Fri, 11 Mar 2016 12:15:07 -0800 Subject: [PATCH 035/323] Release 1.2.0 --- analytics/version.py | 2 +- history.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 9cff1bc2..ee65984a 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.1.0' +VERSION = '1.2.0' diff --git a/history.md b/history.md index 04a4a9c3..8fbd1eab 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,9 @@ +1.2.0 / 2016-03-11 +================== + + * adding versioned requirements.txt file + 1.1.0 / 2015-06-23 ================== From b296ddb9c7b68d101d97b3ca2e8437786c138385 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Fri, 11 Mar 2016 12:42:22 -0800 Subject: [PATCH 036/323] fixing requirements.txt --- .travis.yml | 1 - requirements.txt | 3 --- setup.py | 16 +++++++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) delete mode 100644 requirements.txt diff --git a/.travis.yml b/.travis.yml index cb1a5dd1..be462ddd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ python: - "3.5" install: - "pip install ." - - "pip install -r requirements.txt" script: make test matrix: fast_finish: true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9516dcee..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -python-dateutil==1.5 -requests==2.7.0 -six==1.4.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 91b99b15..7fc0a482 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,16 @@ Documentation and more details at https://github.com/segmentio/analytics-python ''' +install_requires = [ + "requests>=2.7,<2.8", + "six>=1.5" +] + +if sys.version_info <= (3,0): + install_requires.append('python-dateutil>=1,<2') +else: + install_requires.append('python-dateutil>2') + setup( name='analytics-python', version=VERSION, @@ -32,11 +42,7 @@ test_suite='analytics.test.all', packages=['analytics', 'analytics.test'], license='MIT License', - install_requires=[ - 'python-dateutil', - 'requests', - 'six' - ], + install_requires=install_requires, description='The hassle-free way to integrate analytics into any python application.', long_description=long_description, classifiers=[ From 344153aedb7734c0250050a4ad0dd11080956f39 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Fri, 11 Mar 2016 14:08:06 -0800 Subject: [PATCH 037/323] Release 1.2.1 --- analytics/version.py | 2 +- history.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index ee65984a..f30a0b87 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.0' +VERSION = '1.2.1' diff --git a/history.md b/history.md index 8fbd1eab..24c5e85d 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,9 @@ +1.2.1 / 2016-03-11 +================== + + * fixing requirements.txt + 1.2.0 / 2016-03-11 ================== From 4e2bc4b80d307bbb7a146b0f9049963f4b7897af Mon Sep 17 00:00:00 2001 From: Andrew Sherepa Date: Tue, 15 Mar 2016 01:52:51 +0200 Subject: [PATCH 038/323] Use proper way for defining conditional dependencies We always install python-dateutil (>=1,<2), even for Python 3, with previous version of setup.py when using wheel package. So we must use another solution which is described here https://wheel.readthedocs.org/en/latest/#defining-conditional-dependencies --- setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 7fc0a482..460067d6 100644 --- a/setup.py +++ b/setup.py @@ -26,10 +26,10 @@ "six>=1.5" ] -if sys.version_info <= (3,0): - install_requires.append('python-dateutil>=1,<2') -else: - install_requires.append('python-dateutil>2') +extras_require={ + ':python_version<="3.0"': ['python-dateutil>=1,<2'], + ':python_version>"3.0"': ['python-dateutil>2'] +} setup( name='analytics-python', @@ -43,6 +43,7 @@ packages=['analytics', 'analytics.test'], license='MIT License', install_requires=install_requires, + extras_require=extras_require, description='The hassle-free way to integrate analytics into any python application.', long_description=long_description, classifiers=[ From d2f6a4e6217c381cb19ea8f16d05dc0d259fb5cd Mon Sep 17 00:00:00 2001 From: Andrew Sherepa Date: Fri, 18 Mar 2016 02:02:44 +0200 Subject: [PATCH 039/323] Fix environment markers definition Setuptools<17.1 doesn't support some comparison operators. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 460067d6..c3aed26d 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,8 @@ ] extras_require={ - ':python_version<="3.0"': ['python-dateutil>=1,<2'], - ':python_version>"3.0"': ['python-dateutil>2'] + ':python_version in "2.6, 2.7"': ['python-dateutil>=1,<2'], + ':python_version in "3.2, 3.3, 3.4, 3.5"': ['python-dateutil>2'] } setup( From d4cffdcfedbeb063b67a197efb374e3b07ea4d32 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Thu, 17 Mar 2016 17:26:09 -0700 Subject: [PATCH 040/323] Release 1.2.2 --- analytics/version.py | 2 +- history.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index f30a0b87..6eb0219c 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.1' +VERSION = '1.2.2' diff --git a/history.md b/history.md index 24c5e85d..ad580181 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,10 @@ +1.2.2 / 2016-03-17 +================== + + * Fix environment markers definition + * Use proper way for defining conditional dependencies + 1.2.1 / 2016-03-11 ================== From 5dd4ff040272673f34b9947f8f5fb929f8edb86b Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Wed, 23 Mar 2016 22:24:25 -0700 Subject: [PATCH 041/323] relaxing requests dep --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c3aed26d..0f6314af 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ ''' install_requires = [ - "requests>=2.7,<2.8", + "requests>=2.7,<3.0", "six>=1.5" ] From 2d6f33e385e44d5b437c374dcf13f3ff243558d8 Mon Sep 17 00:00:00 2001 From: Calvin French-Owen Date: Wed, 23 Mar 2016 22:30:48 -0700 Subject: [PATCH 042/323] Release 1.2.3 --- analytics/version.py | 2 +- history.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 6eb0219c..e6f3c202 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.2' +VERSION = '1.2.3' diff --git a/history.md b/history.md index ad580181..e8e2b9b1 100644 --- a/history.md +++ b/history.md @@ -1,4 +1,9 @@ +1.2.3 / 2016-03-23 +================== + + * relaxing requests dep + 1.2.2 / 2016-03-17 ================== From 7c32b916d84ff78cb6c78272afbe32f6b97715cd Mon Sep 17 00:00:00 2001 From: Adam Chainz Date: Sun, 15 May 2016 18:11:47 +0100 Subject: [PATCH 043/323] Capitalize HISTORY.md (#76) This makes it more in-line with the de-facto changelog 'standards' and it stands out to impatient skim readers like myself. --- history.md => HISTORY.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename history.md => HISTORY.md (100%) diff --git a/history.md b/HISTORY.md similarity index 100% rename from history.md rename to HISTORY.md From e3fdf702a19a478d7ce4a893f1e47624c080bf0b Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Fri, 3 Jun 2016 09:28:01 -0700 Subject: [PATCH 044/323] Join daemon thread on interpreter exit This allows the message sending queue to clean up nicely and prevents an error as the interpreter is destroyed. If you are exiting intentionally you should still call flush() to ensure all your messages are delivered -- this doesn't change that behavior. We also change the blocking semantics on the delivery thread here to deliver messages in a more efficient manner. Fixes #46. Fixes #69. --- analytics/client.py | 10 +++++++++- analytics/consumer.py | 31 +++++++++++-------------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index c4d9d8ac..9cfda249 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -2,6 +2,7 @@ from uuid import uuid4 import logging import numbers +import atexit from dateutil.tz import tzutc from six import string_types @@ -34,6 +35,12 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, self.debug = debug self.send = send + # On program exit, allow the consumer thread to exit cleanly. + # This prevents exceptions and a messy shutdown when the interpreter is + # destroyed before the daemon thread finishes execution. However, it + # is *not* the same as flushing the queue! To guarantee all + atexit.register(self.join) + if debug: self.log.setLevel(logging.DEBUG) @@ -215,7 +222,8 @@ def flush(self): queue = self.queue size = queue.qsize() queue.join() - self.log.debug('successfully flushed %s items.', size) + # Note that this message may not be precise, because of threading. + self.log.debug('successfully flushed about %s items.', size) def join(self): """Ends the consumer thread once the queue is empty. Blocks execution until finished""" diff --git a/analytics/consumer.py b/analytics/consumer.py index ca1761f4..e533926a 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -1,9 +1,13 @@ -from threading import Thread import logging +from threading import Thread from analytics.version import VERSION from analytics.request import post +try: + from queue import Empty +except: + from Queue import Empty class Consumer(Thread): """Consumes the messages from the client's queue.""" @@ -22,7 +26,6 @@ def __init__(self, queue, write_key, upload_size=100, on_error=None): def run(self): """Runs the consumer.""" self.log.debug('consumer is running...') - self.running = True while self.running: self.upload() @@ -49,37 +52,25 @@ def upload(self): if self.on_error: self.on_error(e, batch) finally: - # cleanup + # mark items as acknowledged from queue for item in batch: self.queue.task_done() - return success def next(self): """Return the next batch of items to upload.""" queue = self.queue items = [] - item = self.next_item() - if item is None: - return items - items.append(item) - while len(items) < self.upload_size and not queue.empty(): - item = self.next_item() - if item: + while len(items) < self.upload_size or self.queue.empty(): + try: + item = queue.get(block=True, timeout=0.5) items.append(item) + except Empty: + break return items - def next_item(self): - """Get a single item from the queue.""" - queue = self.queue - try: - item = queue.get(block=True, timeout=5) - return item - except Exception: - return None - def request(self, batch, attempt=0): """Attempt to upload the batch and retry before raising an error """ try: From e5cbe85179445762d084554db7aea78a40ed054d Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Fri, 3 Jun 2016 10:05:55 -0700 Subject: [PATCH 045/323] fix comments --- analytics/client.py | 3 ++- analytics/consumer.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 9cfda249..b9be0ad7 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -38,7 +38,8 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, # On program exit, allow the consumer thread to exit cleanly. # This prevents exceptions and a messy shutdown when the interpreter is # destroyed before the daemon thread finishes execution. However, it - # is *not* the same as flushing the queue! To guarantee all + # is *not* the same as flushing the queue! To guarantee all messages + # have been delivered, you'll still need to call flush(). atexit.register(self.join) if debug: diff --git a/analytics/consumer.py b/analytics/consumer.py index e533926a..a54a89cf 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -16,7 +16,8 @@ class Consumer(Thread): def __init__(self, queue, write_key, upload_size=100, on_error=None): """Create a consumer thread.""" Thread.__init__(self) - self.daemon = True # set as a daemon so the program can exit + # Make consumer a daemon thread so that it doesn't block program exit + self.daemon = True self.upload_size = upload_size self.write_key = write_key self.on_error = on_error From 5874f222bfa872bf3a663429727100c399bb6923 Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Fri, 3 Jun 2016 14:22:43 -0700 Subject: [PATCH 046/323] Fix race condition in flush test --- analytics/test/client.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 071e95aa..e059604e 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -208,23 +208,25 @@ def test_advanced_screen(self): def test_flush(self): client = self.client - # send a few more requests than a single batch will allow - for i in range(60): + # set up the consumer with more requests than a single batch will allow + for i in range(1000): success, msg = client.identify('userId', { 'trait': 'value' }) - - self.assertFalse(client.queue.empty()) + # We can't reliably assert that the queue is non-empty here; that's + # a race condition. We do our best to load it up though. client.flush() + # Make sure that the client queue is empty after flushing self.assertTrue(client.queue.empty()) def test_overflow(self): client = Client('testsecret', max_queue_size=1) client.consumer.pause() - time.sleep(5.1) # allow time for consumer to exit + time.sleep(1.0) # allow time for consumer to exit for i in range(10): client.identify('userId') success, msg = client.identify('userId') + # Make sure we are informed that the queue is at capacity self.assertFalse(success) def test_success_on_invalid_write_key(self): @@ -242,4 +244,4 @@ def test_numeric_user_id(self): self.assertFalse(self.failed) def test_debug(self): - Client('bad_key', debug=True) \ No newline at end of file + Client('bad_key', debug=True) From b8e8a7fe9f38bf0fa8d3e5d8e70f32246b0bbdd7 Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Fri, 3 Jun 2016 15:35:21 -0700 Subject: [PATCH 047/323] Fix overflow test and stop relying on queue.full() using time.sleep to guess what behavior the thread will have is bad --- analytics/client.py | 10 +++++----- analytics/consumer.py | 5 ++++- analytics/test/client.py | 4 ++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index b9be0ad7..5c5b3f11 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -210,14 +210,14 @@ def _enqueue(self, msg): msg = clean(msg) self.log.debug('queueing: %s', msg) - if self.queue.full(): + try: + self.queue.put(msg, block=False) + self.log.debug('enqueued %s.', msg['type']) + return True, msg + except queue.Full: self.log.warn('analytics-python queue is full') return False, msg - self.queue.put(msg) - self.log.debug('enqueued %s.', msg['type']) - return True, msg - def flush(self): """Forces a flush from the internal queue to the server""" queue = self.queue diff --git a/analytics/consumer.py b/analytics/consumer.py index a54a89cf..f49993a1 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -22,12 +22,15 @@ def __init__(self, queue, write_key, upload_size=100, on_error=None): self.write_key = write_key self.on_error = on_error self.queue = queue + # It's important to set running in the constructor: if we are asked to + # pause immediately after construction, we might set running to True in + # run() *after* we set it to False in pause... and keep running forever. + self.running = True self.retries = 3 def run(self): """Runs the consumer.""" self.log.debug('consumer is running...') - self.running = True while self.running: self.upload() diff --git a/analytics/test/client.py b/analytics/test/client.py index e059604e..5b56df7b 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -219,8 +219,8 @@ def test_flush(self): def test_overflow(self): client = Client('testsecret', max_queue_size=1) - client.consumer.pause() - time.sleep(1.0) # allow time for consumer to exit + # Ensure consumer thread is no longer uploading + client.join() for i in range(10): client.identify('userId') From 74abf911c1d1c7817349d5ca79fe1a5510e6277a Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Mon, 6 Jun 2016 14:33:55 -0700 Subject: [PATCH 048/323] Release 1.2.4 --- HISTORY.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index e8e2b9b1..a0daec6c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,12 @@ +1.2.4 / 2016-06-06 +================== + + * Fix race conditions in overflow and flush tests + * Join daemon thread on interpreter exit to prevetn valueerrors + * Capitalize HISTORY.md (#76) + * Quick fix for Decimal to send as a float + 1.2.3 / 2016-03-23 ================== From c1d244dcd218ba75db9d63f80ee82700681ac7be Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Mon, 6 Jun 2016 14:44:59 -0700 Subject: [PATCH 049/323] increment version --- analytics/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index e6f3c202..69378709 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.3' +VERSION = '1.2.4' From 535405469a86635445ae58d3bc8152675110bd70 Mon Sep 17 00:00:00 2001 From: Bigb Date: Fri, 17 Jun 2016 15:15:15 +0200 Subject: [PATCH 050/323] fix outdated python-dateutil<2 requirement for python2 - dateutil > 2.1 runs is python2 compatible --- setup.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 0f6314af..d0cd5ec2 100644 --- a/setup.py +++ b/setup.py @@ -23,14 +23,10 @@ install_requires = [ "requests>=2.7,<3.0", - "six>=1.5" + "six>=1.5", + "python-dateutil>2.1" ] -extras_require={ - ':python_version in "2.6, 2.7"': ['python-dateutil>=1,<2'], - ':python_version in "3.2, 3.3, 3.4, 3.5"': ['python-dateutil>2'] -} - setup( name='analytics-python', version=VERSION, @@ -43,7 +39,6 @@ packages=['analytics', 'analytics.test'], license='MIT License', install_requires=install_requires, - extras_require=extras_require, description='The hassle-free way to integrate analytics into any python application.', long_description=long_description, classifiers=[ From e506457649e4850ccac7383c6f55d72be6ed9709 Mon Sep 17 00:00:00 2001 From: Ryan Haarmann Date: Wed, 22 Jun 2016 10:40:20 -0500 Subject: [PATCH 051/323] Changed atexit.register when send=False When atexist.register(self.join) is called with send=False, it raises an error since the thread is running. --- analytics/client.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 5c5b3f11..cfcc9769 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -35,18 +35,17 @@ def __init__(self, write_key=None, debug=False, max_queue_size=10000, self.debug = debug self.send = send - # On program exit, allow the consumer thread to exit cleanly. - # This prevents exceptions and a messy shutdown when the interpreter is - # destroyed before the daemon thread finishes execution. However, it - # is *not* the same as flushing the queue! To guarantee all messages - # have been delivered, you'll still need to call flush(). - atexit.register(self.join) - if debug: self.log.setLevel(logging.DEBUG) # if we've disabled sending, just don't start the consumer if send: + # On program exit, allow the consumer thread to exit cleanly. + # This prevents exceptions and a messy shutdown when the interpreter is + # destroyed before the daemon thread finishes execution. However, it + # is *not* the same as flushing the queue! To guarantee all messages + # have been delivered, you'll still need to call flush(). + atexit.register(self.join) self.consumer.start() def identify(self, user_id=None, traits=None, context=None, timestamp=None, From a5b4801deb2bb264a1208c28eed554dc40bb36da Mon Sep 17 00:00:00 2001 From: Di Wu Date: Sun, 26 Jun 2016 23:57:55 -0700 Subject: [PATCH 052/323] Update client.py --- analytics/client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/analytics/client.py b/analytics/client.py index 5c5b3f11..0f3aa5cf 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -229,7 +229,11 @@ def flush(self): def join(self): """Ends the consumer thread once the queue is empty. Blocks execution until finished""" self.consumer.pause() - self.consumer.join() + try: + self.consumer.join() + except RuntimeError: + # consumer thread has not started + pass def require(name, field, data_type): From 4bc9c7ba125b751c709b5c50521ed7142e122261 Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Sat, 2 Jul 2016 14:41:08 -0700 Subject: [PATCH 053/323] increment version to 1.2.5 --- analytics/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 69378709..56d2535d 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.4' +VERSION = '1.2.5' From a74f9bd54596ecbce3c378236164ad19da473bb9 Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Sat, 2 Jul 2016 15:00:58 -0700 Subject: [PATCH 054/323] Release 1.2.5 --- HISTORY.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index a0daec6c..4c210235 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,8 @@ +1.2.5 / 2016-07-02 +================== + + * Fix outdated python-dateutil<2 requirement for python2 - dateutil > 2.1 runs is python2 compatible + * Fix a bug introduced in 1.2.4 where we could try to join a thread that was not yet started 1.2.4 / 2016-06-06 ================== From 2c9c66ddd2cef4da09c3c0b80772890746f74e34 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 29 Nov 2016 15:58:53 -0500 Subject: [PATCH 055/323] drop py32 support --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index be462ddd..365424b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "2.6" - "2.7" - - "3.2" - "3.3" - "3.4" - "3.5" From 77c795cec6aff1e4a287e787241b8487a1b2e3ad Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 29 Nov 2016 15:45:23 -0500 Subject: [PATCH 056/323] dont add messages to the queue if send is false --- analytics/client.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/analytics/client.py b/analytics/client.py index 640be5ae..92c8a25d 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -209,6 +209,10 @@ def _enqueue(self, msg): msg = clean(msg) self.log.debug('queueing: %s', msg) + # if send is False, return msg as if it was successfully queued + if not self.send: + return True, msg + try: self.queue.put(msg, block=False) self.log.debug('enqueued %s.', msg['type']) From e5b52f8c14fc0b20fbfcea2cdd921e9b08b1df0a Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Wed, 7 Dec 2016 21:00:41 -0800 Subject: [PATCH 057/323] increment version to 1.2.6 --- analytics/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/version.py b/analytics/version.py index 56d2535d..89930cee 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.5' +VERSION = '1.2.6' From 2933866e9bd98eeee23d885369529ce59b8ed6a9 Mon Sep 17 00:00:00 2001 From: Joe Gershenson Date: Wed, 7 Dec 2016 21:01:20 -0800 Subject: [PATCH 058/323] Release 1.2.6 --- HISTORY.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 4c210235..4c984365 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,10 @@ + +1.2.6 / 2016-12-07 +================== + + * dont add messages to the queue if send is false + * drop py32 support + 1.2.5 / 2016-07-02 ================== From 3e8a499e721487b1032595c5fe7ed78ccfc37514 Mon Sep 17 00:00:00 2001 From: Hadrien David Date: Mon, 23 Jan 2017 14:15:02 -0500 Subject: [PATCH 059/323] Date objects fail json serialization Fix segmentio/analytics-python#91 --- analytics/request.py | 4 ++-- analytics/test/request.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/analytics/request.py b/analytics/request.py index 59578b4b..1e7d0d6a 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import date, datetime from dateutil.tz import tzutc import logging import json @@ -47,7 +47,7 @@ def __str__(self): class DatetimeSerializer(json.JSONEncoder): def default(self, obj): - if isinstance(obj, datetime): + if isinstance(obj, (date, datetime)): return obj.isoformat() return json.JSONEncoder.default(self, obj) diff --git a/analytics/test/request.py b/analytics/test/request.py index 5d1ce5ee..ff4e8916 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, date import unittest import json @@ -22,3 +22,10 @@ def test_datetime_serialization(self): data = { 'created': datetime(2012, 3, 4, 5, 6, 7, 891011) } result = json.dumps(data, cls=DatetimeSerializer) self.assertEqual(result, '{"created": "2012-03-04T05:06:07.891011"}') + + def test_date_serialization(self): + today = date.today() + data = {'created': today} + result = json.dumps(data, cls=DatetimeSerializer) + expected = '{"created": "%s"}' % today.isoformat() + self.assertEqual(result, expected) From bc0ac9160e04c3128dbdba3c2f82e00c1eb2c1b8 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Tue, 31 Jan 2017 12:04:55 -0800 Subject: [PATCH 060/323] Prepare for release 1.2.7. --- HISTORY.md | 7 ++++++- analytics/version.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 4c984365..b8f7d978 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,9 @@ +1.2.7 / 2017-01-31 +================== + + * [Fix](https://github.com/segmentio/analytics-python/pull/92): Correctly serialize date objects. + 1.2.6 / 2016-12-07 ================== @@ -32,7 +37,7 @@ 1.2.1 / 2016-03-11 ================== - + * fixing requirements.txt 1.2.0 / 2016-03-11 diff --git a/analytics/version.py b/analytics/version.py index 89930cee..9005d975 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.6' +VERSION = '1.2.7' From d4ac5b6ca4e48e6357c48a8b1a0e99066eeb5427 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Tue, 31 Jan 2017 12:15:20 -0800 Subject: [PATCH 061/323] add releasing instructions --- RELEASING.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 RELEASING.md diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 00000000..349b8a0f --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,8 @@ +Releasing +========= + +1. Update `VERSION` in `analytics/version.py` to the new version. +2. Update the `HISTORY.md` for the impending release. +3. `git commit -am "Prepare for release X.Y.Z."` (where X.Y.Z is the new version) +4. `git tag -a X.Y.Z -m "Version X.Y.Z"` (where X.Y.Z is the new version). +5. `make dist`. From fdc178b00cf0b55e5fcd6be0aa4702a47ff7c6b3 Mon Sep 17 00:00:00 2001 From: Hadrien David Date: Mon, 8 May 2017 14:47:44 -0400 Subject: [PATCH 062/323] Fix: utils.clean must not pop date values Fix segmentio/analytics-python#94 --- HISTORY.md | 6 ++++++ analytics/test/utils.py | 9 ++++++++- analytics/utils.py | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index b8f7d978..aea6cd98 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,9 @@ +Next release +============ + + * [Fix](https://github.com/segmentio/analytics-python/issues/94): Date + objects are removed from event properties. + 1.2.7 / 2017-01-31 ================== diff --git a/analytics/test/utils.py b/analytics/test/utils.py index 30eb0bad..bee8c8de 100644 --- a/analytics/test/utils.py +++ b/analytics/test/utils.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from decimal import Decimal import unittest @@ -49,6 +49,13 @@ def test_clean(self): utils.clean(combined) self.assertEqual(combined.keys(), pre_clean_keys) + def test_clean_with_dates(self): + dict_with_dates = { + 'birthdate': date(1980, 1, 1), + 'registration': datetime.utcnow(), + } + self.assertEqual(dict_with_dates, utils.clean(dict_with_dates)) + def test_bytes(self): if six.PY3: item = bytes(10) diff --git a/analytics/utils.py b/analytics/utils.py index 6840f359..89e5dccd 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -1,5 +1,5 @@ from dateutil.tz import tzlocal, tzutc -from datetime import datetime +from datetime import date, datetime from decimal import Decimal import logging import numbers @@ -38,7 +38,7 @@ def clean(item): if isinstance(item, Decimal): return float(item) elif isinstance(item, (six.string_types, bool, numbers.Number, datetime, - type(None))): + date, type(None))): return item elif isinstance(item, (set, list, tuple)): return _clean_list(item) From de7114d077f578ee14aaef6481086235c57ae080 Mon Sep 17 00:00:00 2001 From: Hadrien David Date: Tue, 19 Sep 2017 15:51:48 -0400 Subject: [PATCH 063/323] Add an end to end test case --- analytics/test/client.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 5b56df7b..a1446492 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import date, datetime import unittest import time import six @@ -245,3 +245,17 @@ def test_numeric_user_id(self): def test_debug(self): Client('bad_key', debug=True) + + def test_identify_with_date_object(self): + client = self.client + success, msg = client.identify( + 'userId', + { + 'birthdate': date(1981, 2, 2), + }, + ) + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['traits'], {'birthdate': date(1981, 2, 2)}) From 8c4b39a54075a9ca3dedcf9539099149672a7fdf Mon Sep 17 00:00:00 2001 From: Eirik Martiniussen Sylliaas Date: Wed, 20 Sep 2017 02:26:08 +0200 Subject: [PATCH 064/323] Add a host variable to support a custom domain (#93) * Add a host variable to support a custom domain This makes it possible to use a custom domain or proxy the request. * Rename remove trailing slash function --- analytics/__init__.py | 3 ++- analytics/client.py | 4 ++-- analytics/consumer.py | 5 +++-- analytics/request.py | 6 ++++-- analytics/test/module.py | 6 +++++- analytics/test/request.py | 5 ++++- analytics/test/utils.py | 4 ++++ analytics/utils.py | 5 +++++ 8 files changed, 29 insertions(+), 9 deletions(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 433ab61e..dce8079f 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -6,6 +6,7 @@ """Settings.""" write_key = None +host = None on_error = None debug = False send = True @@ -49,7 +50,7 @@ def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client if not default_client: - default_client = Client(write_key, debug=debug, on_error=on_error, + default_client = Client(write_key, host=host, debug=debug, on_error=on_error, send=send) fn = getattr(default_client, method) diff --git a/analytics/client.py b/analytics/client.py index 92c8a25d..1995ea79 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -24,12 +24,12 @@ class Client(object): """Create a new Segment client.""" log = logging.getLogger('segment') - def __init__(self, write_key=None, debug=False, max_queue_size=10000, + def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, send=True, on_error=None): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) - self.consumer = Consumer(self.queue, write_key, on_error=on_error) + self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error) self.write_key = write_key self.on_error = on_error self.debug = debug diff --git a/analytics/consumer.py b/analytics/consumer.py index f49993a1..c5af74d8 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -13,13 +13,14 @@ class Consumer(Thread): """Consumes the messages from the client's queue.""" log = logging.getLogger('segment') - def __init__(self, queue, write_key, upload_size=100, on_error=None): + def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit self.daemon = True self.upload_size = upload_size self.write_key = write_key + self.host = host self.on_error = on_error self.queue = queue # It's important to set running in the constructor: if we are asked to @@ -78,7 +79,7 @@ def next(self): def request(self, batch, attempt=0): """Attempt to upload the batch and retry before raising an error """ try: - post(self.write_key, batch=batch) + post(self.write_key, self.host, batch=batch) except: if attempt > self.retries: raise diff --git a/analytics/request.py b/analytics/request.py index 1e7d0d6a..384ee68a 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -6,15 +6,17 @@ from requests.auth import HTTPBasicAuth from requests import sessions +from analytics.utils import remove_trailing_slash + _session = sessions.Session() -def post(write_key, **kwargs): +def post(write_key, host=None, **kwargs): """Post the `kwargs` to the API""" log = logging.getLogger('segment') body = kwargs body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat() - url = 'https://api.segment.io/v1/batch' + url = remove_trailing_slash(host or 'https://api.segment.io') + '/v1/batch' auth = HTTPBasicAuth(write_key, '') data = json.dumps(body, cls=DatetimeSerializer) headers = { 'content-type': 'application/json' } diff --git a/analytics/test/module.py b/analytics/test/module.py index 503e9364..8e9c10a3 100644 --- a/analytics/test/module.py +++ b/analytics/test/module.py @@ -17,6 +17,10 @@ def test_no_write_key(self): analytics.write_key = None self.assertRaises(Exception, analytics.track) + def test_no_host(self): + analytics.host = None + self.assertRaises(Exception, analytics.track) + def test_track(self): analytics.track('userId', 'python module event') analytics.flush() @@ -42,4 +46,4 @@ def test_screen(self): analytics.flush() def test_flush(self): - analytics.flush() \ No newline at end of file + analytics.flush() diff --git a/analytics/test/request.py b/analytics/test/request.py index ff4e8916..efc4499a 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -16,7 +16,10 @@ def test_valid_request(self): self.assertEqual(res.status_code, 200) def test_invalid_request_error(self): - self.assertRaises(Exception, post, 'testsecret', '[{]') + self.assertRaises(Exception, post, 'testsecret', 'https://api.segment.io', '[{]') + + def test_invalid_host(self): + self.assertRaises(Exception, post, 'testsecret', 'api.segment.io/', batch=[]) def test_datetime_serialization(self): data = { 'created': datetime(2012, 3, 4, 5, 6, 7, 891011) } diff --git a/analytics/test/utils.py b/analytics/test/utils.py index bee8c8de..09ce0ab3 100644 --- a/analytics/test/utils.py +++ b/analytics/test/utils.py @@ -70,3 +70,7 @@ def test_clean_fn(self): # TODO: fixme, different behavior on python 2 and 3 if 'fn' in cleaned: self.assertEqual(cleaned['fn'], None) + + def test_remove_slash(self): + self.assertEqual('http://segment.io', utils.remove_trailing_slash('http://segment.io/')) + self.assertEqual('http://segment.io', utils.remove_trailing_slash('http://segment.io')) diff --git a/analytics/utils.py b/analytics/utils.py index 89e5dccd..58b9d3a4 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -34,6 +34,11 @@ def guess_timezone(dt): return dt +def remove_trailing_slash(host): + if host.endswith('/'): + return host[:-1] + return host + def clean(item): if isinstance(item, Decimal): return float(item) From e076ccdf3f7c4c39b7aa2b0d40dd13908b636c16 Mon Sep 17 00:00:00 2001 From: Jeff Hu Date: Thu, 21 Sep 2017 00:07:15 -0400 Subject: [PATCH 065/323] fix queue empty bug (#98) --- analytics/consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/consumer.py b/analytics/consumer.py index c5af74d8..ef348eaa 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -67,7 +67,7 @@ def next(self): queue = self.queue items = [] - while len(items) < self.upload_size or self.queue.empty(): + while len(items) < self.upload_size: try: item = queue.get(block=True, timeout=0.5) items.append(item) From f500d62594d19f1a15136e7dc2b9a57241a5c008 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Wed, 20 Sep 2017 21:14:38 -0700 Subject: [PATCH 066/323] Prepare for release 1.2.8. --- HISTORY.md | 10 +++++----- analytics/version.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index aea6cd98..c0138a85 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,8 +1,8 @@ -Next release -============ +1.2.8 / 2017-09-20 +================== - * [Fix](https://github.com/segmentio/analytics-python/issues/94): Date - objects are removed from event properties. + * [Fix](https://github.com/segmentio/analytics-python/issues/94): Date objects are removed from event properties. + * [Fix](https://github.com/segmentio/analytics-python/pull/98): Fix for regression introduced in version 1.2.4. 1.2.7 / 2017-01-31 @@ -26,7 +26,7 @@ Next release ================== * Fix race conditions in overflow and flush tests - * Join daemon thread on interpreter exit to prevetn valueerrors + * Join daemon thread on interpreter exit to prevent value errors * Capitalize HISTORY.md (#76) * Quick fix for Decimal to send as a float diff --git a/analytics/version.py b/analytics/version.py index 9005d975..18434f7c 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.7' +VERSION = '1.2.8' From a32d82ad18ca1ac32590bd1e2c1d58fd84537d43 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Mon, 27 Nov 2017 15:41:07 -0800 Subject: [PATCH 067/323] Stringify userId and anonymousId Ref: https://segment.atlassian.net/browse/LIB-27 Our API truncates bignum values. This is problematic because this library has historically accepted Numbers as userIds and anonymousIds. TTo workaround the truncation, this library needs to stringify userId and anonymousId on the client. The test cases are numbers that will be truncated in Node, e.g. `node -e "console.log(157963456373623802 + 1)"`. --- analytics/client.py | 10 ++++++++++ analytics/test/client.py | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/analytics/client.py b/analytics/client.py index 1995ea79..03e47d7d 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -206,6 +206,9 @@ def _enqueue(self, msg): 'version': VERSION } + msg['userId'] = stringify_id(msg.get('userId', None)) + msg['anonymousId'] = stringify_id(msg.get('anonymousId', None)) + msg = clean(msg) self.log.debug('queueing: %s', msg) @@ -244,3 +247,10 @@ def require(name, field, data_type): if not isinstance(field, data_type): msg = '{0} must have {1}, got: {2}'.format(name, data_type, field) raise AssertionError(msg) + +def stringify_id(val): + if val is None: + return None + if isinstance(val, string_types): + return val + return str(val) diff --git a/analytics/test/client.py b/analytics/test/client.py index a1446492..3a25d227 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -37,6 +37,30 @@ def test_basic_track(self): self.assertEqual(msg['properties'], {}) self.assertEqual(msg['type'], 'track') + def test_stringifies_user_id(self): + # A large number that loses precision in node: + # node -e "console.log(157963456373623802 + 1)" > 157963456373623800 + client = self.client + success, msg = client.track(user_id = 157963456373623802, event = 'python test event') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['userId'], '157963456373623802') + self.assertEqual(msg['anonymousId'], None) + + def test_stringifies_anonymous_id(self): + # A large number that loses precision in node: + # node -e "console.log(157963456373623803 + 1)" > 157963456373623800 + client = self.client + success, msg = client.track(anonymous_id = 157963456373623803, event = 'python test event') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['userId'], None) + self.assertEqual(msg['anonymousId'], '157963456373623803') + def test_advanced_track(self): client = self.client success, msg = client.track( From 8f36191a0d3d4b5452a08775f63c26e862959ba1 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Tue, 28 Nov 2017 16:04:13 -0800 Subject: [PATCH 068/323] Release 1.2.9. --- HISTORY.md | 6 +++++- RELEASING.md | 2 +- analytics/version.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index c0138a85..70d3562f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,10 +1,14 @@ +1.2.9 / 2017-11-28 +================== + + * [Fix](https://github.com/segmentio/analytics-python/pull/102): Stringify non-string userIds and anonymousIds. + 1.2.8 / 2017-09-20 ================== * [Fix](https://github.com/segmentio/analytics-python/issues/94): Date objects are removed from event properties. * [Fix](https://github.com/segmentio/analytics-python/pull/98): Fix for regression introduced in version 1.2.4. - 1.2.7 / 2017-01-31 ================== diff --git a/RELEASING.md b/RELEASING.md index 349b8a0f..94c2da9f 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -3,6 +3,6 @@ Releasing 1. Update `VERSION` in `analytics/version.py` to the new version. 2. Update the `HISTORY.md` for the impending release. -3. `git commit -am "Prepare for release X.Y.Z."` (where X.Y.Z is the new version) +3. `git commit -am "Release X.Y.Z."` (where X.Y.Z is the new version) 4. `git tag -a X.Y.Z -m "Version X.Y.Z"` (where X.Y.Z is the new version). 5. `make dist`. diff --git a/analytics/version.py b/analytics/version.py index 18434f7c..b3951bf9 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.8' +VERSION = '1.2.9' From 3decd60ff1a8e64aefd6aa1ac816d7487ab125a1 Mon Sep 17 00:00:00 2001 From: kevingilliard Date: Tue, 29 May 2018 14:37:57 -0400 Subject: [PATCH 069/323] Implement E2E testing in TravisCI (#106) * Add cmd-line test script for analytics functions * Add e2e test for identify in travisCI * Add e2e testing in TravisCI * Add listing python packages command for debugging purpose * Make simulator.py to Python 2 compatible * Add other e2e test functions * Fix double quote mark in makefile script * Fix script line ending in makefile * Use e2e tester tool https://github.com/segmentio/library-e2e-tester/ * Fix makefile script error * Remove unnecessary line from makefile script * Update path and fix script for e2e-tester * Add logging function into e2e testing script --- .travis.yml | 4 ++- Makefile | 14 +++++++- e2e_test.sh | 5 +++ library-e2e-tester | 1 + simulator.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 e2e_test.sh create mode 160000 library-e2e-tester create mode 100644 simulator.py diff --git a/.travis.yml b/.travis.yml index 365424b9..f44fa3f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ python: - "3.5" install: - "pip install ." -script: make test +script: + - make test + - make e2e_test matrix: fast_finish: true diff --git a/Makefile b/Makefile index 7259d7bf..05ea0539 100644 --- a/Makefile +++ b/Makefile @@ -5,4 +5,16 @@ test: dist: python setup.py sdist bdist_wheel upload -.PHONY: test dist \ No newline at end of file +e2e_test: + if [ "$(RUN_E2E_TESTS)" != "true" ]; then \ + echo "Skipping end to end tests."; \ + else \ + echo "Running end to end tests..."; \ + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.1.1/tester_linux_amd64; \ + chmod +x tester_linux_amd64; \ + chmod +x e2e_test.sh; \ + ./tester_linux_amd64 -segment-write-key="$SEGMENT_WRITE_KEY" -runscope-token="$RUNSCOPE_TOKEN" -runscope-bucket="$RUNSCOPE_BUCKET" -path='./e2e_test.sh'; \ + echo "End to end tests completed!"; \ + fi + +.PHONY: test dist e2e_test \ No newline at end of file diff --git a/e2e_test.sh b/e2e_test.sh new file mode 100644 index 00000000..3828abe2 --- /dev/null +++ b/e2e_test.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -e + +python ./simulator.py "$@" \ No newline at end of file diff --git a/library-e2e-tester b/library-e2e-tester new file mode 160000 index 00000000..5e176853 --- /dev/null +++ b/library-e2e-tester @@ -0,0 +1 @@ +Subproject commit 5e176853ceca14dd656bb8e2187b79adab3216ff diff --git a/simulator.py b/simulator.py new file mode 100644 index 00000000..84a03fe8 --- /dev/null +++ b/simulator.py @@ -0,0 +1,84 @@ +import analytics +import argparse +import json +import logging + +__name__ = 'simulator.py' +__version__ = '0.0.1' +__description__ = 'scripting simulator' + +def json_hash(str): + if str: + return json.loads(str) + +# analytics -method= -segment-write-key= [options] + +parser = argparse.ArgumentParser(description='send a segment message') + +parser.add_argument('--writeKey', help='the Segment writeKey') +parser.add_argument('--type', help='The Segment message type') + +parser.add_argument('--userId', help='the user id to send the event as') +parser.add_argument('--anonymousId', help='the anonymous user id to send the event as') +parser.add_argument('--context', help='additional context for the event (JSON-encoded)') + +parser.add_argument('--event', help='the event name to send with the event') +parser.add_argument('--properties', help='the event properties to send (JSON-encoded)') + +parser.add_argument('--name', help='name of the screen or page to send with the message') + +parser.add_argument('--traits', help='the identify/group traits to send (JSON-encoded)') + +parser.add_argument('--groupId', help='the group id') + +options = parser.parse_args() + +def failed(status, msg): + raise Exception(msg) + +def track(): + analytics.track(options.userId, options.event, anonymous_id = options.anonymousId, + properties = json_hash(options.properties), context = json_hash(options.context)) + +def page(): + analytics.page(options.userId, name = options.name, anonymous_id = options.anonymousId, + properties = json_hash(options.properties), context = json_hash(options.context)) + +def screen(): + analytics.screen(options.userId, name = options.name, anonymous_id = options.anonymousId, + properties = json_hash(options.properties), context = json_hash(options.context)) + +def identify(): + analytics.identify(options.userId, anonymous_id = options.anonymousId, + traits = json_hash(options.traits), context = json_hash(options.context)) + +def group(): + analytics.group(options.userId, options.groupId, json_hash(options.traits), + json_hash(options.context), anonymous_id = options.anonymousId) + +def unknown(): + print() + +analytics.write_key = options.writeKey +analytics.on_error = failed +analytics.debug = True + +log = logging.getLogger('segment') +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +log.addHandler(ch) + +switcher = { + "track": track, + "page": page, + "screen": screen, + "identify": identify, + "group": group +} + +func = switcher.get(options.type) +if func: + func() + analytics.flush() +else: + print("Invalid Message Type " + options.type) From 1d153da55552854363382ee18321c1f6457d7846 Mon Sep 17 00:00:00 2001 From: kevingilliard Date: Tue, 29 May 2018 20:32:47 -0400 Subject: [PATCH 070/323] Library sends a User-Agent header in the form of analytics-python/{library_version} as per RFC 7231. (#107) --- analytics/request.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/analytics/request.py b/analytics/request.py index 384ee68a..2b3284f1 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -2,6 +2,7 @@ from dateutil.tz import tzutc import logging import json +from analytics.version import VERSION from requests.auth import HTTPBasicAuth from requests import sessions @@ -19,7 +20,10 @@ def post(write_key, host=None, **kwargs): url = remove_trailing_slash(host or 'https://api.segment.io') + '/v1/batch' auth = HTTPBasicAuth(write_key, '') data = json.dumps(body, cls=DatetimeSerializer) - headers = { 'content-type': 'application/json' } + headers = { + 'Content-Type': 'application/json', + 'User-Agent': 'analytics-python/' + VERSION + } log.debug('making request: %s', data) res = _session.post(url, data=data, auth=auth, headers=headers, timeout=15) From 45c89bb16c05bf40f03e388732ac7d5769952c02 Mon Sep 17 00:00:00 2001 From: kevingilliard Date: Thu, 7 Jun 2018 20:24:11 -0400 Subject: [PATCH 071/323] CircleCI config (#108) * Rebase to latest version * Update CircleCI config to do tests in various platform * Remove venv config from config.yml * Fix permission problem * Run e2e test in multi platform * Fix bug and add workflow * Update badge to CircleCI --- .circleci/config.yml | 51 ++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..96a5c2bf --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,51 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/python:2.7.15-stretch + steps: + - checkout + - run: pip install --user . + - run: make test + + test_27: + docker: + - image: circleci/python:2.7.15-stretch + steps: + - checkout + - run: pip install --user . + - run: make e2e_test + + test_34: + docker: + - image: circleci/python:3.4.8-jessie-node + steps: + - checkout + - run: pip install --user . + - run: make e2e_test + + test_35: + docker: + - image: circleci/python:3.5.5-jessie + steps: + - checkout + - run: pip install --user . + - run: make e2e_test + + test_36: + docker: + - image: circleci/python:3.6.5-jessie + steps: + - checkout + - run: pip install --user . + - run: make e2e_test + +workflows: + version: 2 + build_and_test: + jobs: + - build + - test_27 + - test_34 + - test_35 + - test_36 \ No newline at end of file diff --git a/README.md b/README.md index 4f3fd9ad..970d6920 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ analytics-python ============== -[![Build Status](https://travis-ci.org/segmentio/analytics-python.svg?branch=master)](https://travis-ci.org/segmentio/analytics-python) +[![Build Status](https://secure.gravatar.com/avatar?s=60)](https://circleci.com/gh/segmentio/analytics-python) analytics-python is a python client for [Segment](https://segment.com) From 0e44344e5dcd662dc01e35f04d0b0dd77917ef71 Mon Sep 17 00:00:00 2001 From: kevingilliard Date: Wed, 13 Jun 2018 16:25:08 -0400 Subject: [PATCH 072/323] Scheduled E2E testing (Every hour) (#109) * Schedule cronjob for hourly e2e testing * Trying to fix trigger error * Try to remove filter for cronjob * Fix schedule error in config.yml * Try to fix schedule error in config.yml (Add indent after schedule) --- .circleci/config.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96a5c2bf..78e0fe57 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,4 +48,18 @@ workflows: - test_27 - test_34 - test_35 - - test_36 \ No newline at end of file + - test_36 + scheduled_e2e_test: + triggers: + - schedule: + cron: "0 * * * *" + filters: + branches: + only: + - master + - scheduled_e2e_testing + jobs: + - test_27 + - test_34 + - test_35 + - test_36 From c4a041619e309d2a1e62a24c21d4cb2c9635c5af Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 22 Aug 2018 12:56:55 -0700 Subject: [PATCH 073/323] Makefile: fix e2e_test target e2e_test should fail according to the test runner's exit code --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 05ea0539..68e6650f 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ e2e_test: if [ "$(RUN_E2E_TESTS)" != "true" ]; then \ echo "Skipping end to end tests."; \ else \ + set -e; \ echo "Running end to end tests..."; \ wget https://github.com/segmentio/library-e2e-tester/releases/download/0.1.1/tester_linux_amd64; \ chmod +x tester_linux_amd64; \ @@ -17,4 +18,4 @@ e2e_test: echo "End to end tests completed!"; \ fi -.PHONY: test dist e2e_test \ No newline at end of file +.PHONY: test dist e2e_test From 0f5398906070a4f4a3eab79d53cd745f3300212b Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 22 Aug 2018 16:14:21 -0700 Subject: [PATCH 074/323] Use webhook instead of runscope for e2e tests --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 68e6650f..334ebebe 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,10 @@ e2e_test: else \ set -e; \ echo "Running end to end tests..."; \ - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.1.1/tester_linux_amd64; \ + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.0/tester_linux_amd64; \ chmod +x tester_linux_amd64; \ chmod +x e2e_test.sh; \ - ./tester_linux_amd64 -segment-write-key="$SEGMENT_WRITE_KEY" -runscope-token="$RUNSCOPE_TOKEN" -runscope-bucket="$RUNSCOPE_BUCKET" -path='./e2e_test.sh'; \ + ./tester_linux_amd64 -segment-write-key="$(SEGMENT_WRITE_KEY)" -webhook-auth-username="$(WEBHOOK_AUTH_USERNAME)" -webhook-bucket="$(WEBHOOK_BUCKET)" -path='./e2e_test.sh'; \ echo "End to end tests completed!"; \ fi From b98c208d3f2d8ef6845f341d99d346a72d3e1416 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 22 Aug 2018 08:40:14 -0700 Subject: [PATCH 075/323] Don't retry sending on client errors except 429 --- analytics/consumer.py | 22 +++++++++++++++++----- e2e_test.sh | 0 2 files changed, 17 insertions(+), 5 deletions(-) mode change 100644 => 100755 e2e_test.sh diff --git a/analytics/consumer.py b/analytics/consumer.py index ef348eaa..79ea1bfc 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -2,7 +2,7 @@ from threading import Thread from analytics.version import VERSION -from analytics.request import post +from analytics.request import post, APIError try: from queue import Empty @@ -80,7 +80,19 @@ def request(self, batch, attempt=0): """Attempt to upload the batch and retry before raising an error """ try: post(self.write_key, self.host, batch=batch) - except: - if attempt > self.retries: - raise - self.request(batch, attempt+1) + except Exception as exc: + def maybe_retry(): + if attempt > self.retries: + raise + self.request(batch, attempt+1) + + if isinstance(exc, APIError): + if exc.status >= 500 or exc.status == 429: + # retry on server errors and client errors with 429 status code (rate limited) + maybe_retry() + elif exc.status >= 400: # don't retry on other client errors + self.log.error('API error: %s', exc) + else: + self.log.debug('Unexpected APIError: %s', exc) + else: # retry on all other errors (eg. network) + maybe_retry() diff --git a/e2e_test.sh b/e2e_test.sh old mode 100644 new mode 100755 From c17fbb5fc606010848d2c62b891be407389f7fba Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 23 Aug 2018 15:57:15 -0700 Subject: [PATCH 076/323] Add unit tests for retry logic --- analytics/test/consumer.py | 46 ++++++++++++++++++++++++++++++++++++++ setup.py | 1 + 2 files changed, 47 insertions(+) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 782d1377..52f927a6 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -1,4 +1,5 @@ import unittest +import mock try: from queue import Queue @@ -6,6 +7,7 @@ from Queue import Queue from analytics.consumer import Consumer +from analytics.request import APIError class TestConsumer(unittest.TestCase): @@ -47,6 +49,50 @@ def test_request(self): } consumer.request([track]) + def _test_request_retry(self, expected_exception, exception_count): + + def mock_post(*args, **kwargs): + mock_post.call_count += 1 + if mock_post.call_count <= exception_count: + raise expected_exception + mock_post.call_count = 0 + + with mock.patch('analytics.consumer.post', mock.Mock(side_effect=mock_post)): + consumer = Consumer(None, 'testsecret') + track = { + 'type': 'track', + 'event': 'python event', + 'userId': 'userId' + } + # request() should succeed if the number of exceptions raised is less + # than the retries paramater. + if exception_count <= consumer.retries: + consumer.request([track]) + else: + # if exceptions are raised more times than the retries parameter, + # we expect the exception to be returned to the caller. + with self.assertRaises(type(expected_exception)) as exc: + consumer.request([track]) + self.assertEqual(exc.exception, expected_exception) + + def test_request_retry(self): + # we should retry on general errors + self._test_request_retry(Exception('generic exception'), 2) + + # we should retry on server errors + self._test_request_retry(APIError(500, 'code', 'Internal Server Error'), 2) + + # we should retry on HTTP 429 errors + self._test_request_retry(APIError(429, 'code', 'Too Many Requests'), 2) + + # we should NOT retry on other client errors + api_error = APIError(400, 'code', 'Client Errors') + with self.assertRaises(APIError) as exc: + self._test_request_retry(api_error, 1) + + # test for number of exceptions raise > retries value + self._test_request_retry(APIError(500, 'code', 'Internal Server Error'), 4) + def test_pause(self): consumer = Consumer(None, 'testsecret') consumer.pause() diff --git a/setup.py b/setup.py index d0cd5ec2..53e2134f 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ install_requires = [ "requests>=2.7,<3.0", "six>=1.5", + "mock>=2.0.0", "python-dateutil>2.1" ] From 7df0a874163703af03de74556a7e6af06a64cd10 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 23 Aug 2018 15:56:23 -0700 Subject: [PATCH 077/323] Fix off-by-one error in retry logic Also raise an exception on client errors --- analytics/consumer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/analytics/consumer.py b/analytics/consumer.py index 79ea1bfc..2fb3c895 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -82,7 +82,7 @@ def request(self, batch, attempt=0): post(self.write_key, self.host, batch=batch) except Exception as exc: def maybe_retry(): - if attempt > self.retries: + if attempt >= self.retries: raise self.request(batch, attempt+1) @@ -92,6 +92,7 @@ def maybe_retry(): maybe_retry() elif exc.status >= 400: # don't retry on other client errors self.log.error('API error: %s', exc) + raise else: self.log.debug('Unexpected APIError: %s', exc) else: # retry on all other errors (eg. network) From 66f2eac793dbe77a8abb8f8e7f9f583176628628 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 24 Aug 2018 10:26:51 -0700 Subject: [PATCH 078/323] Fix unit tests for 2.6 Avoid using assertRaises as a context manager, which was introduced in 2.7. --- analytics/test/consumer.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 52f927a6..d284ccb9 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -71,9 +71,12 @@ def mock_post(*args, **kwargs): else: # if exceptions are raised more times than the retries parameter, # we expect the exception to be returned to the caller. - with self.assertRaises(type(expected_exception)) as exc: + try: consumer.request([track]) - self.assertEqual(exc.exception, expected_exception) + except type(expected_exception) as exc: + self.assertEqual(exc, expected_exception) + else: + self.fail("request() should've raised %s'" % str(type(expected_exception))) def test_request_retry(self): # we should retry on general errors From d73b9c10c5abe132afc3982cfce3441942e9be85 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 24 Aug 2018 10:46:15 -0700 Subject: [PATCH 079/323] More 2.6 fixes (no assertRaises) for unit tests --- analytics/test/consumer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index d284ccb9..e13436f9 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -76,7 +76,7 @@ def mock_post(*args, **kwargs): except type(expected_exception) as exc: self.assertEqual(exc, expected_exception) else: - self.fail("request() should've raised %s'" % str(type(expected_exception))) + self.fail("request() should raise an exception if still failing after %d retries" % consumer.retries) def test_request_retry(self): # we should retry on general errors @@ -90,8 +90,12 @@ def test_request_retry(self): # we should NOT retry on other client errors api_error = APIError(400, 'code', 'Client Errors') - with self.assertRaises(APIError) as exc: + try: self._test_request_retry(api_error, 1) + except APIError: + pass + else: + self.fail('request() should not retry on client errors') # test for number of exceptions raise > retries value self._test_request_retry(APIError(500, 'code', 'Internal Server Error'), 4) From d6631d6250cae38eeec51dbcb0412dccf2fa45a7 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 24 Aug 2018 16:24:44 -0700 Subject: [PATCH 080/323] Run unit tests with coverage --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 334ebebe..ef1471d1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ test: - python setup.py test + coverage run --branch --include=analytics/\* setup.py test dist: python setup.py sdist bdist_wheel upload From 25fef39b47a526e140e0846f1b7db01ffa62d69a Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 24 Aug 2018 16:25:31 -0700 Subject: [PATCH 081/323] Configure CircleCI to report test coverage --- .circleci/config.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78e0fe57..569b6d84 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,24 @@ jobs: steps: - checkout - run: pip install --user . + - run: sudo pip install coverage - run: make test + - persist_to_workspace: + root: . + paths: + - .coverage + + coverage: + docker: + - image: circleci/python:2.7.15-stretch + steps: + - checkout + - attach_workspace: { at: . } + - run: sudo pip install coverage + - run: coverage report --show-missing > /tmp/coverage + - store_artifacts: + path: /tmp/coverage + destination: test-coverage test_27: docker: @@ -49,6 +66,12 @@ workflows: - test_34 - test_35 - test_36 + static_analysis: + jobs: + - build + - coverage: + requires: + - build scheduled_e2e_test: triggers: - schedule: From 210dac0acd9f8bf3a66b255d51226f40394a3b17 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 24 Aug 2018 16:46:21 -0700 Subject: [PATCH 082/323] Update Travis config to report test coverage --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f44fa3f8..3a136d7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,10 @@ python: - "3.5" install: - "pip install ." + - "sudo pip install coverage" script: - make test - make e2e_test + - coverage report --show-missing matrix: fast_finish: true From d983631af22774daab2564b0138c4c6a353d9b1f Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Mon, 27 Aug 2018 10:33:58 -0700 Subject: [PATCH 083/323] Move "mock" depenpendy to tests only --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53e2134f..3441c782 100644 --- a/setup.py +++ b/setup.py @@ -24,10 +24,13 @@ install_requires = [ "requests>=2.7,<3.0", "six>=1.5", - "mock>=2.0.0", "python-dateutil>2.1" ] +tests_require = [ + "mock>=2.0.0" +] + setup( name='analytics-python', version=VERSION, @@ -40,6 +43,7 @@ packages=['analytics', 'analytics.test'], license='MIT License', install_requires=install_requires, + tests_require=tests_require, description='The hassle-free way to integrate analytics into any python application.', long_description=long_description, classifiers=[ From af613a139a331b33c93adaf7680f6850a32fb967 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Mon, 27 Aug 2018 11:05:29 -0700 Subject: [PATCH 084/323] Integrate with codecov --- .circleci/config.yml | 3 ++- .travis.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 569b6d84..420764bf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,8 @@ jobs: - checkout - attach_workspace: { at: . } - run: sudo pip install coverage - - run: coverage report --show-missing > /tmp/coverage + - run: coverage report --show-missing | tee /tmp/coverage + - run: bash <(curl -s https://codecov.io/bash) - store_artifacts: path: /tmp/coverage destination: test-coverage diff --git a/.travis.yml b/.travis.yml index 3a136d7e..3dfc6b47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,5 +12,6 @@ script: - make test - make e2e_test - coverage report --show-missing + - bash <(curl -s https://codecov.io/bash) matrix: fast_finish: true From 84ca04724f968d519d341b3f894a61acd93db450 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Mon, 27 Aug 2018 11:32:24 -0700 Subject: [PATCH 085/323] Don't report coverage for test files --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef1471d1..4c5aaf1d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ test: - coverage run --branch --include=analytics/\* setup.py test + coverage run --branch --include=analytics/\* --omit=*/test* setup.py test dist: python setup.py sdist bdist_wheel upload From 90d9fbe8cfc046449c0ed818ee6d79340123cc94 Mon Sep 17 00:00:00 2001 From: nhi-nguyen Date: Mon, 27 Aug 2018 16:46:30 -0700 Subject: [PATCH 086/323] Rename "dist" target to "release" in Makefile (#118) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4c5aaf1d..28d30de4 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ test: coverage run --branch --include=analytics/\* --omit=*/test* setup.py test -dist: +release: python setup.py sdist bdist_wheel upload e2e_test: From 77e004d439fbdcc3ad4f025b708844816ea4e59c Mon Sep 17 00:00:00 2001 From: nhi-nguyen Date: Tue, 28 Aug 2018 09:26:55 -0700 Subject: [PATCH 087/323] Add linter (#117) * Run pylint in "make test" * Add pylintrc, as generated by pylint --generate-rcfile --- .circleci/config.yml | 2 +- .pylintrc | 549 +++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 2 +- Makefile | 1 + 4 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 .pylintrc diff --git a/.circleci/config.yml b/.circleci/config.yml index 420764bf..fba39d24 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage + - run: sudo pip install coverage pylint==1.9.3 - run: make test - persist_to_workspace: root: . diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..229f0eca --- /dev/null +++ b/.pylintrc @@ -0,0 +1,549 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + non-ascii-bytes-literal, + invalid-unicode-literal, + raw-checker-failed, + bad-inline-option, + locally-disabled, + locally-enabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + deprecated-itertools-function, + deprecated-types-field, + next-method-defined, + dict-items-not-iterating, + dict-keys-not-iterating, + dict-values-not-iterating, + deprecated-operator-function, + deprecated-urllib-function, + xreadlines-attribute, + deprecated-sys-function, + exception-escape, + comprehension-escape + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=optparse.Values,sys.exit + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma, + dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[BASIC] + +# Naming style matching correct argument names +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style +#argument-rgx= + +# Naming style matching correct attribute names +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style +#class-attribute-rgx= + +# Naming style matching correct class names +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming-style +#class-rgx= + +# Naming style matching correct constant names +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma +good-names=i, + j, + k, + ex, + Run, + _ + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming style matching correct inline iteration names +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style +#inlinevar-rgx= + +# Naming style matching correct method names +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style +#method-rgx= + +# Naming style matching correct module names +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style +#variable-rgx= + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub, + TERMIOS, + Bastion, + rexec + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml index 3dfc6b47..75f29279 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: - "3.5" install: - "pip install ." - - "sudo pip install coverage" + - "sudo pip install coverage pylint==1.9.3" script: - make test - make e2e_test diff --git a/Makefile b/Makefile index 28d30de4..ebcb0b5d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ test: + pylint --rcfile=.pylintrc --reports=y --exit-zero analytics coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: From e75afcb7e6ca74f1c7c8c1967b2f9fe85d6ba988 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Tue, 28 Aug 2018 18:38:47 -0700 Subject: [PATCH 088/323] Check codestyle using pycodestyle (aka. pep8) Run pycodestyle as part of build: * fail the build if the newly introduced change result in pycodestyle errors/warnings * run pycodestyle on the whole project and archive the result as a build artifact. --- .circleci/config.yml | 4 +++- .travis.yml | 2 +- Makefile | 4 +++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fba39d24..0890150e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,12 +6,14 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 + - run: sudo pip install coverage pylint==1.9.3 pycodestyle - run: make test - persist_to_workspace: root: . paths: - .coverage + - store_artifacts: + path: pycodestyle.out coverage: docker: diff --git a/.travis.yml b/.travis.yml index 75f29279..b9677a3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ python: - "3.5" install: - "pip install ." - - "sudo pip install coverage pylint==1.9.3" + - "sudo pip install coverage pylint==1.9.3 pycodestyle" script: - make test - make e2e_test diff --git a/Makefile b/Makefile index ebcb0b5d..0783c569 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ - test: pylint --rcfile=.pylintrc --reports=y --exit-zero analytics + # fail on pycodestyle errors on the code change + git diff origin/master..HEAD analytics | pycodestyle --ignore=E501 --diff --statistics --count + pycodestyle --ignore=E501 --statistics analytics > pycodestyle.out || true coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: From 4d9e0606c6602d434a494b6cdf4b4c55d033c69f Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Tue, 28 Aug 2018 18:41:54 -0700 Subject: [PATCH 089/323] Archive pylint output as build artifact --- .circleci/config.yml | 2 ++ Makefile | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0890150e..b3f2e60d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,8 @@ jobs: root: . paths: - .coverage + - store_artifacts: + path: pylint.out - store_artifacts: path: pycodestyle.out diff --git a/Makefile b/Makefile index 0783c569..a90d7e06 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ test: - pylint --rcfile=.pylintrc --reports=y --exit-zero analytics + pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out # fail on pycodestyle errors on the code change git diff origin/master..HEAD analytics | pycodestyle --ignore=E501 --diff --statistics --count pycodestyle --ignore=E501 --statistics analytics > pycodestyle.out || true From a380efbb2e799fb239ba25f49f94a3b90366438d Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 30 Aug 2018 17:33:24 -0700 Subject: [PATCH 090/323] Update "release" target in Makefile and RELEASING.md "make dist" has been replaced by "make release" --- Makefile | 2 +- RELEASING.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a90d7e06..740f6aac 100644 --- a/Makefile +++ b/Makefile @@ -21,4 +21,4 @@ e2e_test: echo "End to end tests completed!"; \ fi -.PHONY: test dist e2e_test +.PHONY: test release e2e_test diff --git a/RELEASING.md b/RELEASING.md index 94c2da9f..9ae22f90 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -5,4 +5,4 @@ Releasing 2. Update the `HISTORY.md` for the impending release. 3. `git commit -am "Release X.Y.Z."` (where X.Y.Z is the new version) 4. `git tag -a X.Y.Z -m "Version X.Y.Z"` (where X.Y.Z is the new version). -5. `make dist`. +5. `make release`. From 6bf0f591229305f27c07d44416d44434cc021fee Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 29 Aug 2018 13:47:38 -0700 Subject: [PATCH 091/323] Allow user-defined upload interval (#120) Upload interval now can be specified as a parameter in Client. --- analytics/client.py | 5 +++-- analytics/consumer.py | 12 +++++++++-- analytics/test/consumer.py | 42 ++++++++++++++++++++++++++++++++++++-- setup.py | 1 + 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 03e47d7d..b4ab65d8 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -25,11 +25,12 @@ class Client(object): log = logging.getLogger('segment') def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, - send=True, on_error=None): + send=True, on_error=None, upload_interval=0.5): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) - self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error) + self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, + upload_interval=upload_interval) self.write_key = write_key self.on_error = on_error self.debug = debug diff --git a/analytics/consumer.py b/analytics/consumer.py index 2fb3c895..a7907a96 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -1,5 +1,6 @@ import logging from threading import Thread +import monotonic from analytics.version import VERSION from analytics.request import post, APIError @@ -13,12 +14,14 @@ class Consumer(Thread): """Consumes the messages from the client's queue.""" log = logging.getLogger('segment') - def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None): + def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, + upload_interval=0.5): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit self.daemon = True self.upload_size = upload_size + self.upload_interval = upload_interval self.write_key = write_key self.host = host self.on_error = on_error @@ -67,9 +70,14 @@ def next(self): queue = self.queue items = [] + start_time = monotonic.monotonic() + while len(items) < self.upload_size: + elapsed = monotonic.monotonic() - start_time + if elapsed >= self.upload_interval: + break try: - item = queue.get(block=True, timeout=0.5) + item = queue.get(block=True, timeout=self.upload_interval - elapsed) items.append(item) except Empty: break diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index e13436f9..8bce0e02 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -1,9 +1,10 @@ import unittest import mock +import time try: from queue import Queue -except: +except ImportError: from Queue import Queue from analytics.consumer import Consumer @@ -30,7 +31,7 @@ def test_next_limit(self): def test_upload(self): q = Queue() - consumer = Consumer(q, 'testsecret') + consumer = Consumer(q, 'testsecret') track = { 'type': 'track', 'event': 'python event', @@ -40,6 +41,43 @@ def test_upload(self): success = consumer.upload() self.assertTrue(success) + def test_upload_interval(self): + # Put _n_ items in the queue, pausing a little bit more than _upload_interval_ + # after each one. The consumer should upload _n_ times. + q = Queue() + upload_interval = 0.3 + consumer = Consumer(q, 'testsecret', upload_size=10, upload_interval=upload_interval) + with mock.patch('analytics.consumer.post') as mock_post: + consumer.start() + for i in range(0, 3): + track = { + 'type': 'track', + 'event': 'python event %d' % i, + 'userId': 'userId' + } + q.put(track) + time.sleep(upload_interval * 1.1) + self.assertEqual(mock_post.call_count, 3) + + def test_multiple_uploads_per_interval(self): + # Put _upload_size*2_ items in the queue at once, then pause for _upload_interval_. + # The consumer should upload 2 times. + q = Queue() + upload_interval = 0.5 + upload_size = 10 + consumer = Consumer(q, 'testsecret', upload_size=upload_size, upload_interval=upload_interval) + with mock.patch('analytics.consumer.post') as mock_post: + consumer.start() + for i in range(0, upload_size * 2): + track = { + 'type': 'track', + 'event': 'python event %d' % i, + 'userId': 'userId' + } + q.put(track) + time.sleep(upload_interval * 1.1) + self.assertEqual(mock_post.call_count, 2) + def test_request(self): consumer = Consumer(None, 'testsecret') track = { diff --git a/setup.py b/setup.py index 3441c782..4b190d82 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ install_requires = [ "requests>=2.7,<3.0", "six>=1.5", + "monotonic>=1.5", "python-dateutil>2.1" ] From c267e414eb9f5bcc75de3f38826f8ae0f7e8959d Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 30 Aug 2018 16:29:14 -0700 Subject: [PATCH 092/323] Implement shutdown function --- analytics/__init__.py | 7 +++++++ analytics/client.py | 5 +++++ analytics/test/client.py | 12 ++++++++++++ 3 files changed, 24 insertions(+) diff --git a/analytics/__init__.py b/analytics/__init__.py index dce8079f..23246a48 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -46,6 +46,13 @@ def join(): """Block program until the client clears the queue""" _proxy('join') + +def shutdown(): + """Flush all messages and cleanly shutdown the client""" + _proxy('flush') + _proxy('join') + + def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client diff --git a/analytics/client.py b/analytics/client.py index b4ab65d8..dd195368 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -242,6 +242,11 @@ def join(self): # consumer thread has not started pass + def shutdown(self): + """Flush all messages and cleanly shutdown the client""" + self.flush() + self.join() + def require(name, field, data_type): """Require that the named `field` has the right `data_type`""" diff --git a/analytics/test/client.py b/analytics/test/client.py index 3a25d227..e2908acb 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -241,6 +241,18 @@ def test_flush(self): # Make sure that the client queue is empty after flushing self.assertTrue(client.queue.empty()) + def test_shutdown(self): + client = self.client + # set up the consumer with more requests than a single batch will allow + for i in range(1000): + success, msg = client.identify('userId', {'trait': 'value'}) + client.shutdown() + # we expect two things after shutdown: + # 1. client queue is empty + # 2. consumer thread has stopped + self.assertTrue(client.queue.empty()) + self.assertFalse(client.consumer.is_alive()) + def test_overflow(self): client = Client('testsecret', max_queue_size=1) # Ensure consumer thread is no longer uploading From 3fc42fdc31780ced7036e0d931bc6d66cd64d713 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 30 Aug 2018 13:20:26 -0700 Subject: [PATCH 093/323] Add snyk scan to build process snyk scans the dependencies in requirements.txt for vulnerabilties. We use setup.py to specify requirements (via "install_requires" parameter), so we need an intermediate step to generate a temporary requirements.txt file for snyk. --- .circleci/config.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3f2e60d..11efbcc8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,17 @@ jobs: path: /tmp/coverage destination: test-coverage + snyk: + docker: + - image: circleci/python:2.7.15-stretch + steps: + - checkout + - attach_workspace: { at: . } + - run: python setup.py egg_info + - run: cp analytics_python.egg-info/requires.txt requirements.txt + - run: pip install --user -r requirements.txt + - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh + test_27: docker: - image: circleci/python:2.7.15-stretch @@ -77,6 +88,10 @@ workflows: - coverage: requires: - build + - snyk: + context: snyk + requires: + - build scheduled_e2e_test: triggers: - schedule: From 7a1d0fea779b6b115ab7ec0523a1ab4d809809e2 Mon Sep 17 00:00:00 2001 From: Prateek Srivastava Date: Tue, 4 Sep 2018 12:21:07 -0700 Subject: [PATCH 094/323] Remove travis config since we use CircleCI now. (#128) CircleCI was added in https://github.com/segmentio/analytics-python/pull/108. We don't need Travis CI anymore. --- .travis.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b9677a3c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python -python: - - "2.6" - - "2.7" - - "3.3" - - "3.4" - - "3.5" -install: - - "pip install ." - - "sudo pip install coverage pylint==1.9.3 pycodestyle" -script: - - make test - - make e2e_test - - coverage report --show-missing - - bash <(curl -s https://codecov.io/bash) -matrix: - fast_finish: true From e5d736cb974dd83ceeb3726de05b1ee4447b5111 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 31 Aug 2018 22:16:31 -0700 Subject: [PATCH 095/323] Add gzip support --- analytics/client.py | 4 ++-- analytics/consumer.py | 5 +++-- analytics/request.py | 19 +++++++++++++++---- analytics/test/client.py | 8 +++++++- analytics/test/request.py | 2 +- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index dd195368..3b7b6fdc 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -25,12 +25,12 @@ class Client(object): log = logging.getLogger('segment') def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, - send=True, on_error=None, upload_interval=0.5): + send=True, on_error=None, upload_interval=0.5, gzip=False): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - upload_interval=upload_interval) + upload_interval=upload_interval, gzip=gzip) self.write_key = write_key self.on_error = on_error self.debug = debug diff --git a/analytics/consumer.py b/analytics/consumer.py index a7907a96..c87df0e3 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -15,7 +15,7 @@ class Consumer(Thread): log = logging.getLogger('segment') def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, - upload_interval=0.5): + upload_interval=0.5, gzip=False): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit @@ -26,6 +26,7 @@ def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, self.host = host self.on_error = on_error self.queue = queue + self.gzip = gzip # It's important to set running in the constructor: if we are asked to # pause immediately after construction, we might set running to True in # run() *after* we set it to False in pause... and keep running forever. @@ -87,7 +88,7 @@ def next(self): def request(self, batch, attempt=0): """Attempt to upload the batch and retry before raising an error """ try: - post(self.write_key, self.host, batch=batch) + post(self.write_key, self.host, gzip=self.gzip, batch=batch) except Exception as exc: def maybe_retry(): if attempt >= self.retries: diff --git a/analytics/request.py b/analytics/request.py index 2b3284f1..47352633 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -2,17 +2,21 @@ from dateutil.tz import tzutc import logging import json -from analytics.version import VERSION - +from gzip import GzipFile from requests.auth import HTTPBasicAuth from requests import sessions +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO +from analytics.version import VERSION from analytics.utils import remove_trailing_slash _session = sessions.Session() -def post(write_key, host=None, **kwargs): +def post(write_key, host=None, gzip=False, **kwargs): """Post the `kwargs` to the API""" log = logging.getLogger('segment') body = kwargs @@ -20,11 +24,18 @@ def post(write_key, host=None, **kwargs): url = remove_trailing_slash(host or 'https://api.segment.io') + '/v1/batch' auth = HTTPBasicAuth(write_key, '') data = json.dumps(body, cls=DatetimeSerializer) + log.debug('making request: %s', data) headers = { 'Content-Type': 'application/json', 'User-Agent': 'analytics-python/' + VERSION } - log.debug('making request: %s', data) + if gzip: + headers['Content-Encoding'] = 'gzip' + buf = StringIO() + with GzipFile(fileobj=buf, mode='w') as gz: + gz.write(data.encode()) + data = buf.getvalue() + res = _session.post(url, data=data, auth=auth, headers=headers, timeout=15) if res.status_code == 200: diff --git a/analytics/test/client.py b/analytics/test/client.py index e2908acb..81fd57c6 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -1,6 +1,5 @@ from datetime import date, datetime import unittest -import time import six from analytics.version import VERSION @@ -295,3 +294,10 @@ def test_identify_with_date_object(self): self.assertFalse(self.failed) self.assertEqual(msg['traits'], {'birthdate': date(1981, 2, 2)}) + + def test_gzip(self): + client = Client('testsecret', on_error=self.fail, gzip=True) + for _ in range(10): + client.identify('userId', {'trait': 'value'}) + client.flush() + self.assertFalse(self.failed) diff --git a/analytics/test/request.py b/analytics/test/request.py index efc4499a..038d423b 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -16,7 +16,7 @@ def test_valid_request(self): self.assertEqual(res.status_code, 200) def test_invalid_request_error(self): - self.assertRaises(Exception, post, 'testsecret', 'https://api.segment.io', '[{]') + self.assertRaises(Exception, post, 'testsecret', 'https://api.segment.io', False, '[{]') def test_invalid_host(self): self.assertRaises(Exception, post, 'testsecret', 'api.segment.io/', batch=[]) From 1380b475c728dc172101ca5bfa3aa0ef43e3415b Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Sun, 2 Sep 2018 14:47:07 -0700 Subject: [PATCH 096/323] Use BytesIO instead of StringIO to write gzip Simplify code by using BytesIO, which is supported in both Python 2 and 3. --- analytics/request.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/analytics/request.py b/analytics/request.py index 47352633..17b982b1 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -5,10 +5,7 @@ from gzip import GzipFile from requests.auth import HTTPBasicAuth from requests import sessions -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from io import BytesIO from analytics.version import VERSION from analytics.utils import remove_trailing_slash @@ -31,9 +28,10 @@ def post(write_key, host=None, gzip=False, **kwargs): } if gzip: headers['Content-Encoding'] = 'gzip' - buf = StringIO() + buf = BytesIO() with GzipFile(fileobj=buf, mode='w') as gz: - gz.write(data.encode()) + # 'data' was produced by json.dumps(), whose default encoding is utf-8. + gz.write(data.encode('utf-8')) data = buf.getvalue() res = _session.post(url, data=data, auth=auth, headers=headers, timeout=15) From efb2a44a6dce517668f6e2541ccc3035d45b366d Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Sun, 2 Sep 2018 14:51:02 -0700 Subject: [PATCH 097/323] Run unit tests in 3.4 build, in addition to 2.7 Run the unit tests in 3.4, to catch obvious 2.7-only code. --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 11efbcc8..dfc044ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,8 @@ jobs: steps: - checkout - run: pip install --user . + - run: sudo pip install coverage pylint==1.9.3 pycodestyle + - run: make test - run: make e2e_test test_35: From 9d3b7b9311b3910a7e6c5a2b1b4802dfd5da93ee Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Tue, 4 Sep 2018 13:01:55 -0700 Subject: [PATCH 098/323] Run unit tests for all Python versions --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfc044ac..1bed3867 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,6 +65,8 @@ jobs: steps: - checkout - run: pip install --user . + - run: sudo pip install coverage pylint==1.9.3 pycodestyle + - run: make test - run: make e2e_test test_36: @@ -73,6 +75,8 @@ jobs: steps: - checkout - run: pip install --user . + - run: sudo pip install coverage pylint==1.9.3 pycodestyle + - run: make test - run: make e2e_test workflows: From 5448538fc5ee2d45ec5ad4324190ccb2ebae5b12 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Tue, 4 Sep 2018 16:53:02 -0700 Subject: [PATCH 099/323] Use library-e2e-tester v0.2.1 to run e2e tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 740f6aac..19e540f5 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ e2e_test: else \ set -e; \ echo "Running end to end tests..."; \ - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.0/tester_linux_amd64; \ + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64; \ chmod +x tester_linux_amd64; \ chmod +x e2e_test.sh; \ ./tester_linux_amd64 -segment-write-key="$(SEGMENT_WRITE_KEY)" -webhook-auth-username="$(WEBHOOK_AUTH_USERNAME)" -webhook-bucket="$(WEBHOOK_BUCKET)" -path='./e2e_test.sh'; \ From 2e281054f37859072221d342ab16af7ce3b2c068 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 5 Sep 2018 09:45:05 -0700 Subject: [PATCH 100/323] Move e2e test script out of Makefile --- .buildscripts/e2e.sh | 16 ++++++++++++++++ Makefile | 12 +----------- 2 files changed, 17 insertions(+), 11 deletions(-) create mode 100755 .buildscripts/e2e.sh diff --git a/.buildscripts/e2e.sh b/.buildscripts/e2e.sh new file mode 100755 index 00000000..0b6763e8 --- /dev/null +++ b/.buildscripts/e2e.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -e + +if [ "${RUN_E2E_TESTS}" != "true" ]; then + echo "Skipping end to end tests." +else + echo "Running end to end tests..." + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64 + chmod +x tester_linux_amd64 + chmod +x e2e_test.sh + ./tester_linux_amd64 -segment-write-key="${SEGMENT_WRITE_KEY}" -webhook-auth-username="${WEBHOOK_AUTH_USERNAME}" -webhook-bucket="${WEBHOOK_BUCKET}" -path='./e2e_test.sh' + echo "End to end tests completed!" +fi + + diff --git a/Makefile b/Makefile index 19e540f5..772fa51a 100644 --- a/Makefile +++ b/Makefile @@ -9,16 +9,6 @@ release: python setup.py sdist bdist_wheel upload e2e_test: - if [ "$(RUN_E2E_TESTS)" != "true" ]; then \ - echo "Skipping end to end tests."; \ - else \ - set -e; \ - echo "Running end to end tests..."; \ - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64; \ - chmod +x tester_linux_amd64; \ - chmod +x e2e_test.sh; \ - ./tester_linux_amd64 -segment-write-key="$(SEGMENT_WRITE_KEY)" -webhook-auth-username="$(WEBHOOK_AUTH_USERNAME)" -webhook-bucket="$(WEBHOOK_BUCKET)" -path='./e2e_test.sh'; \ - echo "End to end tests completed!"; \ - fi + .buildscripts/e2e.sh .PHONY: test release e2e_test From 3cd1630d3bd576a2ad871341d06cbae0c91b9c1a Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 5 Sep 2018 16:17:57 -0700 Subject: [PATCH 101/323] Add exponential backoff with jitter when retrying (#129) * Space out request retries with exponential backoff and full jitter * Change default number of retries from 3 to 10 --- analytics/consumer.py | 40 ++++++++++++++++++-------------------- analytics/test/consumer.py | 18 ++++++++++------- setup.py | 1 + 3 files changed, 31 insertions(+), 28 deletions(-) diff --git a/analytics/consumer.py b/analytics/consumer.py index c87df0e3..86d187d0 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -1,21 +1,23 @@ import logging from threading import Thread import monotonic +import backoff from analytics.version import VERSION from analytics.request import post, APIError try: from queue import Empty -except: +except ImportError: from Queue import Empty + class Consumer(Thread): """Consumes the messages from the client's queue.""" log = logging.getLogger('segment') def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, - upload_interval=0.5, gzip=False): + upload_interval=0.5, gzip=False, retries=10): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit @@ -31,7 +33,7 @@ def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, # pause immediately after construction, we might set running to True in # run() *after* we set it to False in pause... and keep running forever. self.running = True - self.retries = 3 + self.retries = retries def run(self): """Runs the consumer.""" @@ -85,24 +87,20 @@ def next(self): return items - def request(self, batch, attempt=0): + def request(self, batch): """Attempt to upload the batch and retry before raising an error """ - try: - post(self.write_key, self.host, gzip=self.gzip, batch=batch) - except Exception as exc: - def maybe_retry(): - if attempt >= self.retries: - raise - self.request(batch, attempt+1) + def fatal_exception(exc): if isinstance(exc, APIError): - if exc.status >= 500 or exc.status == 429: - # retry on server errors and client errors with 429 status code (rate limited) - maybe_retry() - elif exc.status >= 400: # don't retry on other client errors - self.log.error('API error: %s', exc) - raise - else: - self.log.debug('Unexpected APIError: %s', exc) - else: # retry on all other errors (eg. network) - maybe_retry() + # retry on server errors and client errors with 429 status code (rate limited), + # don't retry on other client errors + return (400 <= exc.status < 500) and exc.status != 429 + else: + # retry on all other errors (eg. network) + return False + + @backoff.on_exception(backoff.expo, Exception, max_tries=self.retries + 1, giveup=fatal_exception) + def send_request(): + post(self.write_key, self.host, gzip=self.gzip, batch=batch) + + send_request() diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 8bce0e02..2991d9e9 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -87,7 +87,7 @@ def test_request(self): } consumer.request([track]) - def _test_request_retry(self, expected_exception, exception_count): + def _test_request_retry(self, consumer, expected_exception, exception_count): def mock_post(*args, **kwargs): mock_post.call_count += 1 @@ -96,7 +96,6 @@ def mock_post(*args, **kwargs): mock_post.call_count = 0 with mock.patch('analytics.consumer.post', mock.Mock(side_effect=mock_post)): - consumer = Consumer(None, 'testsecret') track = { 'type': 'track', 'event': 'python event', @@ -118,25 +117,30 @@ def mock_post(*args, **kwargs): def test_request_retry(self): # we should retry on general errors - self._test_request_retry(Exception('generic exception'), 2) + consumer = Consumer(None, 'testsecret') + self._test_request_retry(consumer, Exception('generic exception'), 2) # we should retry on server errors - self._test_request_retry(APIError(500, 'code', 'Internal Server Error'), 2) + consumer = Consumer(None, 'testsecret') + self._test_request_retry(consumer, APIError(500, 'code', 'Internal Server Error'), 2) # we should retry on HTTP 429 errors - self._test_request_retry(APIError(429, 'code', 'Too Many Requests'), 2) + consumer = Consumer(None, 'testsecret') + self._test_request_retry(consumer, APIError(429, 'code', 'Too Many Requests'), 2) # we should NOT retry on other client errors + consumer = Consumer(None, 'testsecret') api_error = APIError(400, 'code', 'Client Errors') try: - self._test_request_retry(api_error, 1) + self._test_request_retry(consumer, api_error, 1) except APIError: pass else: self.fail('request() should not retry on client errors') # test for number of exceptions raise > retries value - self._test_request_retry(APIError(500, 'code', 'Internal Server Error'), 4) + consumer = Consumer(None, 'testsecret', retries=3) + self._test_request_retry(consumer, APIError(500, 'code', 'Internal Server Error'), 3) def test_pause(self): consumer = Consumer(None, 'testsecret') diff --git a/setup.py b/setup.py index 4b190d82..22e8c66e 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ "requests>=2.7,<3.0", "six>=1.5", "monotonic>=1.5", + "backoff==1.6.0", "python-dateutil>2.1" ] From a41ec7205d1b8f6e184d29f207fb5a963ec61759 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 5 Sep 2018 16:19:30 -0700 Subject: [PATCH 102/323] Add a paramater in Client to configure max retries (#129) --- analytics/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 3b7b6fdc..93298983 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -25,12 +25,12 @@ class Client(object): log = logging.getLogger('segment') def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, - send=True, on_error=None, upload_interval=0.5, gzip=False): + send=True, on_error=None, upload_interval=0.5, gzip=False, max_retries=10): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - upload_interval=upload_interval, gzip=gzip) + upload_interval=upload_interval, gzip=gzip, retries=max_retries) self.write_key = write_key self.on_error = on_error self.debug = debug From 8dacb4c1157164ec65ea6bae6b6eb08f8cc6e15b Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 5 Sep 2018 17:50:00 -0700 Subject: [PATCH 103/323] Limit batch upload size to 500KB (#126) --- analytics/consumer.py | 12 +++++++++++- analytics/test/consumer.py | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/analytics/consumer.py b/analytics/consumer.py index 86d187d0..c5dc77de 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -2,15 +2,20 @@ from threading import Thread import monotonic import backoff +import json from analytics.version import VERSION -from analytics.request import post, APIError +from analytics.request import post, APIError, DatetimeSerializer try: from queue import Empty except ImportError: from Queue import Empty +# Our servers only accept batches less than 500KB. Here limit is set slightly +# lower to leave space for extra data that will be added later, eg. "sentAt". +BATCH_SIZE_LIMIT = 475000 + class Consumer(Thread): """Consumes the messages from the client's queue.""" @@ -74,6 +79,7 @@ def next(self): items = [] start_time = monotonic.monotonic() + total_size = 0 while len(items) < self.upload_size: elapsed = monotonic.monotonic() - start_time @@ -82,6 +88,10 @@ def next(self): try: item = queue.get(block=True, timeout=self.upload_interval - elapsed) items.append(item) + total_size += len(json.dumps(item, cls=DatetimeSerializer).encode()) + if total_size >= BATCH_SIZE_LIMIT: + self.log.debug('hit batch size limit (size: %d)', total_size) + break except Empty: break diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 2991d9e9..4a1fbb28 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -1,6 +1,7 @@ import unittest import mock import time +import json try: from queue import Queue @@ -146,3 +147,27 @@ def test_pause(self): consumer = Consumer(None, 'testsecret') consumer.pause() self.assertFalse(consumer.running) + + def test_max_batch_size(self): + q = Queue() + consumer = Consumer(q, 'testsecret', upload_size=100000, upload_interval=3) + track = { + 'type': 'track', + 'event': 'python event', + 'userId': 'userId' + } + msg_size = len(json.dumps(track)) + n_msgs = 475000 / msg_size # number of messages in a maximum-size batch + + def mock_post_fn(_, data, **kwargs): + res = mock.Mock() + res.status_code = 200 + self.assertTrue(len(data.encode()) < 500000, 'batch size (%d) exceeds 500KB limit' % len(data.encode())) + return res + + with mock.patch('analytics.request._session.post', side_effect=mock_post_fn) as mock_post: + consumer.start() + for _ in range(0, n_msgs + 2): + q.put(track) + q.join() + self.assertEquals(mock_post.call_count, 2) From df91d8862c1509017c54ee9257e53dc7d477d18c Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Wed, 5 Sep 2018 17:32:20 -0700 Subject: [PATCH 104/323] Drop messages greater than 32kb (#126) --- analytics/consumer.py | 8 +++++++- analytics/test/consumer.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/analytics/consumer.py b/analytics/consumer.py index c5dc77de..8ac1cd87 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -12,6 +12,8 @@ except ImportError: from Queue import Empty +MAX_MSG_SIZE = 32 << 10 + # Our servers only accept batches less than 500KB. Here limit is set slightly # lower to leave space for extra data that will be added later, eg. "sentAt". BATCH_SIZE_LIMIT = 475000 @@ -87,8 +89,12 @@ def next(self): break try: item = queue.get(block=True, timeout=self.upload_interval - elapsed) + item_size = len(json.dumps(item, cls=DatetimeSerializer).encode()) + if item_size > MAX_MSG_SIZE: + self.log.error('Item exceeds 32kb limit, dropping. (%s)', str(item)) + continue items.append(item) - total_size += len(json.dumps(item, cls=DatetimeSerializer).encode()) + total_size += item_size if total_size >= BATCH_SIZE_LIMIT: self.log.debug('hit batch size limit (size: %d)', total_size) break diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 4a1fbb28..2acf3019 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -8,7 +8,7 @@ except ImportError: from Queue import Queue -from analytics.consumer import Consumer +from analytics.consumer import Consumer, MAX_MSG_SIZE from analytics.request import APIError @@ -30,6 +30,15 @@ def test_next_limit(self): next = consumer.next() self.assertEqual(next, list(range(upload_size))) + def test_dropping_oversize_msg(self): + q = Queue() + consumer = Consumer(q, '') + oversize_msg = {'m': 'x' * MAX_MSG_SIZE} + q.put(oversize_msg) + next = consumer.next() + self.assertEqual(next, []) + self.assertTrue(q.empty()) + def test_upload(self): q = Queue() consumer = Consumer(q, 'testsecret') From 5878bde7add1c7c3b2d2c66671d65b1e6ae8dfd6 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 6 Sep 2018 09:28:45 -0700 Subject: [PATCH 105/323] Fix unit test for maximum batch upload size --- analytics/test/consumer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 2acf3019..0ef868df 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -165,8 +165,8 @@ def test_max_batch_size(self): 'event': 'python event', 'userId': 'userId' } - msg_size = len(json.dumps(track)) - n_msgs = 475000 / msg_size # number of messages in a maximum-size batch + msg_size = len(json.dumps(track).encode()) + n_msgs = int(475000 / msg_size) # number of messages in a maximum-size batch def mock_post_fn(_, data, **kwargs): res = mock.Mock() From 7ee24f966ce14930bb81e4671d3cc8895c145eec Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 6 Sep 2018 09:27:59 -0700 Subject: [PATCH 106/323] Allow user-defined upload size --- analytics/client.py | 6 ++++-- analytics/test/client.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 93298983..0f198dfc 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -25,12 +25,14 @@ class Client(object): log = logging.getLogger('segment') def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, - send=True, on_error=None, upload_interval=0.5, gzip=False, max_retries=10): + send=True, on_error=None, upload_size=100, upload_interval=0.5, + gzip=False, max_retries=10): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - upload_interval=upload_interval, gzip=gzip, retries=max_retries) + upload_size=upload_size, upload_interval=upload_interval, + gzip=gzip, retries=max_retries) self.write_key = write_key self.on_error = on_error self.debug = debug diff --git a/analytics/test/client.py b/analytics/test/client.py index 81fd57c6..2a65f868 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -1,6 +1,8 @@ from datetime import date, datetime import unittest import six +import mock +import time from analytics.version import VERSION from analytics.client import Client @@ -301,3 +303,18 @@ def test_gzip(self): client.identify('userId', {'trait': 'value'}) client.flush() self.assertFalse(self.failed) + + def test_user_defined_upload_size(self): + client = Client('testsecret', on_error=self.fail, + upload_size=10, upload_interval=3) + + def mock_post_fn(*args, **kwargs): + self.assertEquals(len(kwargs['batch']), 10) + + # the post function should be called 2 times, with a batch size of 10 + # each time. + with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) as mock_post: + for _ in range(20): + client.identify('userId', {'trait': 'value'}) + time.sleep(1) + self.assertEquals(mock_post.call_count, 2) From 9c97346c6692808b930a5ee522d14e142c7c667d Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 6 Sep 2018 06:56:45 -0700 Subject: [PATCH 107/323] Disable some Pylint warnings --- .pylintrc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pylintrc b/.pylintrc index 229f0eca..8f7fd04a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -55,6 +55,11 @@ confidence= # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=print-statement, + invalid-name, + global-statement, + too-many-arguments, + missing-docstring, + too-many-instance-attributes, parameter-unpacking, unpacking-in-except, old-raise-syntax, From d4fe387d5613d50967b936f5eac395c82cc9ee99 Mon Sep 17 00:00:00 2001 From: nhi-nguyen Date: Wed, 12 Sep 2018 10:58:23 -0700 Subject: [PATCH 108/323] Call shutdown at the end of e2e test script (#135) --- simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator.py b/simulator.py index 84a03fe8..d62deaf7 100644 --- a/simulator.py +++ b/simulator.py @@ -79,6 +79,6 @@ def unknown(): func = switcher.get(options.type) if func: func() - analytics.flush() + analytics.shutdown() else: print("Invalid Message Type " + options.type) From efabaad4fd144c08d85445c1112218bdf8357df6 Mon Sep 17 00:00:00 2001 From: nhi-nguyen Date: Thu, 13 Sep 2018 15:38:21 -0700 Subject: [PATCH 109/323] Update e2e tester version to 0.2.2 (#136) --- .buildscripts/e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildscripts/e2e.sh b/.buildscripts/e2e.sh index 0b6763e8..cef1e262 100755 --- a/.buildscripts/e2e.sh +++ b/.buildscripts/e2e.sh @@ -6,7 +6,7 @@ if [ "${RUN_E2E_TESTS}" != "true" ]; then echo "Skipping end to end tests." else echo "Running end to end tests..." - wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.1/tester_linux_amd64 + wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.2/tester_linux_amd64 chmod +x tester_linux_amd64 chmod +x e2e_test.sh ./tester_linux_amd64 -segment-write-key="${SEGMENT_WRITE_KEY}" -webhook-auth-username="${WEBHOOK_AUTH_USERNAME}" -webhook-bucket="${WEBHOOK_BUCKET}" -path='./e2e_test.sh' From 0d6334dd7ac547f02d56cef654b3ba268b2ad5f4 Mon Sep 17 00:00:00 2001 From: nhi-nguyen Date: Tue, 25 Sep 2018 16:50:24 -0700 Subject: [PATCH 110/323] DRY-er CircleCI config file (#137) --- .circleci/config.yml | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1bed3867..9fcb8e1e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: - run: pip install --user . - run: make e2e_test - test_34: + test_34: &test_34 docker: - image: circleci/python:3.4.8-jessie-node steps: @@ -60,24 +60,14 @@ jobs: - run: make e2e_test test_35: + <<: *test_34 docker: - image: circleci/python:3.5.5-jessie - steps: - - checkout - - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 pycodestyle - - run: make test - - run: make e2e_test test_36: + <<: *test_34 docker: - image: circleci/python:3.6.5-jessie - steps: - - checkout - - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 pycodestyle - - run: make test - - run: make e2e_test workflows: version: 2 From d1628a359d3333132e868d1c377b622eb0a5b683 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 5 Oct 2018 12:39:03 -0700 Subject: [PATCH 111/323] Use twine to publish releases --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 772fa51a..7653009b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ test: coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: - python setup.py sdist bdist_wheel upload + python3 setup.py sdist bdist_wheel + twine upload dist/* e2e_test: .buildscripts/e2e.sh From 853cc585612fd81dd2eff76ab936773aa72254ef Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 5 Oct 2018 12:47:20 -0700 Subject: [PATCH 112/323] Automatically publish release whenever a tag is pushed --- .circleci/config.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9fcb8e1e..b3a2c881 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,15 +69,35 @@ jobs: docker: - image: circleci/python:3.6.5-jessie + publish: + docker: + - image: circleci/python:3.6.5-jessie + steps: + - checkout + - run: sudo pip install twine + - run: make release + workflows: version: 2 - build_and_test: + build_test_release: jobs: - build - test_27 - test_34 - test_35 - test_36 + - publish: + requires: + - build + - test_27 + - test_34 + - test_35 + - test_36 + filters: + branches: + ignore: /.*/ + tags: + only: /^\d+\.\d+\.\d+$/ static_analysis: jobs: - build From b3d09f7f16214b3774f291b71badb9005e4c2c02 Mon Sep 17 00:00:00 2001 From: Andrew Thal Date: Tue, 9 Oct 2018 16:39:32 -0400 Subject: [PATCH 113/323] Support custom messageId. (#111) So that a Python client can make use of the deduplication feature in a system that has at-least-once delivery up until the point of calling Segment. Fixes #66 --- analytics/client.py | 31 ++++++++++++++++++++----------- analytics/test/client.py | 22 ++++++++++++---------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 0f198dfc..55d0f5c6 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -52,7 +52,7 @@ def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, self.consumer.start() def identify(self, user_id=None, traits=None, context=None, timestamp=None, - anonymous_id=None, integrations=None): + anonymous_id=None, integrations=None, message_id=None): traits = traits or {} context = context or {} integrations = integrations or {} @@ -66,13 +66,14 @@ def identify(self, user_id=None, traits=None, context=None, timestamp=None, 'context': context, 'type': 'identify', 'userId': user_id, - 'traits': traits + 'traits': traits, + 'messageId': message_id, } return self._enqueue(msg) def track(self, user_id=None, event=None, properties=None, context=None, - timestamp=None, anonymous_id=None, integrations=None): + timestamp=None, anonymous_id=None, integrations=None, message_id=None): properties = properties or {} context = context or {} integrations = integrations or {} @@ -88,13 +89,14 @@ def track(self, user_id=None, event=None, properties=None, context=None, 'context': context, 'userId': user_id, 'type': 'track', - 'event': event + 'event': event, + 'messageId': message_id, } return self._enqueue(msg) def alias(self, previous_id=None, user_id=None, context=None, - timestamp=None, integrations=None): + timestamp=None, integrations=None, message_id=None): context = context or {} integrations = integrations or {} require('previous_id', previous_id, ID_TYPES) @@ -106,13 +108,14 @@ def alias(self, previous_id=None, user_id=None, context=None, 'timestamp': timestamp, 'context': context, 'userId': user_id, - 'type': 'alias' + 'type': 'alias', + 'messageId': message_id, } return self._enqueue(msg) def group(self, user_id=None, group_id=None, traits=None, context=None, - timestamp=None, anonymous_id=None, integrations=None): + timestamp=None, anonymous_id=None, integrations=None, message_id=None): traits = traits or {} context = context or {} integrations = integrations or {} @@ -128,14 +131,15 @@ def group(self, user_id=None, group_id=None, traits=None, context=None, 'context': context, 'userId': user_id, 'traits': traits, - 'type': 'group' + 'type': 'group', + 'messageId': message_id, } return self._enqueue(msg) def page(self, user_id=None, category=None, name=None, properties=None, context=None, timestamp=None, anonymous_id=None, - integrations=None): + integrations=None, message_id=None): properties = properties or {} context = context or {} integrations = integrations or {} @@ -157,13 +161,14 @@ def page(self, user_id=None, category=None, name=None, properties=None, 'userId': user_id, 'type': 'page', 'name': name, + 'messageId': message_id, } return self._enqueue(msg) def screen(self, user_id=None, category=None, name=None, properties=None, context=None, timestamp=None, anonymous_id=None, - integrations=None): + integrations=None, message_id=None): properties = properties or {} context = context or {} integrations = integrations or {} @@ -185,6 +190,7 @@ def screen(self, user_id=None, category=None, name=None, properties=None, 'userId': user_id, 'type': 'screen', 'name': name, + 'messageId': message_id, } return self._enqueue(msg) @@ -194,6 +200,9 @@ def _enqueue(self, msg): timestamp = msg['timestamp'] if timestamp is None: timestamp = datetime.utcnow().replace(tzinfo=tzutc()) + message_id = msg.get('messageId') + if message_id is None: + message_id = uuid4() require('integrations', msg['integrations'], dict) require('type', msg['type'], string_types) @@ -203,7 +212,7 @@ def _enqueue(self, msg): # add common timestamp = guess_timezone(timestamp) msg['timestamp'] = timestamp.isoformat() - msg['messageId'] = str(uuid4()) + msg['messageId'] = stringify_id(message_id) msg['context']['library'] = { 'name': 'analytics-python', 'version': VERSION diff --git a/analytics/test/client.py b/analytics/test/client.py index 2a65f868..9eadbfcc 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -67,7 +67,7 @@ def test_advanced_track(self): success, msg = client.track( 'userId', 'python test event', { 'property': 'value' }, { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', - { 'Amplitude': True }) + { 'Amplitude': True }, 'messageId') self.assertTrue(success) @@ -81,7 +81,7 @@ def test_advanced_track(self): 'name': 'analytics-python', 'version': VERSION }) - self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['messageId'], 'messageId') self.assertEqual(msg['userId'], 'userId') self.assertEqual(msg['type'], 'track') @@ -102,7 +102,8 @@ def test_advanced_identify(self): client = self.client success, msg = client.identify( 'userId', { 'trait': 'value' }, { 'ip': '192.168.0.1' }, - datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }) + datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }, + 'messageId') self.assertTrue(success) @@ -116,7 +117,7 @@ def test_advanced_identify(self): 'version': VERSION }) self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['messageId'], 'messageId') self.assertEqual(msg['userId'], 'userId') self.assertEqual(msg['type'], 'identify') @@ -135,7 +136,8 @@ def test_advanced_group(self): client = self.client success, msg = client.group( 'userId', 'groupId', { 'trait': 'value' }, { 'ip': '192.168.0.1' }, - datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }) + datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }, + 'messageId') self.assertTrue(success) @@ -149,7 +151,7 @@ def test_advanced_group(self): 'version': VERSION }) self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['messageId'], 'messageId') self.assertEqual(msg['userId'], 'userId') self.assertEqual(msg['type'], 'group') @@ -177,7 +179,7 @@ def test_advanced_page(self): success, msg = client.page( 'userId', 'category', 'name', { 'property': 'value' }, { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', - { 'Amplitude': True }) + { 'Amplitude': True }, 'messageId') self.assertTrue(success) @@ -192,7 +194,7 @@ def test_advanced_page(self): }) self.assertEqual(msg['category'], 'category') self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['messageId'], 'messageId') self.assertEqual(msg['userId'], 'userId') self.assertEqual(msg['type'], 'page') self.assertEqual(msg['name'], 'name') @@ -211,7 +213,7 @@ def test_advanced_screen(self): success, msg = client.screen( 'userId', 'category', 'name', { 'property': 'value' }, { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', - { 'Amplitude': True }) + { 'Amplitude': True }, 'messageId') self.assertTrue(success) @@ -225,7 +227,7 @@ def test_advanced_screen(self): 'version': VERSION }) self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['messageId'], 'messageId') self.assertEqual(msg['category'], 'category') self.assertEqual(msg['userId'], 'userId') self.assertEqual(msg['type'], 'screen') From 297f6ef765db74ca214228f2b8d27224d7d98e8b Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Tue, 9 Oct 2018 15:01:13 -0700 Subject: [PATCH 114/323] Fix typo in Makefile Use python instead of python3 to prepare a release, just like before. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7653009b..06c93bf1 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ test: coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: - python3 setup.py sdist bdist_wheel + python setup.py sdist bdist_wheel twine upload dist/* e2e_test: From c1492d395eece093aa53ff34a982eaf2e4e29dec Mon Sep 17 00:00:00 2001 From: nhi-nguyen Date: Tue, 9 Oct 2018 16:24:11 -0700 Subject: [PATCH 115/323] Include pre-release versions in auto-release workflow (#139) --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b3a2c881..2d5d3ba6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,7 +97,7 @@ workflows: branches: ignore: /.*/ tags: - only: /^\d+\.\d+\.\d+$/ + only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. static_analysis: jobs: - build From df1f09dedd7f78e2f24c8391609aecf39bdf2007 Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Thu, 11 Oct 2018 20:36:55 -0700 Subject: [PATCH 116/323] Release 1.3.0b0 --- HISTORY.md | 15 +++++++++++++++ analytics/version.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 70d3562f..bd11606a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,18 @@ +1.3.0 / 2018-10-10 +================== + + * Add User-Agent header to messages + * Don't retry sending on client errors except 429 + * Allow user-defined upload interval + * Add `shutdown` function + * Add gzip support + * Add exponential backoff with jitter when retrying + * Add a paramater in Client to configure max retries + * Limit batch upload size to 500KB + * Drop messages greater than 32kb + * Allow user-defined upload size + * Support custom messageId + 1.2.9 / 2017-11-28 ================== diff --git a/analytics/version.py b/analytics/version.py index b3951bf9..da53dc58 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.2.9' +VERSION = '1.3.0b0' From bd809020baa516f8dc307ef54207c1918a9fdf2a Mon Sep 17 00:00:00 2001 From: Nhi Nguyen Date: Fri, 12 Oct 2018 14:24:44 -0700 Subject: [PATCH 117/323] CircleCI: Add tag filter for publish's upstream jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit “CircleCI does not run workflows for tags unless you explicitly specify tag filters. Additionally, if a job requires any other jobs (directly or indirectly), you must specify tag filters for those jobs." --- .circleci/config.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d5d3ba6..9b56f491 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,8 @@ version: 2 +defaults: + taggedReleasesFilter: &taggedReleasesFilter + tags: + only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. jobs: build: docker: @@ -81,11 +85,21 @@ workflows: version: 2 build_test_release: jobs: - - build - - test_27 - - test_34 - - test_35 - - test_36 + - build: + filters: + <<: *taggedReleasesFilter + - test_27: + filters: + <<: *taggedReleasesFilter + - test_34: + filters: + <<: *taggedReleasesFilter + - test_35: + filters: + <<: *taggedReleasesFilter + - test_36: + filters: + <<: *taggedReleasesFilter - publish: requires: - build @@ -94,10 +108,9 @@ workflows: - test_35 - test_36 filters: + <<: *taggedReleasesFilter branches: ignore: /.*/ - tags: - only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. static_analysis: jobs: - build From 3cbc94072ec08bf8f56eccff0b14c5efe8b3aacb Mon Sep 17 00:00:00 2001 From: William Grosset Date: Thu, 28 Feb 2019 20:44:39 -0800 Subject: [PATCH 118/323] Update README --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 970d6920..6ece2bdb 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,48 @@ analytics-python is a python client for [Segment](https://segment.com) +
+ +

You can't fix what you can't measure

+
+ +Analytics helps you measure your users, product, and business. It unlocks insights into your app's funnel, core business metrics, and whether you have product-market fit. + +## How to get started +1. **Collect analytics data** from your app(s). + - The top 200 Segment companies collect data from 5+ source types (web, mobile, server, CRM, etc.). +2. **Send the data to analytics tools** (for example, Google Analytics, Amplitude, Mixpanel). + - Over 250+ Segment companies send data to eight categories of destinations such as analytics tools, warehouses, email marketing and remarketing systems, session recording, and more. +3. **Explore your data** by creating metrics (for example, new signups, retention cohorts, and revenue generation). + - The best Segment companies use retention cohorts to measure product market fit. Netflix has 70% paid retention after 12 months, 30% after 7 years. + +[Segment](https://segment.com) collects analytics data and allows you to send it to more than 250 apps (such as Google Analytics, Mixpanel, Optimizely, Facebook Ads, Slack, Sentry) just by flipping a switch. You only need one Segment code snippet, and you can turn integrations on and off at will, with no additional code. [Sign up with Segment today](https://app.segment.com/signup). + +### Why? +1. **Power all your analytics apps with the same data**. Instead of writing code to integrate all of your tools individually, send data to Segment, once. + +2. **Install tracking for the last time**. We're the last integration you'll ever need to write. You only need to instrument Segment once. Reduce all of your tracking code and advertising tags into a single set of API calls. + +3. **Send data from anywhere**. Send Segment data from any device, and we'll transform and send it on to any tool. + +4. **Query your data in SQL**. Slice, dice, and analyze your data in detail with Segment SQL. We'll transform and load your customer behavioral data directly from your apps into Amazon Redshift, Google BigQuery, or Postgres. Save weeks of engineering time by not having to invent your own data warehouse and ETL pipeline. + + For example, you can capture data on any app: + ```js + analytics.track('Order Completed', { price: 99.84 }) + ``` + Then, query the resulting data in SQL: + ```sql + select * from app.order_completed + order by price desc + ``` + +### 🚀 Startup Program +
+ +
+If you are part of a new startup (<$5M raised, <2 years since founding), we just launched a new startup program for you. You can get a Segment Team plan (up to $25,000 value in Segment credits) for free up to 2 years — apply here! + ## Documentation Documentation is available at [https://segment.com/libraries/python](https://segment.com/libraries/python). From 2917013778c661d55fb7159300879bfc0305c12d Mon Sep 17 00:00:00 2001 From: Fathy Boundjadj Date: Sun, 28 Apr 2019 04:13:30 +0200 Subject: [PATCH 119/323] Add sync_mode option (#147) Adds a `sync_mode` option, this option prevents the Consumer thread from being created and directly makes a blocking HTTP request every call. Fixes #101. Closes #105, #113. --- analytics/__init__.py | 3 ++- analytics/client.py | 42 ++++++++++++++++++++++++++-------------- analytics/test/client.py | 8 ++++++++ 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 23246a48..d51b45fb 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -10,6 +10,7 @@ on_error = None debug = False send = True +sync_mode = False default_client = None @@ -58,7 +59,7 @@ def _proxy(method, *args, **kwargs): global default_client if not default_client: default_client = Client(write_key, host=host, debug=debug, on_error=on_error, - send=send) + send=send, sync_mode=sync_mode) fn = getattr(default_client, method) fn(*args, **kwargs) diff --git a/analytics/client.py b/analytics/client.py index 55d0f5c6..b95c01de 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -9,6 +9,7 @@ from analytics.utils import guess_timezone, clean from analytics.consumer import Consumer +from analytics.request import post from analytics.version import VERSION try: @@ -26,30 +27,37 @@ class Client(object): def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, send=True, on_error=None, upload_size=100, upload_interval=0.5, - gzip=False, max_retries=10): + gzip=False, max_retries=10, sync_mode=False): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) - self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - upload_size=upload_size, upload_interval=upload_interval, - gzip=gzip, retries=max_retries) self.write_key = write_key self.on_error = on_error self.debug = debug self.send = send + self.sync_mode = sync_mode + self.host = host + self.gzip = gzip if debug: self.log.setLevel(logging.DEBUG) - # if we've disabled sending, just don't start the consumer - if send: - # On program exit, allow the consumer thread to exit cleanly. - # This prevents exceptions and a messy shutdown when the interpreter is - # destroyed before the daemon thread finishes execution. However, it - # is *not* the same as flushing the queue! To guarantee all messages - # have been delivered, you'll still need to call flush(). - atexit.register(self.join) - self.consumer.start() + if sync_mode: + self.consumer = None + else: + self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, + upload_size=upload_size, upload_interval=upload_interval, + gzip=gzip, retries=max_retries) + + # if we've disabled sending, just don't start the consumer + if send: + # On program exit, allow the consumer thread to exit cleanly. + # This prevents exceptions and a messy shutdown when the interpreter is + # destroyed before the daemon thread finishes execution. However, it + # is *not* the same as flushing the queue! To guarantee all messages + # have been delivered, you'll still need to call flush(). + atexit.register(self.join) + self.consumer.start() def identify(self, user_id=None, traits=None, context=None, timestamp=None, anonymous_id=None, integrations=None, message_id=None): @@ -228,12 +236,18 @@ def _enqueue(self, msg): if not self.send: return True, msg + if self.sync_mode: + self.log.debug('enqueued with blocking %s.', msg['type']) + post(self.write_key, self.host, gzip=self.gzip, batch=[msg]) + + return True, msg + try: self.queue.put(msg, block=False) self.log.debug('enqueued %s.', msg['type']) return True, msg except queue.Full: - self.log.warn('analytics-python queue is full') + self.log.warning('analytics-python queue is full') return False, msg def flush(self): diff --git a/analytics/test/client.py b/analytics/test/client.py index 9eadbfcc..d63884e2 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -256,6 +256,14 @@ def test_shutdown(self): self.assertTrue(client.queue.empty()) self.assertFalse(client.consumer.is_alive()) + def test_synchronous(self): + client = Client('testsecret', sync_mode=True) + + success, message = client.identify('userId') + self.assertIsNone(client.consumer) + self.assertTrue(client.queue.empty()) + self.assertTrue(success) + def test_overflow(self): client = Client('testsecret', max_queue_size=1) # Ensure consumer thread is no longer uploading From 7b078c5792b8ff5248edf4a02cf44b39c0384934 Mon Sep 17 00:00:00 2001 From: Fathy Boundjadj Date: Sun, 28 Apr 2019 04:16:43 +0200 Subject: [PATCH 120/323] Release 1.3.0b1 --- HISTORY.md | 1 + analytics/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index bd11606a..776e9f71 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -12,6 +12,7 @@ * Drop messages greater than 32kb * Allow user-defined upload size * Support custom messageId + * Add `sync_mode` option ([#147](https://github.com/segmentio/analytics-python/pull/147)) 1.2.9 / 2017-11-28 ================== diff --git a/analytics/version.py b/analytics/version.py index da53dc58..e91d81c9 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.3.0b0' +VERSION = '1.3.0b1' From 223561499b5c6b01e2877c7b46711e8aa4ea09ba Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 12:58:17 -0700 Subject: [PATCH 121/323] add configurable timeout option --- analytics/client.py | 7 ++++--- analytics/consumer.py | 5 +++-- analytics/request.py | 9 +++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index b95c01de..6ea9fda0 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -27,7 +27,7 @@ class Client(object): def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, send=True, on_error=None, upload_size=100, upload_interval=0.5, - gzip=False, max_retries=10, sync_mode=False): + gzip=False, max_retries=10, sync_mode=False, timeout=15): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) @@ -38,6 +38,7 @@ def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, self.sync_mode = sync_mode self.host = host self.gzip = gzip + self.timeout = timeout if debug: self.log.setLevel(logging.DEBUG) @@ -47,7 +48,7 @@ def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, else: self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, upload_size=upload_size, upload_interval=upload_interval, - gzip=gzip, retries=max_retries) + gzip=gzip, retries=max_retries, timeout=timeout) # if we've disabled sending, just don't start the consumer if send: @@ -238,7 +239,7 @@ def _enqueue(self, msg): if self.sync_mode: self.log.debug('enqueued with blocking %s.', msg['type']) - post(self.write_key, self.host, gzip=self.gzip, batch=[msg]) + post(self.write_key, self.host, gzip=self.gzip, timeout=self.timeout, batch=[msg]) return True, msg diff --git a/analytics/consumer.py b/analytics/consumer.py index 8ac1cd87..ae6e3c92 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -24,7 +24,7 @@ class Consumer(Thread): log = logging.getLogger('segment') def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, - upload_interval=0.5, gzip=False, retries=10): + upload_interval=0.5, gzip=False, retries=10, timeout=15): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit @@ -41,6 +41,7 @@ def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, # run() *after* we set it to False in pause... and keep running forever. self.running = True self.retries = retries + self.timeout = timeout def run(self): """Runs the consumer.""" @@ -117,6 +118,6 @@ def fatal_exception(exc): @backoff.on_exception(backoff.expo, Exception, max_tries=self.retries + 1, giveup=fatal_exception) def send_request(): - post(self.write_key, self.host, gzip=self.gzip, batch=batch) + post(self.write_key, self.host, gzip=self.gzip, timeout=self.timeout, batch=batch) send_request() diff --git a/analytics/request.py b/analytics/request.py index 17b982b1..ff5736d5 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -6,6 +6,7 @@ from requests.auth import HTTPBasicAuth from requests import sessions from io import BytesIO +import sys from analytics.version import VERSION from analytics.utils import remove_trailing_slash @@ -13,7 +14,7 @@ _session = sessions.Session() -def post(write_key, host=None, gzip=False, **kwargs): +def post(write_key, host=None, gzip=False, timeout=15, **kwargs): """Post the `kwargs` to the API""" log = logging.getLogger('segment') body = kwargs @@ -34,7 +35,11 @@ def post(write_key, host=None, gzip=False, **kwargs): gz.write(data.encode('utf-8')) data = buf.getvalue() - res = _session.post(url, data=data, auth=auth, headers=headers, timeout=15) + try: + res = _session.post(url, data=data, auth=auth, headers=headers, timeout=timeout) + except: + print("timedout") + #sys.exit() if res.status_code == 200: log.debug('data uploaded successfully') From 81ae83ded4daf55288c49e129fcedbf16257feb0 Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 13:00:14 -0700 Subject: [PATCH 122/323] take out debugging code --- analytics/request.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/analytics/request.py b/analytics/request.py index ff5736d5..ab9ea280 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -35,11 +35,7 @@ def post(write_key, host=None, gzip=False, timeout=15, **kwargs): gz.write(data.encode('utf-8')) data = buf.getvalue() - try: - res = _session.post(url, data=data, auth=auth, headers=headers, timeout=timeout) - except: - print("timedout") - #sys.exit() + res = _session.post(url, data=data, auth=auth, headers=headers, timeout=timeout) if res.status_code == 200: log.debug('data uploaded successfully') From c7be9bf1e8a960e850a01473163fd01a01052391 Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 13:08:08 -0700 Subject: [PATCH 123/323] take out unused import --- analytics/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/analytics/request.py b/analytics/request.py index ab9ea280..eb7f825d 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -6,7 +6,6 @@ from requests.auth import HTTPBasicAuth from requests import sessions from io import BytesIO -import sys from analytics.version import VERSION from analytics.utils import remove_trailing_slash From 033bfe35c9fd3ed74b87105fde9b814f581f6dba Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 14:32:18 -0700 Subject: [PATCH 124/323] adds unit tests --- analytics/test/client.py | 4 ++++ analytics/test/request.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/analytics/test/client.py b/analytics/test/client.py index d63884e2..7e013d4d 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -328,3 +328,7 @@ def mock_post_fn(*args, **kwargs): client.identify('userId', {'trait': 'value'}) time.sleep(1) self.assertEquals(mock_post.call_count, 2) + + def test_should_set_timeout(self): + client = Client('testsecret', timeout=10) + self.assertEquals(client.consumer.timeout, 10) \ No newline at end of file diff --git a/analytics/test/request.py b/analytics/test/request.py index 038d423b..4776b26c 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -1,6 +1,7 @@ from datetime import datetime, date import unittest import json +import requests from analytics.request import post, DatetimeSerializer @@ -32,3 +33,20 @@ def test_date_serialization(self): result = json.dumps(data, cls=DatetimeSerializer) expected = '{"created": "%s"}' % today.isoformat() self.assertEqual(result, expected) + + def test_should_not_timeout(self): + res = post('testsecret', batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track' + }], timeout=15) + self.assertEqual(res.status_code, 200) + + def test_should_timeout(self): + with self.assertRaises(requests.ReadTimeout): + res = post('testsecret', batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track' + }], timeout=0.0001) + \ No newline at end of file From 50bf4a139585b4c648d60c4ac8916aeb65e4a358 Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 14:38:12 -0700 Subject: [PATCH 125/323] test changes --- analytics/test/client.py | 8 ++++++-- analytics/test/request.py | 1 - 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 7e013d4d..45259d60 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -329,6 +329,10 @@ def mock_post_fn(*args, **kwargs): time.sleep(1) self.assertEquals(mock_post.call_count, 2) - def test_should_set_timeout(self): + def test_user_defined_timeout(self): client = Client('testsecret', timeout=10) - self.assertEquals(client.consumer.timeout, 10) \ No newline at end of file + self.assertEquals(client.consumer.timeout, 10) + + def test_default_timeout_15(self): + client = Client('testsecret') + self.assertEquals(client.consumer.timeout, 15) \ No newline at end of file diff --git a/analytics/test/request.py b/analytics/test/request.py index 4776b26c..9ff74e3e 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -49,4 +49,3 @@ def test_should_timeout(self): 'event': 'python event', 'type': 'track' }], timeout=0.0001) - \ No newline at end of file From ae1a49e90c7beaa3c286a4b02eb7f2d59c6f3f23 Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 14:39:46 -0700 Subject: [PATCH 126/323] test changes again --- analytics/test/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 45259d60..1850fd6f 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -335,4 +335,4 @@ def test_user_defined_timeout(self): def test_default_timeout_15(self): client = Client('testsecret') - self.assertEquals(client.consumer.timeout, 15) \ No newline at end of file + self.assertEquals(client.consumer.timeout, 15) From 05b9f90a25dc3fe1efa2d6e84d14cf4da136bc81 Mon Sep 17 00:00:00 2001 From: Daniel Jackins Date: Mon, 16 Sep 2019 14:42:18 -0700 Subject: [PATCH 127/323] fix test whitespace issue --- analytics/test/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 1850fd6f..dec60958 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -332,7 +332,7 @@ def mock_post_fn(*args, **kwargs): def test_user_defined_timeout(self): client = Client('testsecret', timeout=10) self.assertEquals(client.consumer.timeout, 10) - + def test_default_timeout_15(self): client = Client('testsecret') self.assertEquals(client.consumer.timeout, 15) From 2e88399ee117ea3d282cf9a1184457f08a6d4822 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Wed, 27 Nov 2019 16:57:24 -0600 Subject: [PATCH 128/323] Add CircleCI status badge to readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 6ece2bdb..d9d9066c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ analytics-python ============== -[![Build Status](https://secure.gravatar.com/avatar?s=60)](https://circleci.com/gh/segmentio/analytics-python) +[![Build Status](https://circleci.com/gh/segmentio/analytics-python.svg?style=svg)](https://circleci.com/gh/segmentio/analytics-python) analytics-python is a python client for [Segment](https://segment.com) @@ -75,4 +75,3 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - From 22090c7aa1477bc0164ff2de92cc7d3d90ac0706 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Fri, 29 Nov 2019 11:44:24 -0600 Subject: [PATCH 129/323] Run tests for latest python versions Run tests on Python 3.7 and 3.8 --- .circleci/config.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b56f491..0423a597 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,6 +73,16 @@ jobs: docker: - image: circleci/python:3.6.5-jessie + test_37: + <<: *test_34 + docker: + - image: python:3.7-alpine + + test_38: + <<: *test_34 + docker: + - image: python:3.8-alpine + publish: docker: - image: circleci/python:3.6.5-jessie @@ -100,6 +110,12 @@ workflows: - test_36: filters: <<: *taggedReleasesFilter + - test_37: + filters: + <<: *taggedReleasesFilter + - test_38: + filters: + <<: *taggedReleasesFilter - publish: requires: - build @@ -107,6 +123,8 @@ workflows: - test_34 - test_35 - test_36 + - test_37 + - test_38 filters: <<: *taggedReleasesFilter branches: From 42e6968bb0a6880c87de55ccf92993593e1afb7b Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Fri, 29 Nov 2019 15:01:40 -0600 Subject: [PATCH 130/323] Add prepare test job Fix test config --- .circleci/config.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0423a597..7530c86e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,13 +73,19 @@ jobs: docker: - image: circleci/python:3.6.5-jessie - test_37: - <<: *test_34 + test_37: &test_37 docker: - image: python:3.7-alpine + steps: + - checkout + - run: apk add make + - run: pip install --user . + - run: pip install coverage pylint==1.9.3 pycodestyle + - run: make test + - run: make e2e_test test_38: - <<: *test_34 + <<: *test_37 docker: - image: python:3.8-alpine From 140553200b76dddc4b52da693db04899c9ccf345 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Mon, 2 Dec 2019 09:11:55 -0600 Subject: [PATCH 131/323] Use debian images for testing --- .circleci/config.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7530c86e..6830c77a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,21 +73,15 @@ jobs: docker: - image: circleci/python:3.6.5-jessie - test_37: &test_37 + test_37: + <<: *test_34 docker: - - image: python:3.7-alpine - steps: - - checkout - - run: apk add make - - run: pip install --user . - - run: pip install coverage pylint==1.9.3 pycodestyle - - run: make test - - run: make e2e_test + - image: circleci/python:3.7-stretch test_38: - <<: *test_37 + <<: *test_34 docker: - - image: python:3.8-alpine + - image: circleci/python:3.8-buster publish: docker: From 216d183b6c99c82164d21ba85506869e8385add8 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Mon, 2 Dec 2019 21:15:00 -0600 Subject: [PATCH 132/323] Run tests for Python 2.7 --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6830c77a..3f77fa54 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,6 +51,8 @@ jobs: steps: - checkout - run: pip install --user . + - run: sudo pip install setuptools coverage pylint==1.9.3 pycodestyle + - run: make test - run: make e2e_test test_34: &test_34 From 2dc53a60e74ab13ecdded622572c782c5c97ab3f Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Tue, 3 Dec 2019 09:35:20 -0600 Subject: [PATCH 133/323] Fix CircleCI badge alt property --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d9d9066c..96a50c48 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ analytics-python ============== -[![Build Status](https://circleci.com/gh/segmentio/analytics-python.svg?style=svg)](https://circleci.com/gh/segmentio/analytics-python) +[![CircleCI](https://circleci.com/gh/segmentio/analytics-python.svg?style=svg)](https://circleci.com/gh/segmentio/analytics-python) analytics-python is a python client for [Segment](https://segment.com) From 65531b376be90940bce379274d77eb0a810b052d Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Tue, 3 Dec 2019 12:01:58 -0600 Subject: [PATCH 134/323] Add supported Python versions to setup.py --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 22e8c66e..4643f3ad 100644 --- a/setup.py +++ b/setup.py @@ -62,5 +62,8 @@ "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], ) From c48c44b50ff9df005d39d4a96dfa0c9c51910857 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Wed, 4 Dec 2019 14:49:50 -0600 Subject: [PATCH 135/323] Standardize parameters for client and consumer --- analytics/client.py | 13 ++++++++----- analytics/consumer.py | 27 ++++++++++++++++----------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 6ea9fda0..eed2525c 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -25,9 +25,10 @@ class Client(object): """Create a new Segment client.""" log = logging.getLogger('segment') - def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, - send=True, on_error=None, upload_size=100, upload_interval=0.5, - gzip=False, max_retries=10, sync_mode=False, timeout=15): + def __init__(self, write_key=None, host=None, debug=False, + max_queue_size=10000, send=True, on_error=None, flush_at=100, + flush_interval=0.5, gzip=False, max_retries=3, + sync_mode=False, timeout=15): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) @@ -47,7 +48,7 @@ def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, self.consumer = None else: self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - upload_size=upload_size, upload_interval=upload_interval, + flush_at=flush_at, flush_interval=flush_interval, gzip=gzip, retries=max_retries, timeout=timeout) # if we've disabled sending, just don't start the consumer @@ -239,7 +240,8 @@ def _enqueue(self, msg): if self.sync_mode: self.log.debug('enqueued with blocking %s.', msg['type']) - post(self.write_key, self.host, gzip=self.gzip, timeout=self.timeout, batch=[msg]) + post(self.write_key, self.host, gzip=self.gzip, + timeout=self.timeout, batch=[msg]) return True, msg @@ -280,6 +282,7 @@ def require(name, field, data_type): msg = '{0} must have {1}, got: {2}'.format(name, data_type, field) raise AssertionError(msg) + def stringify_id(val): if val is None: return None diff --git a/analytics/consumer.py b/analytics/consumer.py index ae6e3c92..0aeab3dc 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -23,14 +23,14 @@ class Consumer(Thread): """Consumes the messages from the client's queue.""" log = logging.getLogger('segment') - def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, - upload_interval=0.5, gzip=False, retries=10, timeout=15): + def __init__(self, queue, write_key, flush_at=100, host=None, on_error=None, + flush_interval=0.5, gzip=False, retries=10, timeout=15): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit self.daemon = True - self.upload_size = upload_size - self.upload_interval = upload_interval + self.flush_at = flush_at + self.flush_interval = flush_interval self.write_key = write_key self.host = host self.on_error = on_error @@ -84,20 +84,24 @@ def next(self): start_time = monotonic.monotonic() total_size = 0 - while len(items) < self.upload_size: + while len(items) < self.flush_at: elapsed = monotonic.monotonic() - start_time - if elapsed >= self.upload_interval: + if elapsed >= self.flush_interval: break try: - item = queue.get(block=True, timeout=self.upload_interval - elapsed) - item_size = len(json.dumps(item, cls=DatetimeSerializer).encode()) + item = queue.get( + block=True, timeout=self.flush_interval - elapsed) + item_size = len(json.dumps( + item, cls=DatetimeSerializer).encode()) if item_size > MAX_MSG_SIZE: - self.log.error('Item exceeds 32kb limit, dropping. (%s)', str(item)) + self.log.error( + 'Item exceeds 32kb limit, dropping. (%s)', str(item)) continue items.append(item) total_size += item_size if total_size >= BATCH_SIZE_LIMIT: - self.log.debug('hit batch size limit (size: %d)', total_size) + self.log.debug( + 'hit batch size limit (size: %d)', total_size) break except Empty: break @@ -118,6 +122,7 @@ def fatal_exception(exc): @backoff.on_exception(backoff.expo, Exception, max_tries=self.retries + 1, giveup=fatal_exception) def send_request(): - post(self.write_key, self.host, gzip=self.gzip, timeout=self.timeout, batch=batch) + post(self.write_key, self.host, gzip=self.gzip, + timeout=self.timeout, batch=batch) send_request() From 25d9421947d82de83376f83e769149866249d95e Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Wed, 4 Dec 2019 14:52:37 -0600 Subject: [PATCH 136/323] Update tests for renamed parameters --- analytics/test/client.py | 64 ++++++++++++++++++++------------------ analytics/test/consumer.py | 51 +++++++++++++++++------------- 2 files changed, 63 insertions(+), 52 deletions(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index dec60958..4c648390 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -42,7 +42,8 @@ def test_stringifies_user_id(self): # A large number that loses precision in node: # node -e "console.log(157963456373623802 + 1)" > 157963456373623800 client = self.client - success, msg = client.track(user_id = 157963456373623802, event = 'python test event') + success, msg = client.track( + user_id=157963456373623802, event='python test event') client.flush() self.assertTrue(success) self.assertFalse(self.failed) @@ -54,7 +55,8 @@ def test_stringifies_anonymous_id(self): # A large number that loses precision in node: # node -e "console.log(157963456373623803 + 1)" > 157963456373623800 client = self.client - success, msg = client.track(anonymous_id = 157963456373623803, event = 'python test event') + success, msg = client.track( + anonymous_id=157963456373623803, event='python test event') client.flush() self.assertTrue(success) self.assertFalse(self.failed) @@ -65,15 +67,15 @@ def test_stringifies_anonymous_id(self): def test_advanced_track(self): client = self.client success, msg = client.track( - 'userId', 'python test event', { 'property': 'value' }, - { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', - { 'Amplitude': True }, 'messageId') + 'userId', 'python test event', {'property': 'value'}, + {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', + {'Amplitude': True}, 'messageId') self.assertTrue(success) self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['properties'], { 'property': 'value' }) - self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['properties'], {'property': 'value'}) + self.assertEqual(msg['integrations'], {'Amplitude': True}) self.assertEqual(msg['context']['ip'], '192.168.0.1') self.assertEqual(msg['event'], 'python test event') self.assertEqual(msg['anonymousId'], 'anonymousId') @@ -87,12 +89,12 @@ def test_advanced_track(self): def test_basic_identify(self): client = self.client - success, msg = client.identify('userId', { 'trait': 'value' }) + success, msg = client.identify('userId', {'trait': 'value'}) client.flush() self.assertTrue(success) self.assertFalse(self.failed) - self.assertEqual(msg['traits'], { 'trait': 'value' }) + self.assertEqual(msg['traits'], {'trait': 'value'}) self.assertTrue(isinstance(msg['timestamp'], str)) self.assertTrue(isinstance(msg['messageId'], str)) self.assertEqual(msg['userId'], 'userId') @@ -101,16 +103,16 @@ def test_basic_identify(self): def test_advanced_identify(self): client = self.client success, msg = client.identify( - 'userId', { 'trait': 'value' }, { 'ip': '192.168.0.1' }, - datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }, + 'userId', {'trait': 'value'}, {'ip': '192.168.0.1'}, + datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, 'messageId') self.assertTrue(success) self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['integrations'], {'Amplitude': True}) self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['traits'], { 'trait': 'value' }) + self.assertEqual(msg['traits'], {'trait': 'value'}) self.assertEqual(msg['anonymousId'], 'anonymousId') self.assertEqual(msg['context']['library'], { 'name': 'analytics-python', @@ -135,16 +137,16 @@ def test_basic_group(self): def test_advanced_group(self): client = self.client success, msg = client.group( - 'userId', 'groupId', { 'trait': 'value' }, { 'ip': '192.168.0.1' }, - datetime(2014, 9, 3), 'anonymousId', { 'Amplitude': True }, + 'userId', 'groupId', {'trait': 'value'}, {'ip': '192.168.0.1'}, + datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, 'messageId') self.assertTrue(success) self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['integrations'], {'Amplitude': True}) self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['traits'], { 'trait': 'value' }) + self.assertEqual(msg['traits'], {'trait': 'value'}) self.assertEqual(msg['anonymousId'], 'anonymousId') self.assertEqual(msg['context']['library'], { 'name': 'analytics-python', @@ -177,16 +179,16 @@ def test_basic_page(self): def test_advanced_page(self): client = self.client success, msg = client.page( - 'userId', 'category', 'name', { 'property': 'value' }, - { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', - { 'Amplitude': True }, 'messageId') + 'userId', 'category', 'name', {'property': 'value'}, + {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', + {'Amplitude': True}, 'messageId') self.assertTrue(success) self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['integrations'], {'Amplitude': True}) self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['properties'], { 'property': 'value' }) + self.assertEqual(msg['properties'], {'property': 'value'}) self.assertEqual(msg['anonymousId'], 'anonymousId') self.assertEqual(msg['context']['library'], { 'name': 'analytics-python', @@ -211,16 +213,16 @@ def test_basic_screen(self): def test_advanced_screen(self): client = self.client success, msg = client.screen( - 'userId', 'category', 'name', { 'property': 'value' }, - { 'ip': '192.168.0.1' }, datetime(2014, 9, 3), 'anonymousId', - { 'Amplitude': True }, 'messageId') + 'userId', 'category', 'name', {'property': 'value'}, + {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', + {'Amplitude': True}, 'messageId') self.assertTrue(success) self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], { 'Amplitude': True }) + self.assertEqual(msg['integrations'], {'Amplitude': True}) self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['properties'], { 'property': 'value' }) + self.assertEqual(msg['properties'], {'property': 'value'}) self.assertEqual(msg['anonymousId'], 'anonymousId') self.assertEqual(msg['context']['library'], { 'name': 'analytics-python', @@ -237,7 +239,7 @@ def test_flush(self): client = self.client # set up the consumer with more requests than a single batch will allow for i in range(1000): - success, msg = client.identify('userId', { 'trait': 'value' }) + success, msg = client.identify('userId', {'trait': 'value'}) # We can't reliably assert that the queue is non-empty here; that's # a race condition. We do our best to load it up though. client.flush() @@ -270,7 +272,7 @@ def test_overflow(self): client.join() for i in range(10): - client.identify('userId') + client.identify('userId') success, msg = client.identify('userId') # Make sure we are informed that the queue is at capacity @@ -314,9 +316,9 @@ def test_gzip(self): client.flush() self.assertFalse(self.failed) - def test_user_defined_upload_size(self): + def test_user_defined_flush_at(self): client = Client('testsecret', on_error=self.fail, - upload_size=10, upload_interval=3) + flush_at=10, flush_interval=3) def mock_post_fn(*args, **kwargs): self.assertEquals(len(kwargs['batch']), 10) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 0ef868df..a8982479 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -23,12 +23,12 @@ def test_next(self): def test_next_limit(self): q = Queue() - upload_size = 50 - consumer = Consumer(q, '', upload_size) + flush_at = 50 + consumer = Consumer(q, '', flush_at) for i in range(10000): q.put(i) next = consumer.next() - self.assertEqual(next, list(range(upload_size))) + self.assertEqual(next, list(range(flush_at))) def test_dropping_oversize_msg(self): q = Queue() @@ -51,12 +51,13 @@ def test_upload(self): success = consumer.upload() self.assertTrue(success) - def test_upload_interval(self): - # Put _n_ items in the queue, pausing a little bit more than _upload_interval_ + def test_flush_interval(self): + # Put _n_ items in the queue, pausing a little bit more than _flush_interval_ # after each one. The consumer should upload _n_ times. q = Queue() - upload_interval = 0.3 - consumer = Consumer(q, 'testsecret', upload_size=10, upload_interval=upload_interval) + flush_interval = 0.3 + consumer = Consumer(q, 'testsecret', flush_at=10, + flush_interval=flush_interval) with mock.patch('analytics.consumer.post') as mock_post: consumer.start() for i in range(0, 3): @@ -66,26 +67,27 @@ def test_upload_interval(self): 'userId': 'userId' } q.put(track) - time.sleep(upload_interval * 1.1) + time.sleep(flush_interval * 1.1) self.assertEqual(mock_post.call_count, 3) def test_multiple_uploads_per_interval(self): - # Put _upload_size*2_ items in the queue at once, then pause for _upload_interval_. + # Put _flush_at*2_ items in the queue at once, then pause for _flush_interval_. # The consumer should upload 2 times. q = Queue() - upload_interval = 0.5 - upload_size = 10 - consumer = Consumer(q, 'testsecret', upload_size=upload_size, upload_interval=upload_interval) + flush_interval = 0.5 + flush_at = 10 + consumer = Consumer(q, 'testsecret', flush_at=flush_at, + flush_interval=flush_interval) with mock.patch('analytics.consumer.post') as mock_post: consumer.start() - for i in range(0, upload_size * 2): + for i in range(0, flush_at * 2): track = { 'type': 'track', 'event': 'python event %d' % i, 'userId': 'userId' } q.put(track) - time.sleep(upload_interval * 1.1) + time.sleep(flush_interval * 1.1) self.assertEqual(mock_post.call_count, 2) def test_request(self): @@ -123,7 +125,8 @@ def mock_post(*args, **kwargs): except type(expected_exception) as exc: self.assertEqual(exc, expected_exception) else: - self.fail("request() should raise an exception if still failing after %d retries" % consumer.retries) + self.fail( + "request() should raise an exception if still failing after %d retries" % consumer.retries) def test_request_retry(self): # we should retry on general errors @@ -132,11 +135,13 @@ def test_request_retry(self): # we should retry on server errors consumer = Consumer(None, 'testsecret') - self._test_request_retry(consumer, APIError(500, 'code', 'Internal Server Error'), 2) + self._test_request_retry(consumer, APIError( + 500, 'code', 'Internal Server Error'), 2) # we should retry on HTTP 429 errors consumer = Consumer(None, 'testsecret') - self._test_request_retry(consumer, APIError(429, 'code', 'Too Many Requests'), 2) + self._test_request_retry(consumer, APIError( + 429, 'code', 'Too Many Requests'), 2) # we should NOT retry on other client errors consumer = Consumer(None, 'testsecret') @@ -150,7 +155,8 @@ def test_request_retry(self): # test for number of exceptions raise > retries value consumer = Consumer(None, 'testsecret', retries=3) - self._test_request_retry(consumer, APIError(500, 'code', 'Internal Server Error'), 3) + self._test_request_retry(consumer, APIError( + 500, 'code', 'Internal Server Error'), 3) def test_pause(self): consumer = Consumer(None, 'testsecret') @@ -159,19 +165,22 @@ def test_pause(self): def test_max_batch_size(self): q = Queue() - consumer = Consumer(q, 'testsecret', upload_size=100000, upload_interval=3) + consumer = Consumer( + q, 'testsecret', flush_at=100000, flush_interval=3) track = { 'type': 'track', 'event': 'python event', 'userId': 'userId' } msg_size = len(json.dumps(track).encode()) - n_msgs = int(475000 / msg_size) # number of messages in a maximum-size batch + # number of messages in a maximum-size batch + n_msgs = int(475000 / msg_size) def mock_post_fn(_, data, **kwargs): res = mock.Mock() res.status_code = 200 - self.assertTrue(len(data.encode()) < 500000, 'batch size (%d) exceeds 500KB limit' % len(data.encode())) + self.assertTrue(len(data.encode()) < 500000, + 'batch size (%d) exceeds 500KB limit' % len(data.encode())) return res with mock.patch('analytics.request._session.post', side_effect=mock_post_fn) as mock_post: From e554033c9a545d9fb6be683e1cff2415b92636a8 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Fri, 6 Dec 2019 12:21:45 -0600 Subject: [PATCH 137/323] Test CircleCI build cycle From 5b37760f4c3ff7d4cf6467c31a9c997bd45c8018 Mon Sep 17 00:00:00 2001 From: Kelly Lu Date: Tue, 10 Dec 2019 16:35:43 -0800 Subject: [PATCH 138/323] Updated CircleCI badge with status API token --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96a50c48..d458c054 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ analytics-python ============== -[![CircleCI](https://circleci.com/gh/segmentio/analytics-python.svg?style=svg)](https://circleci.com/gh/segmentio/analytics-python) +[![CircleCI](https://circleci.com/gh/segmentio/analytics-python.svg?style=svg&circle-token=b09aadca8e15901549cf885adcb8e1eaf671a5d8)](https://circleci.com/gh/segmentio/analytics-python) analytics-python is a python client for [Segment](https://segment.com) From 16e266109f6af7032826093fa36844100557a189 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Wed, 11 Dec 2019 17:01:06 -0600 Subject: [PATCH 139/323] Add configurable number of threads to run consumer --- analytics/client.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index eed2525c..d4fb469d 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -28,7 +28,7 @@ class Client(object): def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, send=True, on_error=None, flush_at=100, flush_interval=0.5, gzip=False, max_retries=3, - sync_mode=False, timeout=15): + sync_mode=False, timeout=15, thread=1): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) @@ -45,21 +45,25 @@ def __init__(self, write_key=None, host=None, debug=False, self.log.setLevel(logging.DEBUG) if sync_mode: - self.consumer = None + self.consumers = None else: - self.consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - flush_at=flush_at, flush_interval=flush_interval, - gzip=gzip, retries=max_retries, timeout=timeout) - - # if we've disabled sending, just don't start the consumer + # On program exit, allow the consumer thread to exit cleanly. + # This prevents exceptions and a messy shutdown when the interpreter is + # destroyed before the daemon thread finishes execution. However, it + # is *not* the same as flushing the queue! To guarantee all messages + # have been delivered, you'll still need to call flush(). if send: - # On program exit, allow the consumer thread to exit cleanly. - # This prevents exceptions and a messy shutdown when the interpreter is - # destroyed before the daemon thread finishes execution. However, it - # is *not* the same as flushing the queue! To guarantee all messages - # have been delivered, you'll still need to call flush(). atexit.register(self.join) - self.consumer.start() + for n in range(thread): + self.consumers = [] + consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, + flush_at=flush_at, flush_interval=flush_interval, + gzip=gzip, retries=max_retries, timeout=timeout) + self.consumers.append(consumer) + + # if we've disabled sending, just don't start the consumer + if send: + consumer.start() def identify(self, user_id=None, traits=None, context=None, timestamp=None, anonymous_id=None, integrations=None, message_id=None): @@ -263,12 +267,13 @@ def flush(self): def join(self): """Ends the consumer thread once the queue is empty. Blocks execution until finished""" - self.consumer.pause() - try: - self.consumer.join() - except RuntimeError: - # consumer thread has not started - pass + for consumer in self.consumers: + consumer.pause() + try: + consumer.join() + except RuntimeError: + # consumer thread has not started + pass def shutdown(self): """Flush all messages and cleanly shutdown the client""" From d81f6515da6ee2049c9217dc4745558c778175fe Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 12 Dec 2019 14:56:18 -0600 Subject: [PATCH 140/323] Update tests for the multiple number of threads --- analytics/test/client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/analytics/test/client.py b/analytics/test/client.py index 4c648390..52637e75 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -256,13 +256,14 @@ def test_shutdown(self): # 1. client queue is empty # 2. consumer thread has stopped self.assertTrue(client.queue.empty()) - self.assertFalse(client.consumer.is_alive()) + for consumer in client.consumers: + self.assertFalse(consumer.is_alive()) def test_synchronous(self): client = Client('testsecret', sync_mode=True) success, message = client.identify('userId') - self.assertIsNone(client.consumer) + self.assertFalse(client.consumers) self.assertTrue(client.queue.empty()) self.assertTrue(success) @@ -333,8 +334,10 @@ def mock_post_fn(*args, **kwargs): def test_user_defined_timeout(self): client = Client('testsecret', timeout=10) - self.assertEquals(client.consumer.timeout, 10) + for consumer in client.consumers: + self.assertEquals(consumer.timeout, 10) def test_default_timeout_15(self): client = Client('testsecret') - self.assertEquals(client.consumer.timeout, 15) + for consumer in client.consumers: + self.assertEquals(consumer.timeout, 15) From 3e6f03f7aa13129e0e588f3646b1d761b23706b4 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 09:14:53 -0600 Subject: [PATCH 141/323] Fix linting errors on client.py --- analytics/client.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index d4fb469d..69bf5991 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -14,7 +14,7 @@ try: import queue -except: +except ImportError: import Queue as queue @@ -48,17 +48,20 @@ def __init__(self, write_key=None, host=None, debug=False, self.consumers = None else: # On program exit, allow the consumer thread to exit cleanly. - # This prevents exceptions and a messy shutdown when the interpreter is - # destroyed before the daemon thread finishes execution. However, it - # is *not* the same as flushing the queue! To guarantee all messages - # have been delivered, you'll still need to call flush(). + # This prevents exceptions and a messy shutdown when the + # interpreter is destroyed before the daemon thread finishes + # execution. However, it is *not* the same as flushing the queue! + # To guarantee all messages have been delivered, you'll still need + # to call flush(). if send: atexit.register(self.join) for n in range(thread): self.consumers = [] - consumer = Consumer(self.queue, write_key, host=host, on_error=on_error, - flush_at=flush_at, flush_interval=flush_interval, - gzip=gzip, retries=max_retries, timeout=timeout) + consumer = Consumer( + self.queue, write_key, host=host, on_error=on_error, + flush_at=flush_at, flush_interval=flush_interval, + gzip=gzip, retries=max_retries, timeout=timeout, + ) self.consumers.append(consumer) # if we've disabled sending, just don't start the consumer @@ -87,7 +90,8 @@ def identify(self, user_id=None, traits=None, context=None, timestamp=None, return self._enqueue(msg) def track(self, user_id=None, event=None, properties=None, context=None, - timestamp=None, anonymous_id=None, integrations=None, message_id=None): + timestamp=None, anonymous_id=None, integrations=None, + message_id=None): properties = properties or {} context = context or {} integrations = integrations or {} @@ -129,7 +133,8 @@ def alias(self, previous_id=None, user_id=None, context=None, return self._enqueue(msg) def group(self, user_id=None, group_id=None, traits=None, context=None, - timestamp=None, anonymous_id=None, integrations=None, message_id=None): + timestamp=None, anonymous_id=None, integrations=None, + message_id=None): traits = traits or {} context = context or {} integrations = integrations or {} @@ -266,7 +271,9 @@ def flush(self): self.log.debug('successfully flushed about %s items.', size) def join(self): - """Ends the consumer thread once the queue is empty. Blocks execution until finished""" + """Ends the consumer thread once the queue is empty. + Blocks execution until finished + """ for consumer in self.consumers: consumer.pause() try: From b7c7e401a33433c947e019c2cc2afae4d2659734 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 14:59:20 -0600 Subject: [PATCH 142/323] Fix linting errors on __init__.py --- analytics/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index d51b45fb..6589a965 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -19,30 +19,37 @@ def track(*args, **kwargs): """Send a track call.""" _proxy('track', *args, **kwargs) + def identify(*args, **kwargs): """Send a identify call.""" _proxy('identify', *args, **kwargs) + def group(*args, **kwargs): """Send a group call.""" _proxy('group', *args, **kwargs) + def alias(*args, **kwargs): """Send a alias call.""" _proxy('alias', *args, **kwargs) + def page(*args, **kwargs): """Send a page call.""" _proxy('page', *args, **kwargs) + def screen(*args, **kwargs): """Send a screen call.""" _proxy('screen', *args, **kwargs) + def flush(): """Tell the client to flush.""" _proxy('flush') + def join(): """Block program until the client clears the queue""" _proxy('join') @@ -58,8 +65,9 @@ def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client if not default_client: - default_client = Client(write_key, host=host, debug=debug, on_error=on_error, - send=send, sync_mode=sync_mode) + default_client = Client(write_key, host=host, debug=debug, + on_error=on_error, send=send, + sync_mode=sync_mode) fn = getattr(default_client, method) fn(*args, **kwargs) From f2b42eae1b5d712eef2676b9dc84b6fa02417adb Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 14:59:35 -0600 Subject: [PATCH 143/323] Fix linting errors on consumer.py --- analytics/consumer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/analytics/consumer.py b/analytics/consumer.py index 0aeab3dc..a823edfc 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -4,7 +4,6 @@ import backoff import json -from analytics.version import VERSION from analytics.request import post, APIError, DatetimeSerializer try: @@ -23,8 +22,9 @@ class Consumer(Thread): """Consumes the messages from the client's queue.""" log = logging.getLogger('segment') - def __init__(self, queue, write_key, flush_at=100, host=None, on_error=None, - flush_interval=0.5, gzip=False, retries=10, timeout=15): + def __init__(self, queue, write_key, flush_at=100, host=None, + on_error=None, flush_interval=0.5, gzip=False, retries=10, + timeout=15): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit @@ -38,7 +38,8 @@ def __init__(self, queue, write_key, flush_at=100, host=None, on_error=None, self.gzip = gzip # It's important to set running in the constructor: if we are asked to # pause immediately after construction, we might set running to True in - # run() *after* we set it to False in pause... and keep running forever. + # run() *after* we set it to False in pause... and keep running + # forever. self.running = True self.retries = retries self.timeout = timeout @@ -113,14 +114,19 @@ def request(self, batch): def fatal_exception(exc): if isinstance(exc, APIError): - # retry on server errors and client errors with 429 status code (rate limited), + # retry on server errors and client errors + # with 429 status code (rate limited), # don't retry on other client errors return (400 <= exc.status < 500) and exc.status != 429 else: # retry on all other errors (eg. network) return False - @backoff.on_exception(backoff.expo, Exception, max_tries=self.retries + 1, giveup=fatal_exception) + @backoff.on_exception( + backoff.expo, + Exception, + max_tries=self.retries + 1, + giveup=fatal_exception) def send_request(): post(self.write_key, self.host, gzip=self.gzip, timeout=self.timeout, batch=batch) From 8b56d0b230e45f02a1aef3d372db3dfd44d96405 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 16:18:56 -0600 Subject: [PATCH 144/323] Fix linting errors on request.py --- analytics/request.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/analytics/request.py b/analytics/request.py index eb7f825d..e1b62129 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -30,11 +30,13 @@ def post(write_key, host=None, gzip=False, timeout=15, **kwargs): headers['Content-Encoding'] = 'gzip' buf = BytesIO() with GzipFile(fileobj=buf, mode='w') as gz: - # 'data' was produced by json.dumps(), whose default encoding is utf-8. + # 'data' was produced by json.dumps(), + # whose default encoding is utf-8. gz.write(data.encode('utf-8')) data = buf.getvalue() - res = _session.post(url, data=data, auth=auth, headers=headers, timeout=timeout) + res = _session.post(url, data=data, auth=auth, + headers=headers, timeout=timeout) if res.status_code == 200: log.debug('data uploaded successfully') From 768019a6b2428d6f2c54656d097bedf955a1f776 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 16:19:12 -0600 Subject: [PATCH 145/323] Fix linting errors on utils.py --- analytics/utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/analytics/utils.py b/analytics/utils.py index 58b9d3a4..d240116b 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -13,10 +13,13 @@ def is_naive(dt): """Determines if a given datetime.datetime is naive.""" return dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None + def total_seconds(delta): """Determines total seconds with python < 2.7 compat.""" # http://stackoverflow.com/questions/3694835/python-2-6-5-divide-timedelta-with-timedelta - return (delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6 + return (delta.microseconds + + (delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6 + def guess_timezone(dt): """Attempts to convert a naive datetime to an aware datetime.""" @@ -34,11 +37,13 @@ def guess_timezone(dt): return dt + def remove_trailing_slash(host): if host.endswith('/'): - return host[:-1] + return host[:-1] return host + def clean(item): if isinstance(item, Decimal): return float(item) @@ -52,9 +57,11 @@ def clean(item): else: return _coerce_unicode(item) + def _clean_list(list_): return [clean(item) for item in list_] + def _clean_dict(dict_): data = {} for k, v in six.iteritems(dict_): @@ -68,6 +75,7 @@ def _clean_dict(dict_): ) return data + def _coerce_unicode(cmplx): try: item = cmplx.decode("utf-8", "strict") @@ -76,6 +84,4 @@ def _coerce_unicode(cmplx): item.decode("utf-8", "strict") log.warning('Error decoding: %s', item) return None - except: - raise return item From fab0e4b15155be73b4c2aebdda9ed90b75ecb86e Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 16:23:00 -0600 Subject: [PATCH 146/323] Fix linting errors on test files --- analytics/test/__init__.py | 2 ++ analytics/test/client.py | 3 ++- analytics/test/consumer.py | 33 ++++++++++++++++++++------------- analytics/test/module.py | 2 +- analytics/test/request.py | 10 ++++++---- analytics/test/utils.py | 8 +++++--- 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/analytics/test/__init__.py b/analytics/test/__init__.py index 877653af..4a920f8e 100644 --- a/analytics/test/__init__.py +++ b/analytics/test/__init__.py @@ -3,10 +3,12 @@ import logging import sys + def all_names(): for _, modname, _ in pkgutil.iter_modules(__path__): yield 'analytics.test.' + modname + def all(): logging.basicConfig(stream=sys.stderr) return unittest.defaultTestLoader.loadTestsFromNames(all_names()) diff --git a/analytics/test/client.py b/analytics/test/client.py index 52637e75..9b5b5aa8 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -326,7 +326,8 @@ def mock_post_fn(*args, **kwargs): # the post function should be called 2 times, with a batch size of 10 # each time. - with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) as mock_post: + with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) \ + as mock_post: for _ in range(20): client.identify('userId', {'trait': 'value'}) time.sleep(1) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index a8982479..739c2c58 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -52,8 +52,9 @@ def test_upload(self): self.assertTrue(success) def test_flush_interval(self): - # Put _n_ items in the queue, pausing a little bit more than _flush_interval_ - # after each one. The consumer should upload _n_ times. + # Put _n_ items in the queue, pausing a little bit more than + # _flush_interval_ after each one. + # The consumer should upload _n_ times. q = Queue() flush_interval = 0.3 consumer = Consumer(q, 'testsecret', flush_at=10, @@ -71,8 +72,8 @@ def test_flush_interval(self): self.assertEqual(mock_post.call_count, 3) def test_multiple_uploads_per_interval(self): - # Put _flush_at*2_ items in the queue at once, then pause for _flush_interval_. - # The consumer should upload 2 times. + # Put _flush_at*2_ items in the queue at once, then pause for + # _flush_interval_. The consumer should upload 2 times. q = Queue() flush_interval = 0.5 flush_at = 10 @@ -99,7 +100,8 @@ def test_request(self): } consumer.request([track]) - def _test_request_retry(self, consumer, expected_exception, exception_count): + def _test_request_retry(self, consumer, + expected_exception, exception_count): def mock_post(*args, **kwargs): mock_post.call_count += 1 @@ -107,26 +109,29 @@ def mock_post(*args, **kwargs): raise expected_exception mock_post.call_count = 0 - with mock.patch('analytics.consumer.post', mock.Mock(side_effect=mock_post)): + with mock.patch('analytics.consumer.post', + mock.Mock(side_effect=mock_post)): track = { 'type': 'track', 'event': 'python event', 'userId': 'userId' } - # request() should succeed if the number of exceptions raised is less - # than the retries paramater. + # request() should succeed if the number of exceptions raised is + # less than the retries paramater. if exception_count <= consumer.retries: consumer.request([track]) else: - # if exceptions are raised more times than the retries parameter, - # we expect the exception to be returned to the caller. + # if exceptions are raised more times than the retries + # parameter, we expect the exception to be returned to + # the caller. try: consumer.request([track]) except type(expected_exception) as exc: self.assertEqual(exc, expected_exception) else: self.fail( - "request() should raise an exception if still failing after %d retries" % consumer.retries) + "request() should raise an exception if still failing " + "after %d retries" % consumer.retries) def test_request_retry(self): # we should retry on general errors @@ -180,10 +185,12 @@ def mock_post_fn(_, data, **kwargs): res = mock.Mock() res.status_code = 200 self.assertTrue(len(data.encode()) < 500000, - 'batch size (%d) exceeds 500KB limit' % len(data.encode())) + 'batch size (%d) exceeds 500KB limit' + % len(data.encode())) return res - with mock.patch('analytics.request._session.post', side_effect=mock_post_fn) as mock_post: + with mock.patch('analytics.request._session.post', + side_effect=mock_post_fn) as mock_post: consumer.start() for _ in range(0, n_msgs + 2): q.put(track) diff --git a/analytics/test/module.py b/analytics/test/module.py index 8e9c10a3..6fcff6c8 100644 --- a/analytics/test/module.py +++ b/analytics/test/module.py @@ -26,7 +26,7 @@ def test_track(self): analytics.flush() def test_identify(self): - analytics.identify('userId', { 'email': 'user@email.com' }) + analytics.identify('userId', {'email': 'user@email.com'}) analytics.flush() def test_group(self): diff --git a/analytics/test/request.py b/analytics/test/request.py index 9ff74e3e..3c739bca 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -17,13 +17,15 @@ def test_valid_request(self): self.assertEqual(res.status_code, 200) def test_invalid_request_error(self): - self.assertRaises(Exception, post, 'testsecret', 'https://api.segment.io', False, '[{]') + self.assertRaises(Exception, post, 'testsecret', + 'https://api.segment.io', False, '[{]') def test_invalid_host(self): - self.assertRaises(Exception, post, 'testsecret', 'api.segment.io/', batch=[]) + self.assertRaises(Exception, post, 'testsecret', + 'api.segment.io/', batch=[]) def test_datetime_serialization(self): - data = { 'created': datetime(2012, 3, 4, 5, 6, 7, 891011) } + data = {'created': datetime(2012, 3, 4, 5, 6, 7, 891011)} result = json.dumps(data, cls=DatetimeSerializer) self.assertEqual(result, '{"created": "2012-03-04T05:06:07.891011"}') @@ -44,7 +46,7 @@ def test_should_not_timeout(self): def test_should_timeout(self): with self.assertRaises(requests.ReadTimeout): - res = post('testsecret', batch=[{ + post('testsecret', batch=[{ 'userId': 'userId', 'event': 'python event', 'type': 'track' diff --git a/analytics/test/utils.py b/analytics/test/utils.py index 09ce0ab3..3ff8b158 100644 --- a/analytics/test/utils.py +++ b/analytics/test/utils.py @@ -65,12 +65,14 @@ def test_bytes(self): utils.clean(item) def test_clean_fn(self): - cleaned = utils.clean({ 'fn': lambda x: x, 'number': 4 }) + cleaned = utils.clean({'fn': lambda x: x, 'number': 4}) self.assertEqual(cleaned['number'], 4) # TODO: fixme, different behavior on python 2 and 3 if 'fn' in cleaned: self.assertEqual(cleaned['fn'], None) def test_remove_slash(self): - self.assertEqual('http://segment.io', utils.remove_trailing_slash('http://segment.io/')) - self.assertEqual('http://segment.io', utils.remove_trailing_slash('http://segment.io')) + self.assertEqual('http://segment.io', + utils.remove_trailing_slash('http://segment.io/')) + self.assertEqual('http://segment.io', + utils.remove_trailing_slash('http://segment.io')) From 478ac00fa56aa8e1693927ba416b64e0d2dcae26 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 16:40:24 -0600 Subject: [PATCH 147/323] Fix linting errors for simulator.py --- simulator.py | 47 +++++++++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/simulator.py b/simulator.py index d62deaf7..88271c3a 100644 --- a/simulator.py +++ b/simulator.py @@ -7,58 +7,73 @@ __version__ = '0.0.1' __description__ = 'scripting simulator' + def json_hash(str): - if str: - return json.loads(str) + if str: + return json.loads(str) # analytics -method= -segment-write-key= [options] + parser = argparse.ArgumentParser(description='send a segment message') parser.add_argument('--writeKey', help='the Segment writeKey') parser.add_argument('--type', help='The Segment message type') parser.add_argument('--userId', help='the user id to send the event as') -parser.add_argument('--anonymousId', help='the anonymous user id to send the event as') -parser.add_argument('--context', help='additional context for the event (JSON-encoded)') +parser.add_argument( + '--anonymousId', help='the anonymous user id to send the event as') +parser.add_argument( + '--context', help='additional context for the event (JSON-encoded)') parser.add_argument('--event', help='the event name to send with the event') -parser.add_argument('--properties', help='the event properties to send (JSON-encoded)') +parser.add_argument( + '--properties', help='the event properties to send (JSON-encoded)') -parser.add_argument('--name', help='name of the screen or page to send with the message') +parser.add_argument( + '--name', help='name of the screen or page to send with the message') -parser.add_argument('--traits', help='the identify/group traits to send (JSON-encoded)') +parser.add_argument( + '--traits', help='the identify/group traits to send (JSON-encoded)') parser.add_argument('--groupId', help='the group id') options = parser.parse_args() + def failed(status, msg): raise Exception(msg) + def track(): - analytics.track(options.userId, options.event, anonymous_id = options.anonymousId, - properties = json_hash(options.properties), context = json_hash(options.context)) + analytics.track(options.userId, options.event, anonymous_id=options.anonymousId, + properties=json_hash(options.properties), context=json_hash(options.context)) + def page(): - analytics.page(options.userId, name = options.name, anonymous_id = options.anonymousId, - properties = json_hash(options.properties), context = json_hash(options.context)) + analytics.page(options.userId, name=options.name, anonymous_id=options.anonymousId, + properties=json_hash(options.properties), context=json_hash(options.context)) + def screen(): - analytics.screen(options.userId, name = options.name, anonymous_id = options.anonymousId, - properties = json_hash(options.properties), context = json_hash(options.context)) + analytics.screen(options.userId, name=options.name, anonymous_id=options.anonymousId, + properties=json_hash(options.properties), context=json_hash(options.context)) + def identify(): - analytics.identify(options.userId, anonymous_id = options.anonymousId, - traits = json_hash(options.traits), context = json_hash(options.context)) + analytics.identify(options.userId, anonymous_id=options.anonymousId, + traits=json_hash(options.traits), context=json_hash(options.context)) + def group(): analytics.group(options.userId, options.groupId, json_hash(options.traits), - json_hash(options.context), anonymous_id = options.anonymousId) + json_hash(options.context), anonymous_id=options.anonymousId) + def unknown(): print() + analytics.write_key = options.writeKey analytics.on_error = failed analytics.debug = True From 93db1db563cea34bf2978a14947c1a452684f14b Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 16:47:56 -0600 Subject: [PATCH 148/323] Run flake8 on CI --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3f77fa54..01c3486e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,8 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install setuptools coverage pylint==1.9.3 pycodestyle + - run: sudo pip install setuptools coverage pylint==1.9.3 pycodestyle flake8 + - run: flake8 --max-complexity=10 ./analytics - run: make test - run: make e2e_test @@ -61,7 +62,8 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 pycodestyle + - run: sudo pip install coverage pylint==1.9.3 pycodestyle flake8 + - run: flake8 --max-complexity=10 ./analytics - run: make test - run: make e2e_test From a34c427207e817d2fbfc9f93abc005d98797c1b5 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Thu, 19 Dec 2019 17:03:17 -0600 Subject: [PATCH 149/323] Update Makefile for compatibility with Flake8 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 06c93bf1..d44742cd 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ test: pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out # fail on pycodestyle errors on the code change - git diff origin/master..HEAD analytics | pycodestyle --ignore=E501 --diff --statistics --count - pycodestyle --ignore=E501 --statistics analytics > pycodestyle.out || true + git diff origin/master..HEAD analytics | pycodestyle --ignore=E501,W503 --diff --statistics --count + pycodestyle --ignore=E501,W503 --statistics analytics > pycodestyle.out || true coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: From 19e4b846abf500c16e36103385322de4ed047f2b Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Fri, 20 Dec 2019 13:55:52 -0600 Subject: [PATCH 150/323] Run flake8 on diff with master branch --- .circleci/config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01c3486e..d54892a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,10 @@ jobs: - checkout - run: pip install --user . - run: sudo pip install setuptools coverage pylint==1.9.3 pycodestyle flake8 - - run: flake8 --max-complexity=10 ./analytics + - run: + name: Linting with Flake8 + command: | + git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - run: make test - run: make e2e_test @@ -63,7 +66,10 @@ jobs: - checkout - run: pip install --user . - run: sudo pip install coverage pylint==1.9.3 pycodestyle flake8 - - run: flake8 --max-complexity=10 ./analytics + - run: + name: Linting with Flake8 + command: | + git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - run: make test - run: make e2e_test From 6d84dae2ddc4e59099a0684302dd58183f179b39 Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Fri, 20 Dec 2019 13:56:05 -0600 Subject: [PATCH 151/323] Replace pycodestyle with flake8 --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d44742cd..0a4584fe 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ test: pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out - # fail on pycodestyle errors on the code change - git diff origin/master..HEAD analytics | pycodestyle --ignore=E501,W503 --diff --statistics --count - pycodestyle --ignore=E501,W503 --statistics analytics > pycodestyle.out || true + flake8 --max-complexity=10 --statistics analytics > flake8.out || true coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: From b89bda9012bc909d5aa603f5c091c457c938abcb Mon Sep 17 00:00:00 2001 From: Pablo Berganza Date: Fri, 20 Dec 2019 13:57:22 -0600 Subject: [PATCH 152/323] Replace pycodestyle with flake8 on circleci --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d54892a5..f2bee34f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 pycodestyle + - run: sudo pip install coverage pylint==1.9.3 flake8 - run: make test - persist_to_workspace: root: . @@ -19,7 +19,7 @@ jobs: - store_artifacts: path: pylint.out - store_artifacts: - path: pycodestyle.out + path: flake8.out coverage: docker: @@ -51,7 +51,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install setuptools coverage pylint==1.9.3 pycodestyle flake8 + - run: sudo pip install setuptools coverage pylint==1.9.3 flake8 - run: name: Linting with Flake8 command: | @@ -65,7 +65,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 pycodestyle flake8 + - run: sudo pip install coverage pylint==1.9.3 flake8 - run: name: Linting with Flake8 command: | From c2067c03ab03a52cd39078328311c6c796556d0e Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 17 Apr 2020 14:26:32 +0530 Subject: [PATCH 153/323] Install mock version that is compatible with 3.4+ --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2bee34f..a577893e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 flake8 + - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 - run: make test - persist_to_workspace: root: . @@ -51,7 +51,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install setuptools coverage pylint==1.9.3 flake8 + - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 - run: name: Linting with Flake8 command: | @@ -65,7 +65,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 flake8 + - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 - run: name: Linting with Flake8 command: | From d4180cd7add8597c9069e50289cfc05b74b4ef0b Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 17 Apr 2020 17:12:39 +0530 Subject: [PATCH 154/323] Always use latest minor versions --- .circleci/config.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a577893e..473f9580 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ defaults: jobs: build: docker: - - image: circleci/python:2.7.15-stretch + - image: circleci/python:2.7 steps: - checkout - run: pip install --user . @@ -23,7 +23,7 @@ jobs: coverage: docker: - - image: circleci/python:2.7.15-stretch + - image: circleci/python:2.7 steps: - checkout - attach_workspace: { at: . } @@ -36,7 +36,7 @@ jobs: snyk: docker: - - image: circleci/python:2.7.15-stretch + - image: circleci/python:2.7 steps: - checkout - attach_workspace: { at: . } @@ -47,7 +47,7 @@ jobs: test_27: docker: - - image: circleci/python:2.7.15-stretch + - image: circleci/python:2.7 steps: - checkout - run: pip install --user . @@ -61,7 +61,7 @@ jobs: test_34: &test_34 docker: - - image: circleci/python:3.4.8-jessie-node + - image: circleci/python:3.4 steps: - checkout - run: pip install --user . @@ -76,26 +76,26 @@ jobs: test_35: <<: *test_34 docker: - - image: circleci/python:3.5.5-jessie + - image: circleci/python:3.5 test_36: <<: *test_34 docker: - - image: circleci/python:3.6.5-jessie + - image: circleci/python:3.6 test_37: <<: *test_34 docker: - - image: circleci/python:3.7-stretch + - image: circleci/python:3.7 test_38: <<: *test_34 docker: - - image: circleci/python:3.8-buster + - image: circleci/python:3.8 publish: docker: - - image: circleci/python:3.6.5-jessie + - image: circleci/python:3.6 steps: - checkout - run: sudo pip install twine From 1a1e6ec5ebc3ff9699c635a96d6b362ac284b91d Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 17 Apr 2020 18:01:19 +0530 Subject: [PATCH 155/323] Add test dependencies to setup.py Also replaced tests_require with extras_require. The test command (which tests_required was used for) was deprecated in setuptools 41.5.0. --- setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4643f3ad..f00aea21 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,10 @@ ] tests_require = [ - "mock>=2.0.0" + "mock==2.0.0", + "pylint==1.9.3", + "flake8==3.7.9", + "coverage==4.5.4" ] setup( @@ -45,7 +48,9 @@ packages=['analytics', 'analytics.test'], license='MIT License', install_requires=install_requires, - tests_require=tests_require, + extras_require={ + 'test': tests_require + }, description='The hassle-free way to integrate analytics into any python application.', long_description=long_description, classifiers=[ From 2b3c8313f425f44833470df77f827b3b0b607834 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 17 Apr 2020 18:14:54 +0530 Subject: [PATCH 156/323] Add a requirements.txt for snyk to use --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 473f9580..0f3c3545 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,13 +36,12 @@ jobs: snyk: docker: - - image: circleci/python:2.7 + - image: circleci/python:3.8 steps: - checkout - attach_workspace: { at: . } - - run: python setup.py egg_info - - run: cp analytics_python.egg-info/requires.txt requirements.txt - - run: pip install --user -r requirements.txt + - run: pip install dephell + - run: dephell deps convert --from=setup.py --to=requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh test_27: From 9f54e71058bddc72b3e528b3667fa70f965ab439 Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 17 Apr 2020 19:05:28 +0530 Subject: [PATCH 157/323] Reuse YAML config for 3.4 too --- .circleci/config.yml | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f3c3545..f8649497 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: - run: dephell deps convert --from=setup.py --to=requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh - test_27: + test_27: &test docker: - image: circleci/python:2.7 steps: @@ -58,37 +58,28 @@ jobs: - run: make test - run: make e2e_test - test_34: &test_34 + test_34: + <<: *test docker: - image: circleci/python:3.4 - steps: - - checkout - - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 - - run: - name: Linting with Flake8 - command: | - git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - - run: make test - - run: make e2e_test test_35: - <<: *test_34 + <<: *test docker: - image: circleci/python:3.5 test_36: - <<: *test_34 + <<: *test docker: - image: circleci/python:3.6 test_37: - <<: *test_34 + <<: *test docker: - image: circleci/python:3.7 test_38: - <<: *test_34 + <<: *test docker: - image: circleci/python:3.8 From 84ea6adb7698c5deac514e2c0bab559af239d3ae Mon Sep 17 00:00:00 2001 From: Paul Kuruvilla Date: Fri, 17 Apr 2020 19:09:48 +0530 Subject: [PATCH 158/323] ci: install test deps from setup.py Includes special handling for the circleci/python:3.4 image, which doesn't setup PATH properly. This isn't worth fixing right now, as it is likely that we'll deprecate Python 3.4 support very soon. --- .circleci/config.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8649497..d20cfa4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,8 +49,7 @@ jobs: - image: circleci/python:2.7 steps: - checkout - - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 + - run: pip install --user .[test] - run: name: Linting with Flake8 command: | @@ -59,9 +58,22 @@ jobs: - run: make e2e_test test_34: - <<: *test docker: - image: circleci/python:3.4 + steps: + - checkout + - run: pip install --user .[test] + + # The circleci/python:3.4 image doesn't set PATH correctly, so we need to + # install these by hand with sudo + - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 + + - run: + name: Linting with Flake8 + command: | + git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics + - run: make test + - run: make e2e_test test_35: <<: *test From d53fb606bcee14d3509a7693ce861c001beb03af Mon Sep 17 00:00:00 2001 From: Alex Noonan Date: Tue, 14 Jul 2020 11:14:38 -0700 Subject: [PATCH 159/323] use inclusive language (#168) * use inclusive language --- .pylintrc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.pylintrc b/.pylintrc index 8f7fd04a..0d41c235 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,15 +1,10 @@ [MASTER] -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not +# Add files or directories to the ignore list. They should be base names, not # paths. ignore=CVS -# Add files or directories matching the regex patterns to the blacklist. The +# Add files or directories matching the regex patterns to the denylist. The # regex matches against base names, not paths. ignore-patterns= From b64cece960320c920746620a884335f573cb994d Mon Sep 17 00:00:00 2001 From: Harsh Singh Date: Thu, 1 Oct 2020 01:46:21 +0530 Subject: [PATCH 160/323] Update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2d736009..24ee1718 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ dist dist MANIFEST build -.eggs \ No newline at end of file +.eggs +*.bat From fbd3c9283bce3d3c961617a377cf207c36d39fcd Mon Sep 17 00:00:00 2001 From: Alon Diamant Date: Wed, 7 Oct 2020 14:12:32 +0300 Subject: [PATCH 161/323] Upgrade backoff dependency, drop Python 2.6 support --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f00aea21..a5371bc9 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ "requests>=2.7,<3.0", "six>=1.5", "monotonic>=1.5", - "backoff==1.6.0", + "backoff==1.10.0", "python-dateutil>2.1" ] @@ -60,7 +60,6 @@ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.2", From e9de069ab7fd2306d5e49dcf72eb4497e51b1f3b Mon Sep 17 00:00:00 2001 From: Kelly Lu Date: Mon, 14 Dec 2020 14:39:58 -0800 Subject: [PATCH 162/323] Removed test for 3.4 --- .circleci/config.yml | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d20cfa4e..a00ce5d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,25 +56,7 @@ jobs: git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - run: make test - run: make e2e_test - - test_34: - docker: - - image: circleci/python:3.4 - steps: - - checkout - - run: pip install --user .[test] - - # The circleci/python:3.4 image doesn't set PATH correctly, so we need to - # install these by hand with sudo - - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 - - - run: - name: Linting with Flake8 - command: | - git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - - run: make test - - run: make e2e_test - + test_35: <<: *test docker: @@ -113,9 +95,6 @@ workflows: - test_27: filters: <<: *taggedReleasesFilter - - test_34: - filters: - <<: *taggedReleasesFilter - test_35: filters: <<: *taggedReleasesFilter @@ -132,7 +111,6 @@ workflows: requires: - build - test_27 - - test_34 - test_35 - test_36 - test_37 @@ -162,6 +140,5 @@ workflows: - scheduled_e2e_testing jobs: - test_27 - - test_34 - test_35 - test_36 From 648b609878eaf32e8d49a0de010ee02119e0cf74 Mon Sep 17 00:00:00 2001 From: Kelly Lu Date: Mon, 14 Dec 2020 14:47:51 -0800 Subject: [PATCH 163/323] Update changelog with proper entry for beta1 --- HISTORY.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 776e9f71..c6a2268f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,9 @@ -1.3.0 / 2018-10-10 +1.3.0-beta1 / 2019-04-27 +================== + + * Add `sync_mode` option ([#147](https://github.com/segmentio/analytics-python/pull/147)) + +1.3.0-beta0 / 2018-10-10 ================== * Add User-Agent header to messages @@ -12,7 +17,6 @@ * Drop messages greater than 32kb * Allow user-defined upload size * Support custom messageId - * Add `sync_mode` option ([#147](https://github.com/segmentio/analytics-python/pull/147)) 1.2.9 / 2017-11-28 ================== From f236f87fa27f08ffef581d0b0159c82529377676 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 24 Mar 2021 16:34:57 +0100 Subject: [PATCH 164/323] Fix some code liting and deprecated assert functions --- .gitignore | 1 + analytics/test/client.py | 8 ++++---- analytics/test/consumer.py | 2 +- setup.py | 3 +-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 24ee1718..3427202e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ MANIFEST build .eggs *.bat +.vscode/ \ No newline at end of file diff --git a/analytics/test/client.py b/analytics/test/client.py index 9b5b5aa8..7aa52236 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -322,7 +322,7 @@ def test_user_defined_flush_at(self): flush_at=10, flush_interval=3) def mock_post_fn(*args, **kwargs): - self.assertEquals(len(kwargs['batch']), 10) + self.assertEqual(len(kwargs['batch']), 10) # the post function should be called 2 times, with a batch size of 10 # each time. @@ -331,14 +331,14 @@ def mock_post_fn(*args, **kwargs): for _ in range(20): client.identify('userId', {'trait': 'value'}) time.sleep(1) - self.assertEquals(mock_post.call_count, 2) + self.assertEqual(mock_post.call_count, 2) def test_user_defined_timeout(self): client = Client('testsecret', timeout=10) for consumer in client.consumers: - self.assertEquals(consumer.timeout, 10) + self.assertEqual(consumer.timeout, 10) def test_default_timeout_15(self): client = Client('testsecret') for consumer in client.consumers: - self.assertEquals(consumer.timeout, 15) + self.assertEqual(consumer.timeout, 15) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 739c2c58..e4f9153c 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -195,4 +195,4 @@ def mock_post_fn(_, data, **kwargs): for _ in range(0, n_msgs + 2): q.put(track) q.join() - self.assertEquals(mock_post.call_count, 2) + self.assertEqual(mock_post.call_count, 2) diff --git a/setup.py b/setup.py index a5371bc9..69e9a425 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ - +from version import VERSION import os import sys @@ -9,7 +9,6 @@ # Don't import analytics-python module here, since deps may not be installed sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) -from version import VERSION long_description = ''' Segment is the simplest way to integrate analytics into your application. From eb97006e13b5d02973bba36a99a0fefc5bd43584 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Thu, 25 Mar 2021 12:40:33 +0100 Subject: [PATCH 165/323] Readme refactoring --- README.md | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d458c054..bca7e89e 100644 --- a/README.md +++ b/README.md @@ -10,29 +10,29 @@ analytics-python is a python client for [Segment](https://segment.com)

You can't fix what you can't measure

-Analytics helps you measure your users, product, and business. It unlocks insights into your app's funnel, core business metrics, and whether you have product-market fit. +Analytics helps you measure your users, product, and business. It unlocks insights into your app's funnel, core business metrics, and whether you have a product-market fit. -## How to get started +## 🚀 How to get started 1. **Collect analytics data** from your app(s). - The top 200 Segment companies collect data from 5+ source types (web, mobile, server, CRM, etc.). 2. **Send the data to analytics tools** (for example, Google Analytics, Amplitude, Mixpanel). - - Over 250+ Segment companies send data to eight categories of destinations such as analytics tools, warehouses, email marketing and remarketing systems, session recording, and more. + - Over 250+ Segment companies send data to eight categories of destinations such as analytics tools, warehouses, email marketing, and remarketing systems, session recording, and more. 3. **Explore your data** by creating metrics (for example, new signups, retention cohorts, and revenue generation). - - The best Segment companies use retention cohorts to measure product market fit. Netflix has 70% paid retention after 12 months, 30% after 7 years. + - The best Segment companies use retention cohorts to measure product-market fit. Netflix has 70% paid retention after 12 months, 30% after 7 years. [Segment](https://segment.com) collects analytics data and allows you to send it to more than 250 apps (such as Google Analytics, Mixpanel, Optimizely, Facebook Ads, Slack, Sentry) just by flipping a switch. You only need one Segment code snippet, and you can turn integrations on and off at will, with no additional code. [Sign up with Segment today](https://app.segment.com/signup). -### Why? +### 🤔 Why? 1. **Power all your analytics apps with the same data**. Instead of writing code to integrate all of your tools individually, send data to Segment, once. 2. **Install tracking for the last time**. We're the last integration you'll ever need to write. You only need to instrument Segment once. Reduce all of your tracking code and advertising tags into a single set of API calls. 3. **Send data from anywhere**. Send Segment data from any device, and we'll transform and send it on to any tool. -4. **Query your data in SQL**. Slice, dice, and analyze your data in detail with Segment SQL. We'll transform and load your customer behavioral data directly from your apps into Amazon Redshift, Google BigQuery, or Postgres. Save weeks of engineering time by not having to invent your own data warehouse and ETL pipeline. +4. **Query your data in SQL**. Slice, dice, and analyze your data in detail with Segment SQL. We'll transform and load your customer behavioral data directly from your apps into Amazon Redshift, Google BigQuery, or Postgres. Save weeks of engineering time by not having to invent your data warehouse and ETL pipeline. For example, you can capture data on any app: - ```js + ```python analytics.track('Order Completed', { price: 99.84 }) ``` Then, query the resulting data in SQL: @@ -41,6 +41,32 @@ Analytics helps you measure your users, product, and business. It unlocks insigh order by price desc ``` +## 👨‍💻 Getting Started + +Install `analytics-python` using pip: + +```bash +$ pip install analytics-python +``` + +or you can clone this repo: +```bash +$ git clone https://github.com/segmentio/analytics-python.git + +$ cd analytics-python + +$ sudo python3 setup.py install +``` + +Now inside your app, you'll want to **set your** `write_key` before making any analytics calls: + +```python +import analytics + +analytics.write_key = 'YOUR_WRITE_KEY' +``` +**Note** If you need to send data to multiple Segment sources, you can initialize a new Client for each `write_key` + ### 🚀 Startup Program
@@ -75,3 +101,5 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +[![forthebadge](https://forthebadge.com/images/badges/built-with-love.svg)](https://forthebadge.com) From f601dede7bfebfb69d23dcd7b27e5a7ab37b845b Mon Sep 17 00:00:00 2001 From: pho3nix Date: Thu, 25 Mar 2021 13:21:06 +0100 Subject: [PATCH 166/323] code refactoring --- analytics/client.py | 2 +- analytics/consumer.py | 2 +- analytics/request.py | 11 ++++++----- analytics/test/client.py | 16 ++++++++-------- analytics/test/consumer.py | 3 ++- analytics/test/module.py | 4 ++-- analytics/test/utils.py | 4 ++-- analytics/utils.py | 12 ++++++------ setup.py | 2 +- simulator.py | 4 ++-- 10 files changed, 31 insertions(+), 29 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 69bf5991..1b5bf8d0 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -55,7 +55,7 @@ def __init__(self, write_key=None, host=None, debug=False, # to call flush(). if send: atexit.register(self.join) - for n in range(thread): + for _ in range(thread): self.consumers = [] consumer = Consumer( self.queue, write_key, host=host, on_error=on_error, diff --git a/analytics/consumer.py b/analytics/consumer.py index a823edfc..1c3e7fd1 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -73,7 +73,7 @@ def upload(self): self.on_error(e, batch) finally: # mark items as acknowledged from queue - for item in batch: + for _ in batch: self.queue.task_done() return success diff --git a/analytics/request.py b/analytics/request.py index e1b62129..892427f1 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -1,11 +1,11 @@ from datetime import date, datetime -from dateutil.tz import tzutc +from io import BytesIO +from gzip import GzipFile import logging import json -from gzip import GzipFile +from dateutil.tz import tzutc from requests.auth import HTTPBasicAuth from requests import sessions -from io import BytesIO from analytics.version import VERSION from analytics.utils import remove_trailing_slash @@ -46,13 +46,14 @@ def post(write_key, host=None, gzip=False, timeout=15, **kwargs): payload = res.json() log.debug('received response: %s', payload) raise APIError(res.status_code, payload['code'], payload['message']) - except ValueError: - raise APIError(res.status_code, 'unknown', res.text) + except ValueError as value_error: + raise APIError(res.status_code, 'unknown', res.text) from value_error class APIError(Exception): def __init__(self, status, code, message): + super().__init__(self, message) self.message = message self.status = status self.code = code diff --git a/analytics/test/client.py b/analytics/test/client.py index 7aa52236..24e41041 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -1,8 +1,8 @@ from datetime import date, datetime import unittest +import time import six import mock -import time from analytics.version import VERSION from analytics.client import Client @@ -238,8 +238,8 @@ def test_advanced_screen(self): def test_flush(self): client = self.client # set up the consumer with more requests than a single batch will allow - for i in range(1000): - success, msg = client.identify('userId', {'trait': 'value'}) + for _ in range(1000): + _, _ = client.identify('userId', {'trait': 'value'}) # We can't reliably assert that the queue is non-empty here; that's # a race condition. We do our best to load it up though. client.flush() @@ -249,8 +249,8 @@ def test_flush(self): def test_shutdown(self): client = self.client # set up the consumer with more requests than a single batch will allow - for i in range(1000): - success, msg = client.identify('userId', {'trait': 'value'}) + for _ in range(1000): + _, _ = client.identify('userId', {'trait': 'value'}) client.shutdown() # we expect two things after shutdown: # 1. client queue is empty @@ -262,7 +262,7 @@ def test_shutdown(self): def test_synchronous(self): client = Client('testsecret', sync_mode=True) - success, message = client.identify('userId') + success, _ = client.identify('userId') self.assertFalse(client.consumers) self.assertTrue(client.queue.empty()) self.assertTrue(success) @@ -272,10 +272,10 @@ def test_overflow(self): # Ensure consumer thread is no longer uploading client.join() - for i in range(10): + for _ in range(10): client.identify('userId') - success, msg = client.identify('userId') + success, _ = client.identify('userId') # Make sure we are informed that the queue is at capacity self.assertFalse(success) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index e4f9153c..91b3ca05 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -91,7 +91,8 @@ def test_multiple_uploads_per_interval(self): time.sleep(flush_interval * 1.1) self.assertEqual(mock_post.call_count, 2) - def test_request(self): + @classmethod + def test_request(cls): consumer = Consumer(None, 'testsecret') track = { 'type': 'track', diff --git a/analytics/test/module.py b/analytics/test/module.py index 6fcff6c8..3901b1c7 100644 --- a/analytics/test/module.py +++ b/analytics/test/module.py @@ -5,8 +5,8 @@ class TestModule(unittest.TestCase): - def failed(self): - self.failed = True + # def failed(self): + # self.failed = True def setUp(self): self.failed = False diff --git a/analytics/test/utils.py b/analytics/test/utils.py index 3ff8b158..15384f53 100644 --- a/analytics/test/utils.py +++ b/analytics/test/utils.py @@ -56,7 +56,8 @@ def test_clean_with_dates(self): } self.assertEqual(dict_with_dates, utils.clean(dict_with_dates)) - def test_bytes(self): + @classmethod + def test_bytes(cls): if six.PY3: item = bytes(10) else: @@ -67,7 +68,6 @@ def test_bytes(self): def test_clean_fn(self): cleaned = utils.clean({'fn': lambda x: x, 'number': 4}) self.assertEqual(cleaned['number'], 4) - # TODO: fixme, different behavior on python 2 and 3 if 'fn' in cleaned: self.assertEqual(cleaned['fn'], None) diff --git a/analytics/utils.py b/analytics/utils.py index d240116b..0bc7098e 100644 --- a/analytics/utils.py +++ b/analytics/utils.py @@ -1,9 +1,10 @@ -from dateutil.tz import tzlocal, tzutc -from datetime import date, datetime -from decimal import Decimal import logging import numbers +from decimal import Decimal +from datetime import date, datetime +from dateutil.tz import tzlocal, tzutc + import six log = logging.getLogger('segment') @@ -31,9 +32,8 @@ def guess_timezone(dt): # this was created using datetime.datetime.now() # so we are in the local timezone return dt.replace(tzinfo=tzlocal()) - else: - # at this point, the best we can do is guess UTC - return dt.replace(tzinfo=tzutc()) + # at this point, the best we can do is guess UTC + return dt.replace(tzinfo=tzutc()) return dt diff --git a/setup.py b/setup.py index 69e9a425..6a4caec9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ -from version import VERSION import os import sys +from analytics.version import VERSION try: from setuptools import setup diff --git a/simulator.py b/simulator.py index 88271c3a..ad94068a 100644 --- a/simulator.py +++ b/simulator.py @@ -1,7 +1,7 @@ -import analytics +import logging import argparse import json -import logging +import analytics __name__ = 'simulator.py' __version__ = '0.0.1' From 53e60b2ef0c9eae60b3a397fd03e34b7803543ec Mon Sep 17 00:00:00 2001 From: Heitor Sampaio Date: Thu, 25 Mar 2021 15:36:22 +0100 Subject: [PATCH 167/323] Updated config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a00ce5d7..fea4c759 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 3 defaults: taggedReleasesFilter: &taggedReleasesFilter tags: @@ -6,7 +6,7 @@ defaults: jobs: build: docker: - - image: circleci/python:2.7 + - image: circleci/python:3.8 steps: - checkout - run: pip install --user . @@ -23,7 +23,7 @@ jobs: coverage: docker: - - image: circleci/python:2.7 + - image: circleci/python:3.8 steps: - checkout - attach_workspace: { at: . } From 5acd5f1e3406558bd26e13ba16834fa03f978928 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Thu, 25 Mar 2021 15:39:02 +0100 Subject: [PATCH 168/323] fix setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6a4caec9..8959db40 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ import os import sys -from analytics.version import VERSION try: from setuptools import setup @@ -9,6 +8,7 @@ # Don't import analytics-python module here, since deps may not be installed sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) +from version import VERSION long_description = ''' Segment is the simplest way to integrate analytics into your application. From 223cadb6b21a00a38b61c294c698658592d992bd Mon Sep 17 00:00:00 2001 From: pho3nix Date: Thu, 25 Mar 2021 15:39:36 +0100 Subject: [PATCH 169/323] fix build CircleCI --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fea4c759..a00ce5d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 3 +version: 2 defaults: taggedReleasesFilter: &taggedReleasesFilter tags: @@ -6,7 +6,7 @@ defaults: jobs: build: docker: - - image: circleci/python:3.8 + - image: circleci/python:2.7 steps: - checkout - run: pip install --user . @@ -23,7 +23,7 @@ jobs: coverage: docker: - - image: circleci/python:3.8 + - image: circleci/python:2.7 steps: - checkout - attach_workspace: { at: . } From 38039afa40e1110315db47bf88cea00fbedbf39c Mon Sep 17 00:00:00 2001 From: pho3nix Date: Thu, 25 Mar 2021 15:41:34 +0100 Subject: [PATCH 170/323] fix conflict --- analytics/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics/request.py b/analytics/request.py index 892427f1..04cd19e0 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -46,8 +46,8 @@ def post(write_key, host=None, gzip=False, timeout=15, **kwargs): payload = res.json() log.debug('received response: %s', payload) raise APIError(res.status_code, payload['code'], payload['message']) - except ValueError as value_error: - raise APIError(res.status_code, 'unknown', res.text) from value_error + except ValueError: + raise APIError(res.status_code, 'unknown', res.text) class APIError(Exception): From 24fd693eba4f6f73b7284fcfeb883cf2a2c29ad9 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Thu, 25 Mar 2021 15:43:47 +0100 Subject: [PATCH 171/323] fix conflict --- analytics/request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/analytics/request.py b/analytics/request.py index 04cd19e0..8477e46d 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -53,7 +53,6 @@ def post(write_key, host=None, gzip=False, timeout=15, **kwargs): class APIError(Exception): def __init__(self, status, code, message): - super().__init__(self, message) self.message = message self.status = status self.code = code From 427e6563b974ae6cdea184e281707c25aae2c217 Mon Sep 17 00:00:00 2001 From: Heitor Sampaio Date: Thu, 25 Mar 2021 15:50:40 +0100 Subject: [PATCH 172/323] Updating the CircleCI badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bca7e89e..8a226784 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ analytics-python ============== -[![CircleCI](https://circleci.com/gh/segmentio/analytics-python.svg?style=svg&circle-token=b09aadca8e15901549cf885adcb8e1eaf671a5d8)](https://circleci.com/gh/segmentio/analytics-python) +[![CircleCI](https://circleci.com/gh/north-two-five/analytics-python.svg?style=svg&circle-token=7f225692265ab09f1dbe3f3a672efa137b1cfced)](https://circleci.com/gh/north-two-five/analytics-python) analytics-python is a python client for [Segment](https://segment.com) From 26784e73bfaf4f9e2d2188d726234c175990a133 Mon Sep 17 00:00:00 2001 From: Heitor Sampaio Date: Thu, 25 Mar 2021 15:51:14 +0100 Subject: [PATCH 173/323] Fix CircleCI Badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a226784..e73716fc 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ analytics-python ============== -[![CircleCI](https://circleci.com/gh/north-two-five/analytics-python.svg?style=svg&circle-token=7f225692265ab09f1dbe3f3a672efa137b1cfced)](https://circleci.com/gh/north-two-five/analytics-python) +[![CircleCI](https://circleci.com/gh/North-Two-Five/analytics-python.svg?style=svg&circle-token=7f225692265ab09f1dbe3f3a672efa137b1cfced)](https://circleci.com/gh/North-Two-Five/analytics-python) analytics-python is a python client for [Segment](https://segment.com) From 955c1d8465462e3632e93bb3eaecda5a5a844da0 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Tue, 6 Apr 2021 15:39:49 +0200 Subject: [PATCH 174/323] Add proxy support --- analytics/client.py | 7 ++++--- analytics/consumer.py | 5 +++-- analytics/request.py | 12 +++++++++++- analytics/test/client.py | 5 +++++ analytics/test/consumer.py | 10 ++++++++++ analytics/test/request.py | 9 +++++++++ 6 files changed, 42 insertions(+), 6 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 1b5bf8d0..1cfb0a60 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -28,7 +28,7 @@ class Client(object): def __init__(self, write_key=None, host=None, debug=False, max_queue_size=10000, send=True, on_error=None, flush_at=100, flush_interval=0.5, gzip=False, max_retries=3, - sync_mode=False, timeout=15, thread=1): + sync_mode=False, timeout=15, thread=1, proxies=None): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) @@ -40,6 +40,7 @@ def __init__(self, write_key=None, host=None, debug=False, self.host = host self.gzip = gzip self.timeout = timeout + self.proxies = proxies if debug: self.log.setLevel(logging.DEBUG) @@ -60,7 +61,7 @@ def __init__(self, write_key=None, host=None, debug=False, consumer = Consumer( self.queue, write_key, host=host, on_error=on_error, flush_at=flush_at, flush_interval=flush_interval, - gzip=gzip, retries=max_retries, timeout=timeout, + gzip=gzip, retries=max_retries, timeout=timeout, proxies=proxies, ) self.consumers.append(consumer) @@ -250,7 +251,7 @@ def _enqueue(self, msg): if self.sync_mode: self.log.debug('enqueued with blocking %s.', msg['type']) post(self.write_key, self.host, gzip=self.gzip, - timeout=self.timeout, batch=[msg]) + timeout=self.timeout, proxies=self.proxies, batch=[msg]) return True, msg diff --git a/analytics/consumer.py b/analytics/consumer.py index 1c3e7fd1..00137c4b 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -24,7 +24,7 @@ class Consumer(Thread): def __init__(self, queue, write_key, flush_at=100, host=None, on_error=None, flush_interval=0.5, gzip=False, retries=10, - timeout=15): + timeout=15, proxies=None): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit @@ -43,6 +43,7 @@ def __init__(self, queue, write_key, flush_at=100, host=None, self.running = True self.retries = retries self.timeout = timeout + self.proxies = proxies def run(self): """Runs the consumer.""" @@ -129,6 +130,6 @@ def fatal_exception(exc): giveup=fatal_exception) def send_request(): post(self.write_key, self.host, gzip=self.gzip, - timeout=self.timeout, batch=batch) + timeout=self.timeout, batch=batch, proxies=self.proxies) send_request() diff --git a/analytics/request.py b/analytics/request.py index 8477e46d..87b82f4f 100644 --- a/analytics/request.py +++ b/analytics/request.py @@ -13,7 +13,7 @@ _session = sessions.Session() -def post(write_key, host=None, gzip=False, timeout=15, **kwargs): +def post(write_key, host=None, gzip=False, timeout=15, proxies=None, **kwargs): """Post the `kwargs` to the API""" log = logging.getLogger('segment') body = kwargs @@ -35,6 +35,16 @@ def post(write_key, host=None, gzip=False, timeout=15, **kwargs): gz.write(data.encode('utf-8')) data = buf.getvalue() + kwargs = { + "data": data, + "auth": auth, + "headers": headers, + "timeout": 15, + } + + if proxies: + kwargs['proxies'] = proxies + res = _session.post(url, data=data, auth=auth, headers=headers, timeout=timeout) diff --git a/analytics/test/client.py b/analytics/test/client.py index 24e41041..4c38ba70 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -342,3 +342,8 @@ def test_default_timeout_15(self): client = Client('testsecret') for consumer in client.consumers: self.assertEqual(consumer.timeout, 15) + + def test_proxies(self): + client = Client('testsecret', proxies='203.243.63.16:80') + success, msg = client.identify('userId', {'trait': 'value'}) + self.assertTrue(success) \ No newline at end of file diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 91b3ca05..4b8f9f7b 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -197,3 +197,13 @@ def mock_post_fn(_, data, **kwargs): q.put(track) q.join() self.assertEqual(mock_post.call_count, 2) + + @classmethod + def test_proxies(cls): + consumer = Consumer(None, 'testsecret', proxies='203.243.63.16:80') + track = { + 'type': 'track', + 'event': 'python event', + 'userId': 'userId' + } + consumer.request([track]) diff --git a/analytics/test/request.py b/analytics/test/request.py index 3c739bca..b1c7756a 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -51,3 +51,12 @@ def test_should_timeout(self): 'event': 'python event', 'type': 'track' }], timeout=0.0001) + + def test_proxies(self): + res = post('testsecret',batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track', + 'proxies': '203.243.63.16:80' + }]) + self.assertEqual(res.status_code, 200) \ No newline at end of file From c9ac0cd2f39551e8b8ec6c9f71ff15e2ac74fb49 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Tue, 6 Apr 2021 15:46:34 +0200 Subject: [PATCH 175/323] Fix Flake8 --- analytics/client.py | 3 ++- analytics/test/client.py | 2 +- analytics/test/request.py | 14 +++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 1cfb0a60..8e903e43 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -61,7 +61,8 @@ def __init__(self, write_key=None, host=None, debug=False, consumer = Consumer( self.queue, write_key, host=host, on_error=on_error, flush_at=flush_at, flush_interval=flush_interval, - gzip=gzip, retries=max_retries, timeout=timeout, proxies=proxies, + gzip=gzip, retries=max_retries, timeout=timeout, + proxies=proxies, ) self.consumers.append(consumer) diff --git a/analytics/test/client.py b/analytics/test/client.py index 4c38ba70..ca307617 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -346,4 +346,4 @@ def test_default_timeout_15(self): def test_proxies(self): client = Client('testsecret', proxies='203.243.63.16:80') success, msg = client.identify('userId', {'trait': 'value'}) - self.assertTrue(success) \ No newline at end of file + self.assertTrue(success) diff --git a/analytics/test/request.py b/analytics/test/request.py index b1c7756a..3420deca 100644 --- a/analytics/test/request.py +++ b/analytics/test/request.py @@ -53,10 +53,10 @@ def test_should_timeout(self): }], timeout=0.0001) def test_proxies(self): - res = post('testsecret',batch=[{ - 'userId': 'userId', - 'event': 'python event', - 'type': 'track', - 'proxies': '203.243.63.16:80' - }]) - self.assertEqual(res.status_code, 200) \ No newline at end of file + res = post('testsecret', batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track', + 'proxies': '203.243.63.16:80' + }]) + self.assertEqual(res.status_code, 200) From e2817abb424d7a73ca29162ae8e1c39c3f93dd89 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Tue, 6 Apr 2021 15:48:18 +0200 Subject: [PATCH 176/323] Fix whitespace --- analytics/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analytics/client.py b/analytics/client.py index 8e903e43..712c32e3 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -61,7 +61,7 @@ def __init__(self, write_key=None, host=None, debug=False, consumer = Consumer( self.queue, write_key, host=host, on_error=on_error, flush_at=flush_at, flush_interval=flush_interval, - gzip=gzip, retries=max_retries, timeout=timeout, + gzip=gzip, retries=max_retries, timeout=timeout, proxies=proxies, ) self.consumers.append(consumer) From eb7fda44d4df2fc4e3144c4795bf9a6d568d3d1f Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 14 Apr 2021 13:27:46 +0200 Subject: [PATCH 177/323] Update to work with the new code, and fix some tests --- analytics/__init__.py | 22 +++++++----- analytics/client.py | 36 +++++++++++++++++--- analytics/test/__init__.py | 68 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 13 deletions(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 6589a965..92a3ea47 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -5,12 +5,16 @@ __version__ = VERSION """Settings.""" -write_key = None -host = None -on_error = None -debug = False -send = True -sync_mode = False +write_key = Client.DefaultConfig.write_key +host = Client.DefaultConfig.host +on_error = Client.DefaultConfig.on_error +debug = Client.DefaultConfig.debug +send = Client.DefaultConfig.send +sync_mode = Client.DefaultConfig.sync_mode +max_queue_size = Client.DefaultConfig.max_queue_size +gzip = Client.DefaultConfig.gzip +timeout = Client.DefaultConfig.timeout +max_retries = Client.DefaultConfig.max_retries default_client = None @@ -65,9 +69,9 @@ def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client if not default_client: - default_client = Client(write_key, host=host, debug=debug, - on_error=on_error, send=send, - sync_mode=sync_mode) + default_client = Client(write_key, host=host, debug=debug, max_queue_size=max_queue_size, + send=send, on_error=on_error, gzip=gzip, max_retries=max_retries, + sync_mode=sync_mode, timeout=timeout) fn = getattr(default_client, method) fn(*args, **kwargs) diff --git a/analytics/client.py b/analytics/client.py index 712c32e3..9cd6f225 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -22,13 +22,41 @@ class Client(object): + class DefaultConfig(object): + write_key = None + host = None + on_error = None + debug = False + send = True + sync_mode = False + max_queue_size = 10000 + gzip = False + timeout = 15 + max_retries = 10 + proxies = None + thread = 1 + flush_interval = 0.5 + flush_at = 100 + max_retries = 10 + """Create a new Segment client.""" log = logging.getLogger('segment') - def __init__(self, write_key=None, host=None, debug=False, - max_queue_size=10000, send=True, on_error=None, flush_at=100, - flush_interval=0.5, gzip=False, max_retries=3, - sync_mode=False, timeout=15, thread=1, proxies=None): + def __init__(self, + write_key=DefaultConfig.write_key, + host=DefaultConfig.host, + debug=DefaultConfig.debug, + max_queue_size=DefaultConfig.max_queue_size, + send=DefaultConfig.send, + on_error=DefaultConfig.on_error, + gzip=DefaultConfig.gzip, + max_retries=DefaultConfig.max_retries, + sync_mode=DefaultConfig.sync_mode, + timeout=DefaultConfig.timeout, + proxies=DefaultConfig.proxies, + thread=DefaultConfig.thread, + flush_at=DefaultConfig.flush_at, + flush_interval=DefaultConfig.flush_interval,): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) diff --git a/analytics/test/__init__.py b/analytics/test/__init__.py index 4a920f8e..09bf9b63 100644 --- a/analytics/test/__init__.py +++ b/analytics/test/__init__.py @@ -2,6 +2,9 @@ import pkgutil import logging import sys +import analytics + +from analytics.client import Client def all_names(): @@ -12,3 +15,68 @@ def all_names(): def all(): logging.basicConfig(stream=sys.stderr) return unittest.defaultTestLoader.loadTestsFromNames(all_names()) + + +class TestInit(unittest.TestCase): + def test_writeKey(self): + self.assertIsNone(analytics.default_client) + analytics.flush() + self.assertEqual(analytics.default_client.write_key, 'test-init') + + def test_debug(self): + self.assertIsNone(analytics.default_client) + analytics.debug = True + analytics.flush() + self.assertTrue(analytics.default_client.debug) + analytics.default_client = None + analytics.debug = False + analytics.flush() + self.assertFalse(analytics.default_client.debug) + + def test_gzip(self): + self.assertIsNone(analytics.default_client) + analytics.gzip = True + analytics.flush() + self.assertTrue(analytics.default_client.gzip) + analytics.default_client = None + analytics.gzip = False + analytics.flush() + self.assertFalse(analytics.default_client.gzip) + + def test_host(self): + self.assertIsNone(analytics.default_client) + analytics.host = 'test-host' + analytics.flush() + self.assertEqual(analytics.default_client.host, 'test-host') + + def test_max_queue_size(self): + self.assertIsNone(analytics.default_client) + analytics.max_queue_size = 1337 + analytics.flush() + self.assertEqual(analytics.default_client.queue.maxsize, 1337) + + def test_max_retries(self): + self.assertIsNone(analytics.default_client) + client = Client('testsecret', max_retries=42) + for consumer in client.consumers: + self.assertEqual(consumer.retries, 42) + + def test_sync_mode(self): + self.assertIsNone(analytics.default_client) + analytics.sync_mode = True + analytics.flush() + self.assertTrue(analytics.default_client.sync_mode) + analytics.default_client = None + analytics.sync_mode = False + analytics.flush() + self.assertFalse(analytics.default_client.sync_mode) + + def test_timeout(self): + self.assertIsNone(analytics.default_client) + analytics.timeout = 1.234 + analytics.flush() + self.assertEqual(analytics.default_client.timeout, 1.234) + + def setUp(self): + analytics.write_key = 'test-init' + analytics.default_client = None From 8f1eaab234fabcd6b1ebf3f8c58b805b7b42aac3 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 14 Apr 2021 13:30:57 +0200 Subject: [PATCH 178/323] Fix linting E501 --- analytics/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 92a3ea47..7dfac90b 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -69,8 +69,10 @@ def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client if not default_client: - default_client = Client(write_key, host=host, debug=debug, max_queue_size=max_queue_size, - send=send, on_error=on_error, gzip=gzip, max_retries=max_retries, + default_client = Client(write_key, host=host, debug=debug, + max_queue_size=max_queue_size, + send=send, on_error=on_error, + gzip=gzip, max_retries=max_retries, sync_mode=sync_mode, timeout=timeout) fn = getattr(default_client, method) From 544789ed5b233808538ffe78cbbd4b00fa8c8115 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 14 Apr 2021 13:31:54 +0200 Subject: [PATCH 179/323] Fix linting W291 --- analytics/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytics/__init__.py b/analytics/__init__.py index 7dfac90b..8c027ae1 100644 --- a/analytics/__init__.py +++ b/analytics/__init__.py @@ -69,9 +69,9 @@ def _proxy(method, *args, **kwargs): """Create an analytics client if one doesn't exist and send to it.""" global default_client if not default_client: - default_client = Client(write_key, host=host, debug=debug, + default_client = Client(write_key, host=host, debug=debug, max_queue_size=max_queue_size, - send=send, on_error=on_error, + send=send, on_error=on_error, gzip=gzip, max_retries=max_retries, sync_mode=sync_mode, timeout=timeout) From 1d378e060f13ae6ca4e0e466a0e9e38df47727b9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 14 Apr 2021 15:38:54 +0000 Subject: [PATCH 180/323] Create Dependabot config file --- .github/dependabot.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..7ef997ab --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 + reviewers: + - heitorsampaio From 4810d2f1348d4b75f3fefebf88f026217666f838 Mon Sep 17 00:00:00 2001 From: Brandon Sneed Date: Thu, 15 Apr 2021 12:13:44 -0700 Subject: [PATCH 181/323] Remove circleci config --- .circleci/config.yml | 144 ------------------------------------------- 1 file changed, 144 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index a00ce5d7..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,144 +0,0 @@ -version: 2 -defaults: - taggedReleasesFilter: &taggedReleasesFilter - tags: - only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. -jobs: - build: - docker: - - image: circleci/python:2.7 - steps: - - checkout - - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 - - run: make test - - persist_to_workspace: - root: . - paths: - - .coverage - - store_artifacts: - path: pylint.out - - store_artifacts: - path: flake8.out - - coverage: - docker: - - image: circleci/python:2.7 - steps: - - checkout - - attach_workspace: { at: . } - - run: sudo pip install coverage - - run: coverage report --show-missing | tee /tmp/coverage - - run: bash <(curl -s https://codecov.io/bash) - - store_artifacts: - path: /tmp/coverage - destination: test-coverage - - snyk: - docker: - - image: circleci/python:3.8 - steps: - - checkout - - attach_workspace: { at: . } - - run: pip install dephell - - run: dephell deps convert --from=setup.py --to=requirements.txt - - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh - - test_27: &test - docker: - - image: circleci/python:2.7 - steps: - - checkout - - run: pip install --user .[test] - - run: - name: Linting with Flake8 - command: | - git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - - run: make test - - run: make e2e_test - - test_35: - <<: *test - docker: - - image: circleci/python:3.5 - - test_36: - <<: *test - docker: - - image: circleci/python:3.6 - - test_37: - <<: *test - docker: - - image: circleci/python:3.7 - - test_38: - <<: *test - docker: - - image: circleci/python:3.8 - - publish: - docker: - - image: circleci/python:3.6 - steps: - - checkout - - run: sudo pip install twine - - run: make release - -workflows: - version: 2 - build_test_release: - jobs: - - build: - filters: - <<: *taggedReleasesFilter - - test_27: - filters: - <<: *taggedReleasesFilter - - test_35: - filters: - <<: *taggedReleasesFilter - - test_36: - filters: - <<: *taggedReleasesFilter - - test_37: - filters: - <<: *taggedReleasesFilter - - test_38: - filters: - <<: *taggedReleasesFilter - - publish: - requires: - - build - - test_27 - - test_35 - - test_36 - - test_37 - - test_38 - filters: - <<: *taggedReleasesFilter - branches: - ignore: /.*/ - static_analysis: - jobs: - - build - - coverage: - requires: - - build - - snyk: - context: snyk - requires: - - build - scheduled_e2e_test: - triggers: - - schedule: - cron: "0 * * * *" - filters: - branches: - only: - - master - - scheduled_e2e_testing - jobs: - - test_27 - - test_35 - - test_36 From a7e1c3f81619ea0ebdcf154c6e181bc492bb2359 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Fri, 23 Apr 2021 14:15:07 +0200 Subject: [PATCH 182/323] Release 1.3.0b2 --- HISTORY.md | 6 ++++++ analytics/version.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index c6a2268f..f89bf029 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,9 @@ +1.3.0-beta2 / 2020-04-23 +================== + * Fix linting code and readme heling basic things. + * Add support for HTTP proxy + * Allows more settings to be configured from singleton + 1.3.0-beta1 / 2019-04-27 ================== diff --git a/analytics/version.py b/analytics/version.py index e91d81c9..37c89ef2 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.3.0b1' +VERSION = '1.3.0b2' From 58e7a10fff56d2ae3c7549f7e55cb1110ed864fa Mon Sep 17 00:00:00 2001 From: pho3nix Date: Tue, 27 Apr 2021 07:50:21 -0300 Subject: [PATCH 183/323] Non beta version --- HISTORY.md | 2 +- analytics/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index f89bf029..5f317537 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,4 @@ -1.3.0-beta2 / 2020-04-23 +1.3.1 / 2021-04-23 ================== * Fix linting code and readme heling basic things. * Add support for HTTP proxy diff --git a/analytics/version.py b/analytics/version.py index 37c89ef2..d0e714d8 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.3.0b2' +VERSION = '1.3.1' From b97eabce6c5a5d794e5558f29ac7303979a6fefe Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 12 May 2021 18:36:43 -0300 Subject: [PATCH 184/323] CircleCI config --- .circleci/config.yml | 144 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..4c324094 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,144 @@ +version: 2 +defaults: + taggedReleasesFilter: &taggedReleasesFilter + tags: + only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. +jobs: + build: + docker: + - image: circleci/python:2.7 + steps: + - checkout + - run: pip install --user . + - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 + - run: make test + - persist_to_workspace: + root: . + paths: + - .coverage + - store_artifacts: + path: pylint.out + - store_artifacts: + path: flake8.out + + coverage: + docker: + - image: circleci/python:2.7 + steps: + - checkout + - attach_workspace: { at: . } + - run: sudo pip install coverage + - run: coverage report --show-missing | tee /tmp/coverage + - run: bash <(curl -s https://codecov.io/bash) + - store_artifacts: + path: /tmp/coverage + destination: test-coverage + + snyk: + docker: + - image: circleci/python:3.8 + steps: + - checkout + - attach_workspace: { at: . } + - run: pip install dephell + - run: dephell deps convert --from=setup.py --to=requirements.txt + - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh + + test_27: &test + docker: + - image: circleci/python:2.7 + steps: + - checkout + - run: pip install --user .[test] + - run: + name: Linting with Flake8 + command: | + git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics + - run: make test + - run: make e2e_test + + test_35: + <<: *test + docker: + - image: circleci/python:3.5 + + test_36: + <<: *test + docker: + - image: circleci/python:3.6 + + test_37: + <<: *test + docker: + - image: circleci/python:3.7 + + test_38: + <<: *test + docker: + - image: circleci/python:3.8 + + publish: + docker: + - image: circleci/python:3.6 + steps: + - checkout + - run: sudo pip install twine + - run: make release + +workflows: + version: 2 + build_test_release: + jobs: + - build: + filters: + <<: *taggedReleasesFilter + - test_27: + filters: + <<: *taggedReleasesFilter + - test_35: + filters: + <<: *taggedReleasesFilter + - test_36: + filters: + <<: *taggedReleasesFilter + - test_37: + filters: + <<: *taggedReleasesFilter + - test_38: + filters: + <<: *taggedReleasesFilter + - publish: + requires: + - build + - test_27 + - test_35 + - test_36 + - test_37 + - test_38 + filters: + <<: *taggedReleasesFilter + branches: + ignore: /.*/ + static_analysis: + jobs: + - build + - coverage: + requires: + - build + - snyk: + context: snyk + requires: + - build + scheduled_e2e_test: + triggers: + - schedule: + cron: "0 * * * *" + filters: + branches: + only: + - master + - scheduled_e2e_testing + jobs: + - test_27 + - test_35 + - test_36 \ No newline at end of file From 3c8bbb0aa64d42de7fcd12c22f7c7c3222fb2534 Mon Sep 17 00:00:00 2001 From: Heitor Sampaio Date: Wed, 12 May 2021 19:12:20 -0300 Subject: [PATCH 185/323] exclude codecov --- .circleci/config.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c324094..9925b3ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,19 +21,6 @@ jobs: - store_artifacts: path: flake8.out - coverage: - docker: - - image: circleci/python:2.7 - steps: - - checkout - - attach_workspace: { at: . } - - run: sudo pip install coverage - - run: coverage report --show-missing | tee /tmp/coverage - - run: bash <(curl -s https://codecov.io/bash) - - store_artifacts: - path: /tmp/coverage - destination: test-coverage - snyk: docker: - image: circleci/python:3.8 @@ -141,4 +128,4 @@ workflows: jobs: - test_27 - test_35 - - test_36 \ No newline at end of file + - test_36 From 1ba4de3cb9797fd3c8104448037350eebf6e6844 Mon Sep 17 00:00:00 2001 From: Heitor Sampaio Date: Wed, 12 May 2021 19:15:18 -0300 Subject: [PATCH 186/323] Fix remove codecov --- .circleci/config.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9925b3ae..94ccc958 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,12 +10,10 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install coverage pylint==1.9.3 flake8 mock==3.0.5 + - run: sudo pip install pylint==1.9.3 flake8 mock==3.0.5 - run: make test - persist_to_workspace: root: . - paths: - - .coverage - store_artifacts: path: pylint.out - store_artifacts: @@ -109,9 +107,6 @@ workflows: static_analysis: jobs: - build - - coverage: - requires: - - build - snyk: context: snyk requires: From f7584928e5037c69540a7506a58c60c637308622 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 12 May 2021 19:17:13 -0300 Subject: [PATCH 187/323] codecov remove --- Makefile | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 0a4584fe..a62ceec3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ test: pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out flake8 --max-complexity=10 --statistics analytics > flake8.out || true - coverage run --branch --include=analytics/\* --omit=*/test* setup.py test release: python setup.py sdist bdist_wheel diff --git a/setup.py b/setup.py index 8959db40..cb6a2ae7 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ "mock==2.0.0", "pylint==1.9.3", "flake8==3.7.9", - "coverage==4.5.4" ] setup( From 8d9f5e718a75741e000334984149cf0e1900e7c4 Mon Sep 17 00:00:00 2001 From: pho3nix Date: Wed, 12 May 2021 19:18:41 -0300 Subject: [PATCH 188/323] fix workspace --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 94ccc958..3d499305 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,8 +12,6 @@ jobs: - run: pip install --user . - run: sudo pip install pylint==1.9.3 flake8 mock==3.0.5 - run: make test - - persist_to_workspace: - root: . - store_artifacts: path: pylint.out - store_artifacts: From ab5c9188c21b16dc9b280f2d2123461bfec0c9bf Mon Sep 17 00:00:00 2001 From: Pooya Jaferian Date: Wed, 12 May 2021 15:43:28 -0700 Subject: [PATCH 189/323] Release 1.3.1 --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 5f317537..4b86bcf1 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,4 @@ -1.3.1 / 2021-04-23 +1.3.1 / 2021-05-12 ================== * Fix linting code and readme heling basic things. * Add support for HTTP proxy From 68e0d6a624ad555b208a2a5c1b1233194c6c944b Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Tue, 13 Jul 2021 17:00:14 -0500 Subject: [PATCH 190/323] Issue #152: Resolve missing config parameters upload_size and upload_interval --- analytics/client.py | 10 +++++----- analytics/consumer.py | 14 +++++++------- analytics/test/client.py | 4 ++-- analytics/test/consumer.py | 36 ++++++++++++++++++------------------ 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/analytics/client.py b/analytics/client.py index 9cd6f225..29b590f5 100644 --- a/analytics/client.py +++ b/analytics/client.py @@ -35,8 +35,8 @@ class DefaultConfig(object): max_retries = 10 proxies = None thread = 1 - flush_interval = 0.5 - flush_at = 100 + upload_interval = 0.5 + upload_size = 100 max_retries = 10 """Create a new Segment client.""" @@ -55,8 +55,8 @@ def __init__(self, timeout=DefaultConfig.timeout, proxies=DefaultConfig.proxies, thread=DefaultConfig.thread, - flush_at=DefaultConfig.flush_at, - flush_interval=DefaultConfig.flush_interval,): + upload_size=DefaultConfig.upload_size, + upload_interval=DefaultConfig.upload_interval,): require('write_key', write_key, string_types) self.queue = queue.Queue(max_queue_size) @@ -88,7 +88,7 @@ def __init__(self, self.consumers = [] consumer = Consumer( self.queue, write_key, host=host, on_error=on_error, - flush_at=flush_at, flush_interval=flush_interval, + upload_size=upload_size, upload_interval=upload_interval, gzip=gzip, retries=max_retries, timeout=timeout, proxies=proxies, ) diff --git a/analytics/consumer.py b/analytics/consumer.py index 00137c4b..cdcb16ff 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -22,15 +22,15 @@ class Consumer(Thread): """Consumes the messages from the client's queue.""" log = logging.getLogger('segment') - def __init__(self, queue, write_key, flush_at=100, host=None, - on_error=None, flush_interval=0.5, gzip=False, retries=10, + def __init__(self, queue, write_key, upload_size=100, host=None, + on_error=None, upload_interval=0.5, gzip=False, retries=10, timeout=15, proxies=None): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit self.daemon = True - self.flush_at = flush_at - self.flush_interval = flush_interval + self.upload_size = upload_size + self.upload_interval = upload_interval self.write_key = write_key self.host = host self.on_error = on_error @@ -86,13 +86,13 @@ def next(self): start_time = monotonic.monotonic() total_size = 0 - while len(items) < self.flush_at: + while len(items) < self.upload_size: elapsed = monotonic.monotonic() - start_time - if elapsed >= self.flush_interval: + if elapsed >= self.upload_interval: break try: item = queue.get( - block=True, timeout=self.flush_interval - elapsed) + block=True, timeout=self.upload_interval - elapsed) item_size = len(json.dumps( item, cls=DatetimeSerializer).encode()) if item_size > MAX_MSG_SIZE: diff --git a/analytics/test/client.py b/analytics/test/client.py index ca307617..ca45c238 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -317,9 +317,9 @@ def test_gzip(self): client.flush() self.assertFalse(self.failed) - def test_user_defined_flush_at(self): + def test_user_defined_upload_size(self): client = Client('testsecret', on_error=self.fail, - flush_at=10, flush_interval=3) + upload_size=10, upload_interval=3) def mock_post_fn(*args, **kwargs): self.assertEqual(len(kwargs['batch']), 10) diff --git a/analytics/test/consumer.py b/analytics/test/consumer.py index 4b8f9f7b..b7fcabaa 100644 --- a/analytics/test/consumer.py +++ b/analytics/test/consumer.py @@ -23,12 +23,12 @@ def test_next(self): def test_next_limit(self): q = Queue() - flush_at = 50 - consumer = Consumer(q, '', flush_at) + upload_size = 50 + consumer = Consumer(q, '', upload_size) for i in range(10000): q.put(i) next = consumer.next() - self.assertEqual(next, list(range(flush_at))) + self.assertEqual(next, list(range(upload_size))) def test_dropping_oversize_msg(self): q = Queue() @@ -51,14 +51,14 @@ def test_upload(self): success = consumer.upload() self.assertTrue(success) - def test_flush_interval(self): + def test_upload_interval(self): # Put _n_ items in the queue, pausing a little bit more than - # _flush_interval_ after each one. + # _upload_interval_ after each one. # The consumer should upload _n_ times. q = Queue() - flush_interval = 0.3 - consumer = Consumer(q, 'testsecret', flush_at=10, - flush_interval=flush_interval) + upload_interval = 0.3 + consumer = Consumer(q, 'testsecret', upload_size=10, + upload_interval=upload_interval) with mock.patch('analytics.consumer.post') as mock_post: consumer.start() for i in range(0, 3): @@ -68,27 +68,27 @@ def test_flush_interval(self): 'userId': 'userId' } q.put(track) - time.sleep(flush_interval * 1.1) + time.sleep(upload_interval * 1.1) self.assertEqual(mock_post.call_count, 3) def test_multiple_uploads_per_interval(self): - # Put _flush_at*2_ items in the queue at once, then pause for - # _flush_interval_. The consumer should upload 2 times. + # Put _upload_size*2_ items in the queue at once, then pause for + # _upload_interval_. The consumer should upload 2 times. q = Queue() - flush_interval = 0.5 - flush_at = 10 - consumer = Consumer(q, 'testsecret', flush_at=flush_at, - flush_interval=flush_interval) + upload_interval = 0.5 + upload_size = 10 + consumer = Consumer(q, 'testsecret', upload_size=upload_size, + upload_interval=upload_interval) with mock.patch('analytics.consumer.post') as mock_post: consumer.start() - for i in range(0, flush_at * 2): + for i in range(0, upload_size * 2): track = { 'type': 'track', 'event': 'python event %d' % i, 'userId': 'userId' } q.put(track) - time.sleep(flush_interval * 1.1) + time.sleep(upload_interval * 1.1) self.assertEqual(mock_post.call_count, 2) @classmethod @@ -172,7 +172,7 @@ def test_pause(self): def test_max_batch_size(self): q = Queue() consumer = Consumer( - q, 'testsecret', flush_at=100000, flush_interval=3) + q, 'testsecret', upload_size=100000, upload_interval=3) track = { 'type': 'track', 'event': 'python event', From bf7168d7cb9bed5b1599928bca4d83fd468f31ce Mon Sep 17 00:00:00 2001 From: Pooya Jaferian Date: Fri, 16 Jul 2021 09:34:17 -0700 Subject: [PATCH 191/323] Release 1.4.0 --- HISTORY.md | 189 ++++++++++++++++++++----------------------- analytics/version.py | 2 +- 2 files changed, 87 insertions(+), 104 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 4b86bcf1..e973f3c8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,142 +1,125 @@ -1.3.1 / 2021-05-12 -================== - * Fix linting code and readme heling basic things. - * Add support for HTTP proxy - * Allows more settings to be configured from singleton - -1.3.0-beta1 / 2019-04-27 -================== - - * Add `sync_mode` option ([#147](https://github.com/segmentio/analytics-python/pull/147)) +# 1.4.0 / 2021-07-16 -1.3.0-beta0 / 2018-10-10 -================== +- Fix the missing `upload_size` parameter - * Add User-Agent header to messages - * Don't retry sending on client errors except 429 - * Allow user-defined upload interval - * Add `shutdown` function - * Add gzip support - * Add exponential backoff with jitter when retrying - * Add a paramater in Client to configure max retries - * Limit batch upload size to 500KB - * Drop messages greater than 32kb - * Allow user-defined upload size - * Support custom messageId +# 1.3.1 / 2021-05-12 -1.2.9 / 2017-11-28 -================== +- Fix linting code and readme heling basic things. +- Add support for HTTP proxy +- Allows more settings to be configured from singleton - * [Fix](https://github.com/segmentio/analytics-python/pull/102): Stringify non-string userIds and anonymousIds. +# 1.3.0-beta1 / 2019-04-27 -1.2.8 / 2017-09-20 -================== +- Add `sync_mode` option ([#147](https://github.com/segmentio/analytics-python/pull/147)) - * [Fix](https://github.com/segmentio/analytics-python/issues/94): Date objects are removed from event properties. - * [Fix](https://github.com/segmentio/analytics-python/pull/98): Fix for regression introduced in version 1.2.4. +# 1.3.0-beta0 / 2018-10-10 -1.2.7 / 2017-01-31 -================== +- Add User-Agent header to messages +- Don't retry sending on client errors except 429 +- Allow user-defined upload interval +- Add `shutdown` function +- Add gzip support +- Add exponential backoff with jitter when retrying +- Add a paramater in Client to configure max retries +- Limit batch upload size to 500KB +- Drop messages greater than 32kb +- Allow user-defined upload size +- Support custom messageId - * [Fix](https://github.com/segmentio/analytics-python/pull/92): Correctly serialize date objects. +# 1.2.9 / 2017-11-28 -1.2.6 / 2016-12-07 -================== +- [Fix](https://github.com/segmentio/analytics-python/pull/102): Stringify non-string userIds and anonymousIds. - * dont add messages to the queue if send is false - * drop py32 support +# 1.2.8 / 2017-09-20 -1.2.5 / 2016-07-02 -================== +- [Fix](https://github.com/segmentio/analytics-python/issues/94): Date objects are removed from event properties. +- [Fix](https://github.com/segmentio/analytics-python/pull/98): Fix for regression introduced in version 1.2.4. - * Fix outdated python-dateutil<2 requirement for python2 - dateutil > 2.1 runs is python2 compatible - * Fix a bug introduced in 1.2.4 where we could try to join a thread that was not yet started +# 1.2.7 / 2017-01-31 -1.2.4 / 2016-06-06 -================== +- [Fix](https://github.com/segmentio/analytics-python/pull/92): Correctly serialize date objects. - * Fix race conditions in overflow and flush tests - * Join daemon thread on interpreter exit to prevent value errors - * Capitalize HISTORY.md (#76) - * Quick fix for Decimal to send as a float +# 1.2.6 / 2016-12-07 -1.2.3 / 2016-03-23 -================== +- dont add messages to the queue if send is false +- drop py32 support - * relaxing requests dep +# 1.2.5 / 2016-07-02 -1.2.2 / 2016-03-17 -================== +- Fix outdated python-dateutil<2 requirement for python2 - dateutil > 2.1 runs is python2 compatible +- Fix a bug introduced in 1.2.4 where we could try to join a thread that was not yet started - * Fix environment markers definition - * Use proper way for defining conditional dependencies +# 1.2.4 / 2016-06-06 -1.2.1 / 2016-03-11 -================== +- Fix race conditions in overflow and flush tests +- Join daemon thread on interpreter exit to prevent value errors +- Capitalize HISTORY.md (#76) +- Quick fix for Decimal to send as a float - * fixing requirements.txt +# 1.2.3 / 2016-03-23 -1.2.0 / 2016-03-11 -================== +- relaxing requests dep - * adding versioned requirements.txt file +# 1.2.2 / 2016-03-17 -1.1.0 / 2015-06-23 -================== +- Fix environment markers definition +- Use proper way for defining conditional dependencies - * Adding fixes for handling invalid json types - * Fixing byte/bytearray handling - * Adding `logging.DEBUG` fix for `setLevel` - * Support HTTP keep-alive using a Session connection pool - * Suppport universal wheels - * adding .sentAt - * make it really testable - * fixing overflow test - * removing .io's - * Update README.md - * spacing +# 1.2.1 / 2016-03-11 -1.0.3 / 2014-09-30 -================== +- fixing requirements.txt - * adding top level send option +# 1.2.0 / 2016-03-11 -1.0.2 / 2014-09-17 -================== +- adding versioned requirements.txt file - * fixing debug logging levels +# 1.1.0 / 2015-06-23 +- Adding fixes for handling invalid json types +- Fixing byte/bytearray handling +- Adding `logging.DEBUG` fix for `setLevel` +- Support HTTP keep-alive using a Session connection pool +- Suppport universal wheels +- adding .sentAt +- make it really testable +- fixing overflow test +- removing .io's +- Update README.md +- spacing -1.0.1 / 2014-09-08 -================== +# 1.0.3 / 2014-09-30 - * fixing unicode handling, for write_key and events - * adding six to requirements.txt and install scripts +- adding top level send option -1.0.0 / 2014-09-05 -================== +# 1.0.2 / 2014-09-17 - * updating to spec 1.0 - * adding python3 support - * moving to analytics.write_key API - * moving consumer to a separate thread - * adding request retries - * making analytics.flush() syncrhonous - * adding full travis tests +- fixing debug logging levels -0.4.4 / 2013-11-21 -================== +# 1.0.1 / 2014-09-08 - * add < python 2.7 compatibility by removing `delta.total_seconds` +- fixing unicode handling, for write_key and events +- adding six to requirements.txt and install scripts -0.4.3 / 2013-11-13 -================== +# 1.0.0 / 2014-09-05 - * added datetime serialization fix (alexlouden) +- updating to spec 1.0 +- adding python3 support +- moving to analytics.write_key API +- moving consumer to a separate thread +- adding request retries +- making analytics.flush() syncrhonous +- adding full travis tests -0.4.2 / 2013-06-26 -================== +# 0.4.4 / 2013-11-21 - * Added history.d change log - * Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! - * Fixing #12, adding static public API to analytics.__init__ +- add < python 2.7 compatibility by removing `delta.total_seconds` + +# 0.4.3 / 2013-11-13 + +- added datetime serialization fix (alexlouden) + +# 0.4.2 / 2013-06-26 + +- Added history.d change log +- Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! +- Fixing #12, adding static public API to analytics.**init** diff --git a/analytics/version.py b/analytics/version.py index d0e714d8..b2e81777 100644 --- a/analytics/version.py +++ b/analytics/version.py @@ -1 +1 @@ -VERSION = '1.3.1' +VERSION = '1.4.0' From 917ae818a0214ecfd761f6a215c24a216d4fcd70 Mon Sep 17 00:00:00 2001 From: Samuel Dion-Girardeau Date: Fri, 13 Aug 2021 15:45:03 -0400 Subject: [PATCH 192/323] Unpin backoff dependency (semver-compatibly) Closes https://github.com/segmentio/analytics-python/issues/188 This means users will be able to update the backoff dependency without conflicting with the segment analytics library's requirements. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cb6a2ae7..58e16201 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ "requests>=2.7,<3.0", "six>=1.5", "monotonic>=1.5", - "backoff==1.10.0", + "backoff~=1.10", "python-dateutil>2.1" ] From 12fa7618e144327cfee9d70af9b8e9c4e7fd0fff Mon Sep 17 00:00:00 2001 From: Samuel Dion-Girardeau Date: Fri, 13 Aug 2021 15:48:26 -0400 Subject: [PATCH 193/323] Use semver-compatibility operator everywhere This will prevent the library from accidentally installing a possibly backwards-incompatible version. For `requests`, the check is equivalent to >=2.7,<3.0 For `six` and `monotonic` it adds a constraint to prevent major version bumps For `python-dateutil`, the check isn't theoritcally equivalent, but in practice the only release after 2.1 is 2.2 so it should work the same. --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 58e16201..3bd96a6b 100644 --- a/setup.py +++ b/setup.py @@ -21,11 +21,11 @@ ''' install_requires = [ - "requests>=2.7,<3.0", - "six>=1.5", - "monotonic>=1.5", + "requests~=2.7", + "six~=1.5", + "monotonic~=1.5", "backoff~=1.10", - "python-dateutil>2.1" + "python-dateutil~=2.2" ] tests_require = [ From 153df108a7a8d714b21b42d38bbb0ed44d6907c5 Mon Sep 17 00:00:00 2001 From: Samuel Dion-Girardeau Date: Fri, 13 Aug 2021 16:08:21 -0400 Subject: [PATCH 194/323] Add CI tests for Python 3.9 --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d499305..edb1b37b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,6 +60,11 @@ jobs: docker: - image: circleci/python:3.8 + test_39: + <<: *test + docker: + - image: circleci/python:3.9 + publish: docker: - image: circleci/python:3.6 @@ -90,6 +95,9 @@ workflows: - test_38: filters: <<: *taggedReleasesFilter + - test_39: + filters: + <<: *taggedReleasesFilter - publish: requires: - build @@ -98,6 +106,7 @@ workflows: - test_36 - test_37 - test_38 + - test_39 filters: <<: *taggedReleasesFilter branches: From f0b0c033d54eb4d74e6a3fa2bbe2370617c93019 Mon Sep 17 00:00:00 2001 From: Samuel Dion-Girardeau Date: Fri, 13 Aug 2021 16:09:12 -0400 Subject: [PATCH 195/323] Add latest Python 3 version to e2e test Possibly a bad idea -- not sure why they're excluded --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index edb1b37b..73cff7b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,3 +131,6 @@ workflows: - test_27 - test_35 - test_36 + - test_37 + - test_38 + - test_39 From f71db5fd718f4a2f7c4117031f07f36e460ff797 Mon Sep 17 00:00:00 2001 From: Samuel Dion-Girardeau Date: Fri, 13 Aug 2021 16:11:46 -0400 Subject: [PATCH 196/323] Add Python 3.9 to project's Trove classifiers --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index cb6a2ae7..01f6e872 100644 --- a/setup.py +++ b/setup.py @@ -67,5 +67,6 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ], ) From 2085498b4395fefa620d281275617e3843fe1dbc Mon Sep 17 00:00:00 2001 From: Samuel Dion-Girardeau Date: Fri, 13 Aug 2021 16:14:38 -0400 Subject: [PATCH 197/323] Use latest Python image for publish job Python 3.6 end-of-support is in December 2021, let's upgrade ahead of time! --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 73cff7b9..625b1b3a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -67,7 +67,7 @@ jobs: publish: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.9 steps: - checkout - run: sudo pip install twine From ba0d80d855cec98c365824f13fb4753aaa8d924e Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 8 Sep 2021 09:28:08 -0500 Subject: [PATCH 198/323] Update Readme Show badge for proper CI build --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e73716fc..0a2b1d28 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ analytics-python ============== -[![CircleCI](https://circleci.com/gh/North-Two-Five/analytics-python.svg?style=svg&circle-token=7f225692265ab09f1dbe3f3a672efa137b1cfced)](https://circleci.com/gh/North-Two-Five/analytics-python) +[![CircleCI](https://circleci.com/gh/segmentio/analytics-python/tree/master.svg?style=svg&circle-token=c0b411a3e21943918294714ad1d75a1cfc718f79)](https://circleci.com/gh/segmentio/analytics-python/tree/master) + analytics-python is a python client for [Segment](https://segment.com) From 81439946ef69af7ce1359867a3d936210a0769e8 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 15:08:43 -0500 Subject: [PATCH 199/323] Mods to tests and pylint settings --- .pylintrc | 6 ++++-- analytics/consumer.py | 3 ++- analytics/test/client.py | 10 ++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.pylintrc b/.pylintrc index 0d41c235..3d6d2d0c 100644 --- a/.pylintrc +++ b/.pylintrc @@ -49,7 +49,9 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=print-statement, +disable=too-many-public-methods, + no-else-return, + print-statement, invalid-name, global-statement, too-many-arguments, @@ -466,7 +468,7 @@ max-bool-expr=5 max-branches=12 # Maximum number of locals for function / method body -max-locals=15 +max-locals=20 # Maximum number of parents for a class (see R0901). max-parents=7 diff --git a/analytics/consumer.py b/analytics/consumer.py index 00137c4b..5be6e1a1 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -1,8 +1,9 @@ import logging from threading import Thread +import json import monotonic import backoff -import json + from analytics.request import post, APIError, DatetimeSerializer diff --git a/analytics/test/client.py b/analytics/test/client.py index ca307617..5e305861 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -10,7 +10,7 @@ class TestClient(unittest.TestCase): - def fail(self, e, batch): + def fail(self): """Mark the failure handler""" self.failed = True @@ -285,17 +285,11 @@ def test_success_on_invalid_write_key(self): client.flush() self.assertFalse(self.failed) - def test_unicode(self): - Client(six.u('unicode_key')) - def test_numeric_user_id(self): self.client.track(1234, 'python event') self.client.flush() self.assertFalse(self.failed) - def test_debug(self): - Client('bad_key', debug=True) - def test_identify_with_date_object(self): client = self.client success, msg = client.identify( @@ -321,7 +315,7 @@ def test_user_defined_flush_at(self): client = Client('testsecret', on_error=self.fail, flush_at=10, flush_interval=3) - def mock_post_fn(*args, **kwargs): + def mock_post_fn(**kwargs): self.assertEqual(len(kwargs['batch']), 10) # the post function should be called 2 times, with a batch size of 10 From 7072cef7dec18c2439473d14fcb4246b238a85bc Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 17:33:39 -0500 Subject: [PATCH 200/323] add dependencies for snyk --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cb6a2ae7..79c03f13 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,8 @@ "six>=1.5", "monotonic>=1.5", "backoff==1.10.0", - "python-dateutil>2.1" + "python-dateutil>2.1", + "python-appdirs" ] tests_require = [ From 0a35cb94537d63fb80d0b178a70eb0b31b147151 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 17:36:50 -0500 Subject: [PATCH 201/323] updated dep name --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 79c03f13..510ed15d 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ "monotonic>=1.5", "backoff==1.10.0", "python-dateutil>2.1", - "python-appdirs" + "appdirs" ] tests_require = [ From ece102b0c8fca2633da943989c5bd69fc3bd31a9 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 17:42:36 -0500 Subject: [PATCH 202/323] update pip to pip3 for build --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d499305..22b5f62d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,7 @@ jobs: steps: - checkout - attach_workspace: { at: . } - - run: pip install dephell + - run: pip3 install dephell - run: dephell deps convert --from=setup.py --to=requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh From 98a8f87c4f86803f300383a5a0cc97a0e2ea76ee Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:00:12 -0500 Subject: [PATCH 203/323] update settings --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22b5f62d..ac349d0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,7 @@ jobs: - checkout - attach_workspace: { at: . } - run: pip3 install dephell + - run: pip3 install appdirs - run: dephell deps convert --from=setup.py --to=requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh From 80131af51ef99a8d9c1212d018a64c7bd3b6bf32 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:01:53 -0500 Subject: [PATCH 204/323] config update --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac349d0d..27fffae7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ jobs: - checkout - attach_workspace: { at: . } - run: pip3 install dephell - - run: pip3 install appdirs + - run: pip3 install --user appdirs - run: dephell deps convert --from=setup.py --to=requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh From a293f8c56c4f4ea1b5c0106e9001953c2005b83a Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:04:23 -0500 Subject: [PATCH 205/323] requirements installations for dephell --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27fffae7..deefc974 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,6 +26,7 @@ jobs: - run: pip3 install dephell - run: pip3 install --user appdirs - run: dephell deps convert --from=setup.py --to=requirements.txt + - run: pip3 install -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh test_27: &test From 68e440c56e94e109a737303d3ffed1457ce31627 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:07:36 -0500 Subject: [PATCH 206/323] config updates --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index deefc974..7a611739 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,10 +23,9 @@ jobs: steps: - checkout - attach_workspace: { at: . } - - run: pip3 install dephell - - run: pip3 install --user appdirs + - run: pip3 install dephell - run: dephell deps convert --from=setup.py --to=requirements.txt - - run: pip3 install -r requirements.txt + - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh test_27: &test From ffa0450e9ebdf415de77d9b208af619192e9e51d Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:10:04 -0500 Subject: [PATCH 207/323] req update --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a611739..e2b06b16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,7 +23,8 @@ jobs: steps: - checkout - attach_workspace: { at: . } - - run: pip3 install dephell + - run: pip3 install dephell + - run: pip3 install --user appdirs - run: dephell deps convert --from=setup.py --to=requirements.txt - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh From fc195c84a38621f4efe390d0d6f8149b62aa56ee Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:17:01 -0500 Subject: [PATCH 208/323] update pylint per ci recommendations --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2b06b16..e0782d81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install pylint==1.9.3 flake8 mock==3.0.5 + - run: sudo pip install pylint==2.7.0 flake8 mock==3.0.5 - run: make test - store_artifacts: path: pylint.out From 142d9d98d36e81a5d5383586eefc7c38c78c8144 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 20 Sep 2021 18:18:41 -0500 Subject: [PATCH 209/323] update pylint version for correct python --- .circleci/config.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e0782d81..e2b06b16 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip install pylint==2.7.0 flake8 mock==3.0.5 + - run: sudo pip install pylint==1.9.3 flake8 mock==3.0.5 - run: make test - store_artifacts: path: pylint.out diff --git a/setup.py b/setup.py index 510ed15d..7e3e721e 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ tests_require = [ "mock==2.0.0", - "pylint==1.9.3", + "pylint==2.7.0", "flake8==3.7.9", ] From 81f7d31267eb94ef62a8a8ca9ec82023de62864d Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 23 Sep 2021 20:23:46 -0500 Subject: [PATCH 210/323] Upgrade pylint, the new version fixes all issues with tests, but we had to remove 2.7 and 3.5; update history and version since this is a breaking fix --- .circleci/config.yml | 32 +++++++------------------------- HISTORY.md | 5 +++++ setup.py | 9 +-------- 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e2b06b16..2acf8baa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,11 +6,11 @@ defaults: jobs: build: docker: - - image: circleci/python:2.7 + - image: circleci/python:3.8 steps: - checkout - run: pip install --user . - - run: sudo pip install pylint==1.9.3 flake8 mock==3.0.5 + - run: sudo pip3 install pylint==2.7.8 flake8 mock==3.0.5 - run: make test - store_artifacts: path: pylint.out @@ -29,9 +29,9 @@ jobs: - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh - test_27: &test + test_36: &test docker: - - image: circleci/python:2.7 + - image: circleci/python:3.6 steps: - checkout - run: pip install --user .[test] @@ -41,16 +41,6 @@ jobs: git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - run: make test - run: make e2e_test - - test_35: - <<: *test - docker: - - image: circleci/python:3.5 - - test_36: - <<: *test - docker: - - image: circleci/python:3.6 test_37: <<: *test @@ -77,12 +67,6 @@ workflows: - build: filters: <<: *taggedReleasesFilter - - test_27: - filters: - <<: *taggedReleasesFilter - - test_35: - filters: - <<: *taggedReleasesFilter - test_36: filters: <<: *taggedReleasesFilter @@ -95,8 +79,6 @@ workflows: - publish: requires: - build - - test_27 - - test_35 - test_36 - test_37 - test_38 @@ -121,6 +103,6 @@ workflows: - master - scheduled_e2e_testing jobs: - - test_27 - - test_35 - - test_36 + - test_36 + - test_37 + - test_38 \ No newline at end of file diff --git a/HISTORY.md b/HISTORY.md index 4b86bcf1..74813c1e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,8 @@ +1.4.0 / 2021-09-23 +================== + * Update tests with latest dependencies + * Remove unsupported python versions 2.7 & 3.5 + 1.3.1 / 2021-05-12 ================== * Fix linting code and readme heling basic things. diff --git a/setup.py b/setup.py index 7e3e721e..7f14146f 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ tests_require = [ "mock==2.0.0", - "pylint==2.7.0", + "pylint==2.7.8", "flake8==3.7.9", ] @@ -58,13 +58,6 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.2", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", From fbab73043b9ccdc3ac0370052b2baa9a20cc585a Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 23 Sep 2021 20:25:14 -0500 Subject: [PATCH 211/323] Fix pylint version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2acf8baa..27d4eb5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip install --user . - - run: sudo pip3 install pylint==2.7.8 flake8 mock==3.0.5 + - run: sudo pip3 install pylint==2.7.0 flake8 mock==3.0.5 - run: make test - store_artifacts: path: pylint.out From 1464b551b2f81f528ea9a12cdaa69f3951f2a7c2 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 23 Sep 2021 20:26:15 -0500 Subject: [PATCH 212/323] Update pip version --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 27d4eb5e..12e07f99 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,8 @@ jobs: - image: circleci/python:3.8 steps: - checkout - - run: pip install --user . - - run: sudo pip3 install pylint==2.7.0 flake8 mock==3.0.5 + - run: pip3 install --user . + - run: sudo pip3 install pylint==2.7.8 flake8 mock==3.0.5 - run: make test - store_artifacts: path: pylint.out From d7da29f0e7bc9b2d7264ba5a2d03dfa0ea8759d3 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 23 Sep 2021 20:27:21 -0500 Subject: [PATCH 213/323] Update pip to pip3 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12e07f99..f26bb886 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ jobs: - image: circleci/python:3.6 steps: - checkout - - run: pip install --user .[test] + - run: pip3 install --user .[test] - run: name: Linting with Flake8 command: | From 52458ef8a034e6b545a992f88b9b812d47524743 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 23 Sep 2021 20:28:52 -0500 Subject: [PATCH 214/323] Update pylint version --- .circleci/config.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f26bb886..9b0486dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip3 install --user . - - run: sudo pip3 install pylint==2.7.8 flake8 mock==3.0.5 + - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 - run: make test - store_artifacts: path: pylint.out diff --git a/setup.py b/setup.py index 7f14146f..605f4128 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ tests_require = [ "mock==2.0.0", - "pylint==2.7.8", + "pylint==2.8.0", "flake8==3.7.9", ] From 010a238b7c8453c20352444b103bc65bb2ccbca3 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Wed, 29 Sep 2021 13:30:45 -0500 Subject: [PATCH 215/323] Issue #154 --- analytics/version.py | 1 - library-e2e-tester | 1 - {analytics => segment/analytics}/__init__.py | 4 +- {analytics => segment/analytics}/client.py | 0 segment/analytics/consumer.py | 135 +++++++ {analytics => segment/analytics}/request.py | 0 .../analytics}/test/__init__.py | 0 segment/analytics/test/client.py | 349 ++++++++++++++++++ .../analytics}/test/consumer.py | 0 .../analytics}/test/module.py | 0 .../analytics}/test/request.py | 0 .../analytics}/test/utils.py | 0 {analytics => segment/analytics}/utils.py | 0 segment/analytics/version.py | 1 + 14 files changed, 487 insertions(+), 4 deletions(-) delete mode 100644 analytics/version.py delete mode 160000 library-e2e-tester rename {analytics => segment/analytics}/__init__.py (95%) rename {analytics => segment/analytics}/client.py (100%) create mode 100644 segment/analytics/consumer.py rename {analytics => segment/analytics}/request.py (100%) rename {analytics => segment/analytics}/test/__init__.py (100%) create mode 100644 segment/analytics/test/client.py rename {analytics => segment/analytics}/test/consumer.py (100%) rename {analytics => segment/analytics}/test/module.py (100%) rename {analytics => segment/analytics}/test/request.py (100%) rename {analytics => segment/analytics}/test/utils.py (100%) rename {analytics => segment/analytics}/utils.py (100%) create mode 100644 segment/analytics/version.py diff --git a/analytics/version.py b/analytics/version.py deleted file mode 100644 index b2e81777..00000000 --- a/analytics/version.py +++ /dev/null @@ -1 +0,0 @@ -VERSION = '1.4.0' diff --git a/library-e2e-tester b/library-e2e-tester deleted file mode 160000 index 5e176853..00000000 --- a/library-e2e-tester +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5e176853ceca14dd656bb8e2187b79adab3216ff diff --git a/analytics/__init__.py b/segment/analytics/__init__.py similarity index 95% rename from analytics/__init__.py rename to segment/analytics/__init__.py index 8c027ae1..eda9deb4 100644 --- a/analytics/__init__.py +++ b/segment/analytics/__init__.py @@ -1,6 +1,6 @@ -from analytics.version import VERSION -from analytics.client import Client +from segment.analytics.version import VERSION +from segment.analytics.client import Client __version__ = VERSION diff --git a/analytics/client.py b/segment/analytics/client.py similarity index 100% rename from analytics/client.py rename to segment/analytics/client.py diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py new file mode 100644 index 00000000..cdcb16ff --- /dev/null +++ b/segment/analytics/consumer.py @@ -0,0 +1,135 @@ +import logging +from threading import Thread +import monotonic +import backoff +import json + +from analytics.request import post, APIError, DatetimeSerializer + +try: + from queue import Empty +except ImportError: + from Queue import Empty + +MAX_MSG_SIZE = 32 << 10 + +# Our servers only accept batches less than 500KB. Here limit is set slightly +# lower to leave space for extra data that will be added later, eg. "sentAt". +BATCH_SIZE_LIMIT = 475000 + + +class Consumer(Thread): + """Consumes the messages from the client's queue.""" + log = logging.getLogger('segment') + + def __init__(self, queue, write_key, upload_size=100, host=None, + on_error=None, upload_interval=0.5, gzip=False, retries=10, + timeout=15, proxies=None): + """Create a consumer thread.""" + Thread.__init__(self) + # Make consumer a daemon thread so that it doesn't block program exit + self.daemon = True + self.upload_size = upload_size + self.upload_interval = upload_interval + self.write_key = write_key + self.host = host + self.on_error = on_error + self.queue = queue + self.gzip = gzip + # It's important to set running in the constructor: if we are asked to + # pause immediately after construction, we might set running to True in + # run() *after* we set it to False in pause... and keep running + # forever. + self.running = True + self.retries = retries + self.timeout = timeout + self.proxies = proxies + + def run(self): + """Runs the consumer.""" + self.log.debug('consumer is running...') + while self.running: + self.upload() + + self.log.debug('consumer exited.') + + def pause(self): + """Pause the consumer.""" + self.running = False + + def upload(self): + """Upload the next batch of items, return whether successful.""" + success = False + batch = self.next() + if len(batch) == 0: + return False + + try: + self.request(batch) + success = True + except Exception as e: + self.log.error('error uploading: %s', e) + success = False + if self.on_error: + self.on_error(e, batch) + finally: + # mark items as acknowledged from queue + for _ in batch: + self.queue.task_done() + return success + + def next(self): + """Return the next batch of items to upload.""" + queue = self.queue + items = [] + + start_time = monotonic.monotonic() + total_size = 0 + + while len(items) < self.upload_size: + elapsed = monotonic.monotonic() - start_time + if elapsed >= self.upload_interval: + break + try: + item = queue.get( + block=True, timeout=self.upload_interval - elapsed) + item_size = len(json.dumps( + item, cls=DatetimeSerializer).encode()) + if item_size > MAX_MSG_SIZE: + self.log.error( + 'Item exceeds 32kb limit, dropping. (%s)', str(item)) + continue + items.append(item) + total_size += item_size + if total_size >= BATCH_SIZE_LIMIT: + self.log.debug( + 'hit batch size limit (size: %d)', total_size) + break + except Empty: + break + + return items + + def request(self, batch): + """Attempt to upload the batch and retry before raising an error """ + + def fatal_exception(exc): + if isinstance(exc, APIError): + # retry on server errors and client errors + # with 429 status code (rate limited), + # don't retry on other client errors + return (400 <= exc.status < 500) and exc.status != 429 + else: + # retry on all other errors (eg. network) + return False + + @backoff.on_exception( + backoff.expo, + Exception, + max_tries=self.retries + 1, + giveup=fatal_exception) + def send_request(): + post(self.write_key, self.host, gzip=self.gzip, + timeout=self.timeout, batch=batch, proxies=self.proxies) + + send_request() diff --git a/analytics/request.py b/segment/analytics/request.py similarity index 100% rename from analytics/request.py rename to segment/analytics/request.py diff --git a/analytics/test/__init__.py b/segment/analytics/test/__init__.py similarity index 100% rename from analytics/test/__init__.py rename to segment/analytics/test/__init__.py diff --git a/segment/analytics/test/client.py b/segment/analytics/test/client.py new file mode 100644 index 00000000..ca45c238 --- /dev/null +++ b/segment/analytics/test/client.py @@ -0,0 +1,349 @@ +from datetime import date, datetime +import unittest +import time +import six +import mock + +from analytics.version import VERSION +from analytics.client import Client + + +class TestClient(unittest.TestCase): + + def fail(self, e, batch): + """Mark the failure handler""" + self.failed = True + + def setUp(self): + self.failed = False + self.client = Client('testsecret', on_error=self.fail) + + def test_requires_write_key(self): + self.assertRaises(AssertionError, Client) + + def test_empty_flush(self): + self.client.flush() + + def test_basic_track(self): + client = self.client + success, msg = client.track('userId', 'python test event') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['event'], 'python test event') + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['properties'], {}) + self.assertEqual(msg['type'], 'track') + + def test_stringifies_user_id(self): + # A large number that loses precision in node: + # node -e "console.log(157963456373623802 + 1)" > 157963456373623800 + client = self.client + success, msg = client.track( + user_id=157963456373623802, event='python test event') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['userId'], '157963456373623802') + self.assertEqual(msg['anonymousId'], None) + + def test_stringifies_anonymous_id(self): + # A large number that loses precision in node: + # node -e "console.log(157963456373623803 + 1)" > 157963456373623800 + client = self.client + success, msg = client.track( + anonymous_id=157963456373623803, event='python test event') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['userId'], None) + self.assertEqual(msg['anonymousId'], '157963456373623803') + + def test_advanced_track(self): + client = self.client + success, msg = client.track( + 'userId', 'python test event', {'property': 'value'}, + {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', + {'Amplitude': True}, 'messageId') + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['properties'], {'property': 'value'}) + self.assertEqual(msg['integrations'], {'Amplitude': True}) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['event'], 'python test event') + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertEqual(msg['messageId'], 'messageId') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'track') + + def test_basic_identify(self): + client = self.client + success, msg = client.identify('userId', {'trait': 'value'}) + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['traits'], {'trait': 'value'}) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertTrue(isinstance(msg['messageId'], str)) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'identify') + + def test_advanced_identify(self): + client = self.client + success, msg = client.identify( + 'userId', {'trait': 'value'}, {'ip': '192.168.0.1'}, + datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, + 'messageId') + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], {'Amplitude': True}) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['traits'], {'trait': 'value'}) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertEqual(msg['messageId'], 'messageId') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'identify') + + def test_basic_group(self): + client = self.client + success, msg = client.group('userId', 'groupId') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['groupId'], 'groupId') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'group') + + def test_advanced_group(self): + client = self.client + success, msg = client.group( + 'userId', 'groupId', {'trait': 'value'}, {'ip': '192.168.0.1'}, + datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, + 'messageId') + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], {'Amplitude': True}) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['traits'], {'trait': 'value'}) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertEqual(msg['messageId'], 'messageId') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'group') + + def test_basic_alias(self): + client = self.client + success, msg = client.alias('previousId', 'userId') + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + self.assertEqual(msg['previousId'], 'previousId') + self.assertEqual(msg['userId'], 'userId') + + def test_basic_page(self): + client = self.client + success, msg = client.page('userId', name='name') + self.assertFalse(self.failed) + client.flush() + self.assertTrue(success) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'page') + self.assertEqual(msg['name'], 'name') + + def test_advanced_page(self): + client = self.client + success, msg = client.page( + 'userId', 'category', 'name', {'property': 'value'}, + {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', + {'Amplitude': True}, 'messageId') + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], {'Amplitude': True}) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['properties'], {'property': 'value'}) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertEqual(msg['category'], 'category') + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertEqual(msg['messageId'], 'messageId') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'page') + self.assertEqual(msg['name'], 'name') + + def test_basic_screen(self): + client = self.client + success, msg = client.screen('userId', name='name') + client.flush() + self.assertTrue(success) + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'screen') + self.assertEqual(msg['name'], 'name') + + def test_advanced_screen(self): + client = self.client + success, msg = client.screen( + 'userId', 'category', 'name', {'property': 'value'}, + {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', + {'Amplitude': True}, 'messageId') + + self.assertTrue(success) + + self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') + self.assertEqual(msg['integrations'], {'Amplitude': True}) + self.assertEqual(msg['context']['ip'], '192.168.0.1') + self.assertEqual(msg['properties'], {'property': 'value'}) + self.assertEqual(msg['anonymousId'], 'anonymousId') + self.assertEqual(msg['context']['library'], { + 'name': 'analytics-python', + 'version': VERSION + }) + self.assertTrue(isinstance(msg['timestamp'], str)) + self.assertEqual(msg['messageId'], 'messageId') + self.assertEqual(msg['category'], 'category') + self.assertEqual(msg['userId'], 'userId') + self.assertEqual(msg['type'], 'screen') + self.assertEqual(msg['name'], 'name') + + def test_flush(self): + client = self.client + # set up the consumer with more requests than a single batch will allow + for _ in range(1000): + _, _ = client.identify('userId', {'trait': 'value'}) + # We can't reliably assert that the queue is non-empty here; that's + # a race condition. We do our best to load it up though. + client.flush() + # Make sure that the client queue is empty after flushing + self.assertTrue(client.queue.empty()) + + def test_shutdown(self): + client = self.client + # set up the consumer with more requests than a single batch will allow + for _ in range(1000): + _, _ = client.identify('userId', {'trait': 'value'}) + client.shutdown() + # we expect two things after shutdown: + # 1. client queue is empty + # 2. consumer thread has stopped + self.assertTrue(client.queue.empty()) + for consumer in client.consumers: + self.assertFalse(consumer.is_alive()) + + def test_synchronous(self): + client = Client('testsecret', sync_mode=True) + + success, _ = client.identify('userId') + self.assertFalse(client.consumers) + self.assertTrue(client.queue.empty()) + self.assertTrue(success) + + def test_overflow(self): + client = Client('testsecret', max_queue_size=1) + # Ensure consumer thread is no longer uploading + client.join() + + for _ in range(10): + client.identify('userId') + + success, _ = client.identify('userId') + # Make sure we are informed that the queue is at capacity + self.assertFalse(success) + + def test_success_on_invalid_write_key(self): + client = Client('bad_key', on_error=self.fail) + client.track('userId', 'event') + client.flush() + self.assertFalse(self.failed) + + def test_unicode(self): + Client(six.u('unicode_key')) + + def test_numeric_user_id(self): + self.client.track(1234, 'python event') + self.client.flush() + self.assertFalse(self.failed) + + def test_debug(self): + Client('bad_key', debug=True) + + def test_identify_with_date_object(self): + client = self.client + success, msg = client.identify( + 'userId', + { + 'birthdate': date(1981, 2, 2), + }, + ) + client.flush() + self.assertTrue(success) + self.assertFalse(self.failed) + + self.assertEqual(msg['traits'], {'birthdate': date(1981, 2, 2)}) + + def test_gzip(self): + client = Client('testsecret', on_error=self.fail, gzip=True) + for _ in range(10): + client.identify('userId', {'trait': 'value'}) + client.flush() + self.assertFalse(self.failed) + + def test_user_defined_upload_size(self): + client = Client('testsecret', on_error=self.fail, + upload_size=10, upload_interval=3) + + def mock_post_fn(*args, **kwargs): + self.assertEqual(len(kwargs['batch']), 10) + + # the post function should be called 2 times, with a batch size of 10 + # each time. + with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) \ + as mock_post: + for _ in range(20): + client.identify('userId', {'trait': 'value'}) + time.sleep(1) + self.assertEqual(mock_post.call_count, 2) + + def test_user_defined_timeout(self): + client = Client('testsecret', timeout=10) + for consumer in client.consumers: + self.assertEqual(consumer.timeout, 10) + + def test_default_timeout_15(self): + client = Client('testsecret') + for consumer in client.consumers: + self.assertEqual(consumer.timeout, 15) + + def test_proxies(self): + client = Client('testsecret', proxies='203.243.63.16:80') + success, msg = client.identify('userId', {'trait': 'value'}) + self.assertTrue(success) diff --git a/analytics/test/consumer.py b/segment/analytics/test/consumer.py similarity index 100% rename from analytics/test/consumer.py rename to segment/analytics/test/consumer.py diff --git a/analytics/test/module.py b/segment/analytics/test/module.py similarity index 100% rename from analytics/test/module.py rename to segment/analytics/test/module.py diff --git a/analytics/test/request.py b/segment/analytics/test/request.py similarity index 100% rename from analytics/test/request.py rename to segment/analytics/test/request.py diff --git a/analytics/test/utils.py b/segment/analytics/test/utils.py similarity index 100% rename from analytics/test/utils.py rename to segment/analytics/test/utils.py diff --git a/analytics/utils.py b/segment/analytics/utils.py similarity index 100% rename from analytics/utils.py rename to segment/analytics/utils.py diff --git a/segment/analytics/version.py b/segment/analytics/version.py new file mode 100644 index 00000000..f23b0837 --- /dev/null +++ b/segment/analytics/version.py @@ -0,0 +1 @@ +VERSION = '1.6.0' From 135d0017c0adce020337ae8f63e6c0cccdfe91ad Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:33:16 -0500 Subject: [PATCH 216/323] Update history file --- HISTORY.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index feaba7e9..60c5bcef 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ +# 1.6.0 / 2021-10-01 + +- Update package name and namepace name + + # 1.5.0 / 2021-09-23 -================== - Update tests with latest dependencies - Remove unsupported python versions 2.7 & 3.5 From 0eee02fc2bcec09464e322641de3bfa079b55c9c Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:38:54 -0500 Subject: [PATCH 217/323] Update namespace name --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37c16d8a..76ea0ede 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # Don't import analytics-python module here, since deps may not be installed sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) -from version import VERSION +from segment.analytics.version import VERSION long_description = ''' Segment is the simplest way to integrate analytics into your application. From 32b3271c254c21626be1003a28b58f525579c972 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:43:30 -0500 Subject: [PATCH 218/323] Update dateutil version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 76ea0ede..19cfcf19 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ "six>=1.5", "monotonic>=1.5", "backoff==1.10.0", - "python-dateutil>2.1", + "python-dateutil", "appdirs" ] From 1b86e90fcd37dd4add9ec2baf48eeefaa90b1af3 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:45:20 -0500 Subject: [PATCH 219/323] force dateutil install for circleci --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1273c5e..1aee74cf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,8 @@ jobs: - checkout - attach_workspace: { at: . } - run: pip3 install dephell - - run: pip3 install --user appdirs + - run: pip3 install --user appdirs + - run: pip3 install python-dateutil - run: dephell deps convert --from=setup.py --to=requirements.txt - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh From f42bd63e505237e0b30728da8bbfa1d21c015221 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:46:52 -0500 Subject: [PATCH 220/323] Add dateutil to build requirements --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1aee74cf..f5dc7dba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: steps: - checkout - run: pip3 install --user . - - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 + - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil - run: make test - store_artifacts: path: pylint.out @@ -25,7 +25,6 @@ jobs: - attach_workspace: { at: . } - run: pip3 install dephell - run: pip3 install --user appdirs - - run: pip3 install python-dateutil - run: dephell deps convert --from=setup.py --to=requirements.txt - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh From 677a31229bc1e8259c153f26dd09716c5ddf8da7 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:57:01 -0500 Subject: [PATCH 221/323] updating setup config --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 19cfcf19..e11b7780 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ from distutils.core import setup # Don't import analytics-python module here, since deps may not be installed +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'segment')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) from segment.analytics.version import VERSION From c9bb6a4387fe3f63d960b35ccfcd45f868e3d9d6 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 10:58:31 -0500 Subject: [PATCH 222/323] update circleci --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5dc7dba..09395be7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,7 @@ jobs: - image: circleci/python:3.8 steps: - checkout + - run: pip3 install python-dateutil - run: pip3 install --user . - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil - run: make test From 9ca0a4a5f8cc5e89ea2bd7afabf8e4d1bffa3cb3 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 13:49:15 -0500 Subject: [PATCH 223/323] namespace update --- segment/analytics/__init__.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/segment/analytics/__init__.py b/segment/analytics/__init__.py index eda9deb4..8c027ae1 100644 --- a/segment/analytics/__init__.py +++ b/segment/analytics/__init__.py @@ -1,6 +1,6 @@ -from segment.analytics.version import VERSION -from segment.analytics.client import Client +from analytics.version import VERSION +from analytics.client import Client __version__ = VERSION diff --git a/setup.py b/setup.py index e11b7780..30e08708 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # Don't import analytics-python module here, since deps may not be installed sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'segment')) sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) -from segment.analytics.version import VERSION +from analytics.version import VERSION long_description = ''' Segment is the simplest way to integrate analytics into your application. From 52e599e5c2fea705db9ae74ad97b0fcb3654543b Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 13:52:14 -0500 Subject: [PATCH 224/323] config updates --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 30e08708..bdc85880 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ ] setup( - name='analytics-python', + name='segment-analytics-python', version=VERSION, url='https://github.com/segmentio/analytics-python', author='Segment', @@ -45,7 +45,7 @@ maintainer='Segment', maintainer_email='friends@segment.com', test_suite='analytics.test.all', - packages=['analytics', 'analytics.test'], + packages=['segment.analytics', 'analytics.test'], license='MIT License', install_requires=install_requires, extras_require={ From c7870527e7ba5e36cd23946a347ae1a3fa4a4519 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 13:55:23 -0500 Subject: [PATCH 225/323] update config --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 09395be7..1d7b090b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,6 +10,7 @@ jobs: steps: - checkout - run: pip3 install python-dateutil + - run: pip3 install monotonic - run: pip3 install --user . - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil - run: make test From e60526ae6e3d72680a3c8151f99cd26df88a3807 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 13:56:24 -0500 Subject: [PATCH 226/323] config updates --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d7b090b..1de5cd07 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,7 @@ jobs: - image: circleci/python:3.8 steps: - checkout - - run: pip3 install python-dateutil - - run: pip3 install monotonic + - run: pip3 install python-dateutil backoff monotonic - run: pip3 install --user . - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil - run: make test From ccb2c77b95bef8ba8a05a7ededfbe2fcdfaae33a Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 14:01:33 -0500 Subject: [PATCH 227/323] Update config --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1de5cd07..d06aec88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,7 +24,7 @@ jobs: steps: - checkout - attach_workspace: { at: . } - - run: pip3 install dephell + - run: pip3 install dephell - run: pip3 install --user appdirs - run: dephell deps convert --from=setup.py --to=requirements.txt - run: pip3 install --user -r requirements.txt @@ -35,6 +35,7 @@ jobs: - image: circleci/python:3.6 steps: - checkout + - run: pip3 install python-dateutil backoff monotonic - run: pip3 install --user .[test] - run: name: Linting with Flake8 From 1756d2c21ef1577269e55036f73dddb0a753d913 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 14:19:55 -0500 Subject: [PATCH 228/323] namespace update --- segment/analytics/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/segment/analytics/__init__.py b/segment/analytics/__init__.py index 8c027ae1..eda9deb4 100644 --- a/segment/analytics/__init__.py +++ b/segment/analytics/__init__.py @@ -1,6 +1,6 @@ -from analytics.version import VERSION -from analytics.client import Client +from segment.analytics.version import VERSION +from segment.analytics.client import Client __version__ = VERSION From 7c9ab23c7c85413e7796bf808ea5fee32c83e409 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Mon, 4 Oct 2021 14:22:25 -0500 Subject: [PATCH 229/323] more namespace updates --- segment/analytics/client.py | 8 ++++---- segment/analytics/consumer.py | 2 +- segment/analytics/request.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 29b590f5..dc1869d7 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -7,10 +7,10 @@ from dateutil.tz import tzutc from six import string_types -from analytics.utils import guess_timezone, clean -from analytics.consumer import Consumer -from analytics.request import post -from analytics.version import VERSION +from segment.analytics.utils import guess_timezone, clean +from segment.analytics.consumer import Consumer +from segment.analytics.request import post +from segment.analytics.version import VERSION try: import queue diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index cdcb16ff..6192b80c 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -4,7 +4,7 @@ import backoff import json -from analytics.request import post, APIError, DatetimeSerializer +from segment.analytics.request import post, APIError, DatetimeSerializer try: from queue import Empty diff --git a/segment/analytics/request.py b/segment/analytics/request.py index 87b82f4f..d1901f79 100644 --- a/segment/analytics/request.py +++ b/segment/analytics/request.py @@ -7,8 +7,8 @@ from requests.auth import HTTPBasicAuth from requests import sessions -from analytics.version import VERSION -from analytics.utils import remove_trailing_slash +from segment.analytics.version import VERSION +from segment.analytics.utils import remove_trailing_slash _session = sessions.Session() From a7fa7937b63f681d62b2d2010720fc46fe3b569e Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 14 Oct 2021 15:23:30 -0500 Subject: [PATCH 230/323] Remove appdirs dependency --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bdc85880..6f78d6e5 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,7 @@ "six>=1.5", "monotonic>=1.5", "backoff==1.10.0", - "python-dateutil", - "appdirs" + "python-dateutil" ] tests_require = [ From 1eb2979162e9c58b4b3c0235fd2847f0aef1b312 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 14 Oct 2021 15:29:58 -0500 Subject: [PATCH 231/323] Major version update; add in missing 3.9 tests; update dependency version requirements --- .circleci/config.yml | 3 ++- HISTORY.md | 4 ++-- segment/analytics/version.py | 2 +- setup.py | 10 +++++----- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d06aec88..27f26758 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,4 +116,5 @@ workflows: jobs: - test_36 - test_37 - - test_38 \ No newline at end of file + - test_38 + - test_39 \ No newline at end of file diff --git a/HISTORY.md b/HISTORY.md index 60c5bcef..dd0f808b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,6 @@ -# 1.6.0 / 2021-10-01 +# 2.0.0 / 2021-10-01 -- Update package name and namepace name +- Update package name and namespace name # 1.5.0 / 2021-09-23 diff --git a/segment/analytics/version.py b/segment/analytics/version.py index f23b0837..204a96fe 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '1.6.0' +VERSION = '2.0.0' diff --git a/setup.py b/setup.py index 6f78d6e5..9398b249 100644 --- a/setup.py +++ b/setup.py @@ -22,11 +22,11 @@ ''' install_requires = [ - "requests>=2.7,<3.0", - "six>=1.5", - "monotonic>=1.5", - "backoff==1.10.0", - "python-dateutil" + "requests~=2.7", + "six~=1.5", + "monotonic~=1.5", + "backoff~=1.10", + "python-dateutil~=2.2" ] tests_require = [ From f943a7894e4f46572abaec100d2559e956a41771 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 21 Oct 2021 14:36:17 -0500 Subject: [PATCH 232/323] Update test import to fix errors --- simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator.py b/simulator.py index ad94068a..2535e63f 100644 --- a/simulator.py +++ b/simulator.py @@ -1,7 +1,7 @@ import logging import argparse import json -import analytics +import segment.analytics __name__ = 'simulator.py' __version__ = '0.0.1' From a35b145a68fd2701e41b925363629b54ef725a97 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Thu, 21 Oct 2021 14:37:08 -0500 Subject: [PATCH 233/323] import syntax --- simulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simulator.py b/simulator.py index 2535e63f..df83de67 100644 --- a/simulator.py +++ b/simulator.py @@ -1,7 +1,7 @@ import logging import argparse import json -import segment.analytics +import segment.analytics as analytics __name__ = 'simulator.py' __version__ = '0.0.1' From 80450eebd0196c0130a9c987c8b560c6cf210c61 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sat, 27 Nov 2021 12:04:57 +1100 Subject: [PATCH 234/323] docs: fix simple typo, paramater -> parameter There is a small typo in segment/analytics/test/consumer.py. Should read `parameter` rather than `paramater`. --- segment/analytics/test/consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/segment/analytics/test/consumer.py b/segment/analytics/test/consumer.py index b7fcabaa..16d0b213 100644 --- a/segment/analytics/test/consumer.py +++ b/segment/analytics/test/consumer.py @@ -118,7 +118,7 @@ def mock_post(*args, **kwargs): 'userId': 'userId' } # request() should succeed if the number of exceptions raised is - # less than the retries paramater. + # less than the retries parameter. if exception_count <= consumer.retries: consumer.request([track]) else: From 5ad5c6cb507c9348e2d628fb167a9d31bc0c180c Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Wed, 22 Dec 2021 13:02:18 -0600 Subject: [PATCH 235/323] Handle exceptions in the try catch and log them --- segment/analytics/consumer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index 6192b80c..b8013381 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -105,6 +105,8 @@ def next(self): self.log.debug( 'hit batch size limit (size: %d)', total_size) break + except Exception as e: + self.log.error('Exception: %s', e) except Empty: break From 5ec4aff4d8143cef109161f2db95bb0ae6ee6a8f Mon Sep 17 00:00:00 2001 From: William Bradley Date: Tue, 1 Feb 2022 09:25:50 -0700 Subject: [PATCH 236/323] automatically coerce Enum values inside messages --- segment/analytics/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/segment/analytics/utils.py b/segment/analytics/utils.py index 0bc7098e..0ae682d5 100644 --- a/segment/analytics/utils.py +++ b/segment/analytics/utils.py @@ -1,3 +1,4 @@ +from enum import Enum import logging import numbers @@ -54,6 +55,8 @@ def clean(item): return _clean_list(item) elif isinstance(item, dict): return _clean_dict(item) + elif isinstance(item, Enum): + return clean(item.value) else: return _coerce_unicode(item) From 9542b45308949429d6cb977debe747597e3cce82 Mon Sep 17 00:00:00 2001 From: David Pal Date: Mon, 28 Feb 2022 20:21:05 -0500 Subject: [PATCH 237/323] Raise exception on large message --- .gitignore | 4 +++- segment/analytics/client.py | 11 ++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 3427202e..b856542a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ MANIFEST build .eggs *.bat -.vscode/ \ No newline at end of file +.vscode/ +.idea/ +.python-version diff --git a/segment/analytics/client.py b/segment/analytics/client.py index dc1869d7..e93256ca 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -3,13 +3,14 @@ import logging import numbers import atexit +import json from dateutil.tz import tzutc from six import string_types from segment.analytics.utils import guess_timezone, clean -from segment.analytics.consumer import Consumer -from segment.analytics.request import post +from segment.analytics.consumer import Consumer, MAX_MSG_SIZE +from segment.analytics.request import post, DatetimeSerializer from segment.analytics.version import VERSION try: @@ -37,7 +38,6 @@ class DefaultConfig(object): thread = 1 upload_interval = 0.5 upload_size = 100 - max_retries = 10 """Create a new Segment client.""" log = logging.getLogger('segment') @@ -273,6 +273,11 @@ def _enqueue(self, msg): msg = clean(msg) self.log.debug('queueing: %s', msg) + # Check message size. + msg_size = len(json.dumps(msg, cls=DatetimeSerializer).encode()) + if msg_size > MAX_MSG_SIZE: + raise RuntimeError('Message exceeds 32kb limit. (%s)', str(msg)) + # if send is False, return msg as if it was successfully queued if not self.send: return True, msg From 424a736721481941ee73148fbf72189eb3c7d89d Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 2 Mar 2022 13:26:43 -0600 Subject: [PATCH 238/323] Update segment/analytics/client.py Co-authored-by: Patrick K --- segment/analytics/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index e93256ca..35c491fa 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -276,7 +276,7 @@ def _enqueue(self, msg): # Check message size. msg_size = len(json.dumps(msg, cls=DatetimeSerializer).encode()) if msg_size > MAX_MSG_SIZE: - raise RuntimeError('Message exceeds 32kb limit. (%s)', str(msg)) + raise RuntimeError('Message exceeds %skb limit. (%s)', str(int(MAX_MSG_SIZE / 1024)), str(msg)) # if send is False, return msg as if it was successfully queued if not self.send: From 054e79720eb752742bd35818d0ece6479eecca1f Mon Sep 17 00:00:00 2001 From: Pooya Jaferian Date: Fri, 4 Mar 2022 13:01:21 -0800 Subject: [PATCH 239/323] Update version.py --- segment/analytics/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 204a96fe..43082d4b 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.0.0' +VERSION = '2.1.0' From a22c4c793b0aa82512854de4419358407b2ea91c Mon Sep 17 00:00:00 2001 From: Pooya Jaferian Date: Fri, 4 Mar 2022 13:02:26 -0800 Subject: [PATCH 240/323] Update HISTORY.md --- HISTORY.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index dd0f808b..a2b33163 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,10 @@ +# 2.1.0 / 2022-03-04 + +- Raise exception on large message +- Automatically coerce Enum values inside messages +- Handle exceptions in the try catch and log them + + # 2.0.0 / 2021-10-01 - Update package name and namespace name From dda5df51256bdc3dc5516facac324a7656a21223 Mon Sep 17 00:00:00 2001 From: Pooya Jaferian Date: Fri, 4 Mar 2022 13:03:54 -0800 Subject: [PATCH 241/323] Release 2.1.0 From b45648610c57d8123f5828cb46ab8c41dbc2934a Mon Sep 17 00:00:00 2001 From: David Pal Date: Mon, 7 Mar 2022 11:27:56 -0500 Subject: [PATCH 242/323] Remove python 2 support --- .buildscripts/e2e.sh | 2 -- .circleci/config.yml | 8 ++++---- .pylintrc | 2 +- HISTORY.md | 4 ++++ Makefile | 3 +++ analytics/consumer.py | 5 +---- analytics/test/client.py | 1 - segment/analytics/client.py | 9 ++------- segment/analytics/consumer.py | 5 +---- segment/analytics/test/client.py | 3 +-- segment/analytics/test/utils.py | 9 ++------- segment/analytics/utils.py | 6 ++---- segment/analytics/version.py | 2 +- setup.py | 1 - 14 files changed, 22 insertions(+), 38 deletions(-) diff --git a/.buildscripts/e2e.sh b/.buildscripts/e2e.sh index cef1e262..01b6f92a 100755 --- a/.buildscripts/e2e.sh +++ b/.buildscripts/e2e.sh @@ -12,5 +12,3 @@ else ./tester_linux_amd64 -segment-write-key="${SEGMENT_WRITE_KEY}" -webhook-auth-username="${WEBHOOK_AUTH_USERNAME}" -webhook-bucket="${WEBHOOK_BUCKET}" -path='./e2e_test.sh' echo "End to end tests completed!" fi - - diff --git a/.circleci/config.yml b/.circleci/config.yml index 27f26758..118473bc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,8 +24,8 @@ jobs: steps: - checkout - attach_workspace: { at: . } - - run: pip3 install dephell - - run: pip3 install --user appdirs + - run: pip3 install dephell + - run: pip3 install --user appdirs - run: dephell deps convert --from=setup.py --to=requirements.txt - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh @@ -114,7 +114,7 @@ workflows: - master - scheduled_e2e_testing jobs: - - test_36 + - test_36 - test_37 - test_38 - - test_39 \ No newline at end of file + - test_39 diff --git a/.pylintrc b/.pylintrc index 3d6d2d0c..568c4cc2 100644 --- a/.pylintrc +++ b/.pylintrc @@ -306,7 +306,7 @@ init-import=no # List of qualified module names which can have objects that can redefine # builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins +redefining-builtins-modules=past.builtins,future.builtins,io,builtins [FORMAT] diff --git a/HISTORY.md b/HISTORY.md index a2b33163..1b7201cd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,7 @@ +# 2.2.0 / 2022-03-07 +- Remove Python 2 support +- Remove six package + # 2.1.0 / 2022-03-04 - Raise exception on large message diff --git a/Makefile b/Makefile index a62ceec3..51a226e1 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,6 @@ +install_dev: + pip install --edit .[dev] + test: pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out flake8 --max-complexity=10 --statistics analytics > flake8.out || true diff --git a/analytics/consumer.py b/analytics/consumer.py index 6770f1e9..33b9c26c 100644 --- a/analytics/consumer.py +++ b/analytics/consumer.py @@ -7,10 +7,7 @@ from analytics.request import post, APIError, DatetimeSerializer -try: - from queue import Empty -except ImportError: - from Queue import Empty +from queue import Empty MAX_MSG_SIZE = 32 << 10 diff --git a/analytics/test/client.py b/analytics/test/client.py index 0b4b14ff..fcfbc0eb 100644 --- a/analytics/test/client.py +++ b/analytics/test/client.py @@ -1,7 +1,6 @@ from datetime import date, datetime import unittest import time -import six import mock from analytics.version import VERSION diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 35c491fa..b5aa45de 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -6,20 +6,15 @@ import json from dateutil.tz import tzutc -from six import string_types from segment.analytics.utils import guess_timezone, clean from segment.analytics.consumer import Consumer, MAX_MSG_SIZE from segment.analytics.request import post, DatetimeSerializer from segment.analytics.version import VERSION -try: - import queue -except ImportError: - import Queue as queue +import queue - -ID_TYPES = (numbers.Number, string_types) +ID_TYPES = (numbers.Number, str) class Client(object): diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index b8013381..52814ebb 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -6,10 +6,7 @@ from segment.analytics.request import post, APIError, DatetimeSerializer -try: - from queue import Empty -except ImportError: - from Queue import Empty +from queue import Empty MAX_MSG_SIZE = 32 << 10 diff --git a/segment/analytics/test/client.py b/segment/analytics/test/client.py index ca45c238..f01dc2ce 100644 --- a/segment/analytics/test/client.py +++ b/segment/analytics/test/client.py @@ -1,7 +1,6 @@ from datetime import date, datetime import unittest import time -import six import mock from analytics.version import VERSION @@ -286,7 +285,7 @@ def test_success_on_invalid_write_key(self): self.assertFalse(self.failed) def test_unicode(self): - Client(six.u('unicode_key')) + Client('unicode_key') def test_numeric_user_id(self): self.client.track(1234, 'python event') diff --git a/segment/analytics/test/utils.py b/segment/analytics/test/utils.py index 15384f53..6995e799 100644 --- a/segment/analytics/test/utils.py +++ b/segment/analytics/test/utils.py @@ -3,7 +3,6 @@ import unittest from dateutil.tz import tzutc -import six from analytics import utils @@ -25,7 +24,7 @@ def test_timezone_utils(self): def test_clean(self): simple = { 'decimal': Decimal('0.142857'), - 'unicode': six.u('woo'), + 'unicode': 'woo', 'date': datetime.now(), 'long': 200000000, 'integer': 1, @@ -58,11 +57,7 @@ def test_clean_with_dates(self): @classmethod def test_bytes(cls): - if six.PY3: - item = bytes(10) - else: - item = bytearray(10) - + item = bytes(10) utils.clean(item) def test_clean_fn(self): diff --git a/segment/analytics/utils.py b/segment/analytics/utils.py index 0ae682d5..b51ff6b3 100644 --- a/segment/analytics/utils.py +++ b/segment/analytics/utils.py @@ -6,8 +6,6 @@ from datetime import date, datetime from dateutil.tz import tzlocal, tzutc -import six - log = logging.getLogger('segment') @@ -48,7 +46,7 @@ def remove_trailing_slash(host): def clean(item): if isinstance(item, Decimal): return float(item) - elif isinstance(item, (six.string_types, bool, numbers.Number, datetime, + elif isinstance(item, (str, bool, numbers.Number, datetime, date, type(None))): return item elif isinstance(item, (set, list, tuple)): @@ -67,7 +65,7 @@ def _clean_list(list_): def _clean_dict(dict_): data = {} - for k, v in six.iteritems(dict_): + for k, v in dict_.items(): try: data[k] = clean(v) except TypeError: diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 43082d4b..8418bbc5 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.1.0' +VERSION = '2.2.0' diff --git a/setup.py b/setup.py index 9398b249..a04422e9 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ install_requires = [ "requests~=2.7", - "six~=1.5", "monotonic~=1.5", "backoff~=1.10", "python-dateutil~=2.2" From ff03816326f66db18f4ad250909c100ace5aab43 Mon Sep 17 00:00:00 2001 From: David Pal Date: Mon, 7 Mar 2022 11:32:30 -0500 Subject: [PATCH 243/323] Fix Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 51a226e1..cee88eb5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -install_dev: - pip install --edit .[dev] +install: + pip install --edit .[test] test: pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out From 0d9303f80ae1d417d1e5cde80e3a37847d6dcb76 Mon Sep 17 00:00:00 2001 From: David Pal Date: Mon, 7 Mar 2022 11:42:56 -0500 Subject: [PATCH 244/323] Require python 3.6 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a04422e9..d02e2a17 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ maintainer_email='friends@segment.com', test_suite='analytics.test.all', packages=['segment.analytics', 'analytics.test'], + python_requires='>=3.6.0', license='MIT License', install_requires=install_requires, extras_require={ From fd21334fd663dedd274d3e5a31fadc1cca84e480 Mon Sep 17 00:00:00 2001 From: Lateefat_Gitspace <52532904+Tinu-ops@users.noreply.github.com> Date: Wed, 9 Mar 2022 12:20:59 +0000 Subject: [PATCH 245/323] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0a2b1d28..72202e96 100644 --- a/README.md +++ b/README.md @@ -47,16 +47,16 @@ Analytics helps you measure your users, product, and business. It unlocks insigh Install `analytics-python` using pip: ```bash -$ pip install analytics-python +pip3 install analytics-python ``` or you can clone this repo: ```bash -$ git clone https://github.com/segmentio/analytics-python.git +git clone https://github.com/segmentio/analytics-python.git -$ cd analytics-python +cd analytics-python -$ sudo python3 setup.py install +sudo python3 setup.py install ``` Now inside your app, you'll want to **set your** `write_key` before making any analytics calls: From e7653a43508a2b8d68e91a01f59bce000ffd95fd Mon Sep 17 00:00:00 2001 From: Pooya Jaferian Date: Wed, 9 Mar 2022 10:41:30 -0800 Subject: [PATCH 246/323] Release 2.2.0 From e1068d09d16032174969c622ebb90442045b8def Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 9 Mar 2022 15:16:28 -0600 Subject: [PATCH 247/323] Update client.py --- segment/analytics/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index b5aa45de..7ef1783d 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -52,7 +52,7 @@ def __init__(self, thread=DefaultConfig.thread, upload_size=DefaultConfig.upload_size, upload_interval=DefaultConfig.upload_interval,): - require('write_key', write_key, string_types) + require('write_key', write_key) self.queue = queue.Queue(max_queue_size) self.write_key = write_key From 9da22066183e1123e97ce1bfe7d66473ffef16af Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 9 Mar 2022 15:25:23 -0600 Subject: [PATCH 248/323] Update client.py --- segment/analytics/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 7ef1783d..43fa2216 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -52,7 +52,7 @@ def __init__(self, thread=DefaultConfig.thread, upload_size=DefaultConfig.upload_size, upload_interval=DefaultConfig.upload_interval,): - require('write_key', write_key) + require('write_key', write_key, str) self.queue = queue.Queue(max_queue_size) self.write_key = write_key From 30108501d1e7b438f37eea9953454b0cfb11ddba Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 9 Mar 2022 15:28:20 -0600 Subject: [PATCH 249/323] Update client.py --- segment/analytics/client.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 43fa2216..773e21c6 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -122,7 +122,7 @@ def track(self, user_id=None, event=None, properties=None, context=None, integrations = integrations or {} require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) require('properties', properties, dict) - require('event', event, string_types) + require('event', event, str) msg = { 'integrations': integrations, @@ -191,9 +191,9 @@ def page(self, user_id=None, category=None, name=None, properties=None, require('properties', properties, dict) if name: - require('name', name, string_types) + require('name', name, str) if category: - require('category', category, string_types) + require('category', category, str) msg = { 'integrations': integrations, @@ -220,9 +220,9 @@ def screen(self, user_id=None, category=None, name=None, properties=None, require('properties', properties, dict) if name: - require('name', name, string_types) + require('name', name, str) if category: - require('category', category, string_types) + require('category', category, str) msg = { 'integrations': integrations, @@ -249,7 +249,7 @@ def _enqueue(self, msg): message_id = uuid4() require('integrations', msg['integrations'], dict) - require('type', msg['type'], string_types) + require('type', msg['type'], str) require('timestamp', timestamp, datetime) require('context', msg['context'], dict) @@ -328,6 +328,6 @@ def require(name, field, data_type): def stringify_id(val): if val is None: return None - if isinstance(val, string_types): + if isinstance(val, str): return val return str(val) From e221dfc8773f5a7b104f13ecd949f8675a3d9f1b Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Wed, 9 Mar 2022 16:00:24 -0600 Subject: [PATCH 250/323] Update snyk python version to eliminate m2r errors: AttributeError: module 'mistune' has no attribute 'BlockGrammar' --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 118473bc..7a4ff0fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: snyk: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 steps: - checkout - attach_workspace: { at: . } From 0c7a7f6860a5a8bff36167974a53ccaac9f90b5a Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Wed, 9 Mar 2022 16:19:31 -0600 Subject: [PATCH 251/323] Attempt 2 to fix snyk errors --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d02e2a17..8426320b 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ "mock==2.0.0", "pylint==2.8.0", "flake8==3.7.9", + "m2r2" ] setup( From dec461c9d431d3fef076e847d9d5dd54630b6d9a Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 9 Mar 2022 16:27:52 -0600 Subject: [PATCH 252/323] Update setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 8426320b..d02e2a17 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,6 @@ "mock==2.0.0", "pylint==2.8.0", "flake8==3.7.9", - "m2r2" ] setup( From 40eea16d5941680aeaaecc1b163bf8d219466056 Mon Sep 17 00:00:00 2001 From: Andy Freeland Date: Wed, 20 Apr 2022 10:35:15 -0700 Subject: [PATCH 253/323] Update package name in `README.md` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 72202e96..4703d5c0 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Analytics helps you measure your users, product, and business. It unlocks insigh Install `analytics-python` using pip: ```bash -pip3 install analytics-python +pip3 install segment-analytics-python ``` or you can clone this repo: From 0fc04d101a22f81441dc4faba8519e5d81fbc25d Mon Sep 17 00:00:00 2001 From: Samantha Hughes Date: Mon, 20 Jun 2022 14:47:28 -0700 Subject: [PATCH 254/323] fix setup --- setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d02e2a17..95009677 100644 --- a/setup.py +++ b/setup.py @@ -5,11 +5,10 @@ from setuptools import setup except ImportError: from distutils.core import setup - # Don't import analytics-python module here, since deps may not be installed -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'segment')) -sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) -from analytics.version import VERSION +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'segment','analytics')) +# sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) +from version import VERSION long_description = ''' Segment is the simplest way to integrate analytics into your application. From dae446cb97e104ec85399a148c85e775a8b52f7a Mon Sep 17 00:00:00 2001 From: Samantha Hughes Date: Mon, 20 Jun 2022 14:54:17 -0700 Subject: [PATCH 255/323] remove comment --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 95009677..496b6c0c 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ from distutils.core import setup # Don't import analytics-python module here, since deps may not be installed sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'segment','analytics')) -# sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) from version import VERSION long_description = ''' From 06c3541229518367cc8346f943316423306b3539 Mon Sep 17 00:00:00 2001 From: Samantha Hughes Date: Tue, 21 Jun 2022 15:32:21 -0700 Subject: [PATCH 256/323] fix empty catch --- segment/analytics/consumer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index 52814ebb..27586284 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -102,10 +102,10 @@ def next(self): self.log.debug( 'hit batch size limit (size: %d)', total_size) break - except Exception as e: - self.log.error('Exception: %s', e) except Empty: break + except Exception as e: + self.log.exception('Exception: %s', e) return items From becfd8e627bddf0ffa37b20bb753f8ad2f2f81b2 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Thu, 23 Jun 2022 13:12:42 -0500 Subject: [PATCH 257/323] Update HISTORY.md --- HISTORY.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 1b7201cd..9c1bbffd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,7 @@ +# 2.2.1 / 2022-06-23 +- Empty Catch fix #217 +- Build Isolation fix #216 + # 2.2.0 / 2022-03-07 - Remove Python 2 support - Remove six package From 5e3ae5a41dc5a1aebd95a162e6cfee6fbd94af72 Mon Sep 17 00:00:00 2001 From: Sugato Ray Date: Wed, 29 Jun 2022 09:50:23 -0500 Subject: [PATCH 258/323] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..c29156af --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Segment (segment.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From d6c57a07a1e4894465f823b5c19c76b1e47bcc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20HUBSCHER?= Date: Mon, 18 Jul 2022 09:12:39 +0200 Subject: [PATCH 259/323] Loosen the backoff dependency version pin. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 496b6c0c..802abf49 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_requires = [ "requests~=2.7", "monotonic~=1.5", - "backoff~=1.10", + "backoff>=1.10,<3.0", "python-dateutil~=2.2" ] From ceafee5ae5cdca502d6502cbba48972125802c64 Mon Sep 17 00:00:00 2001 From: Shane Duvall Date: Wed, 10 Aug 2022 09:48:16 -0500 Subject: [PATCH 260/323] Update backoff to latest --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 802abf49..11f8ab47 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_requires = [ "requests~=2.7", "monotonic~=1.5", - "backoff>=1.10,<3.0", + "backoff~=2.1", "python-dateutil~=2.2" ] From 425660851aff8c8c5e2b0bf967d17cdbc448cad0 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 1 Sep 2022 16:59:53 -0400 Subject: [PATCH 261/323] Replace outdated build tool and remove python 3.6 test (#236) * Add .circleci/config.yml --- .circleci/config.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a4ff0fc..bb80c74e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,15 +24,15 @@ jobs: steps: - checkout - attach_workspace: { at: . } - - run: pip3 install dephell + - run: pip3 install pipreqs - run: pip3 install --user appdirs - - run: dephell deps convert --from=setup.py --to=requirements.txt + - run: pipreqs . - run: pip3 install --user -r requirements.txt - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh - test_36: &test + test_37: &test docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 steps: - checkout - run: pip3 install python-dateutil backoff monotonic @@ -44,11 +44,6 @@ jobs: - run: make test - run: make e2e_test - test_37: - <<: *test - docker: - - image: circleci/python:3.7 - test_38: <<: *test docker: @@ -74,9 +69,6 @@ workflows: - build: filters: <<: *taggedReleasesFilter - - test_36: - filters: - <<: *taggedReleasesFilter - test_37: filters: <<: *taggedReleasesFilter @@ -89,7 +81,6 @@ workflows: - publish: requires: - build - - test_36 - test_37 - test_38 - test_39 @@ -114,7 +105,6 @@ workflows: - master - scheduled_e2e_testing jobs: - - test_36 - test_37 - test_38 - test_39 From f4bc48ce4d134c22b2a8c08a535734eec70e0a71 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 1 Sep 2022 18:53:57 -0400 Subject: [PATCH 262/323] Release 2.2.1. --- HISTORY.md | 1 + segment/analytics/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 9c1bbffd..77e74e57 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,7 @@ # 2.2.1 / 2022-06-23 - Empty Catch fix #217 - Build Isolation fix #216 +- Removing remaining string_type references # 2.2.0 / 2022-03-07 - Remove Python 2 support diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 8418bbc5..8ec4737c 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.2.0' +VERSION = '2.2.1' From ebfbcbb467a76e44469fcd94e04b6052ba7caf35 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 6 Sep 2022 18:42:49 -0400 Subject: [PATCH 263/323] Updating documentation to reflect proper import statment for namespace change --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4703d5c0..409b5b04 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ sudo python3 setup.py install Now inside your app, you'll want to **set your** `write_key` before making any analytics calls: ```python -import analytics +import segment.analytics as analytics analytics.write_key = 'YOUR_WRITE_KEY' ``` From 709ca901b4de18dead6f4bc9da50c11437305ba6 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 29 Nov 2022 08:55:03 -0500 Subject: [PATCH 264/323] Specifying milliseconds as the isoformat rather than the default microseconds (#242) --- segment/analytics/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 773e21c6..515da899 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -255,7 +255,7 @@ def _enqueue(self, msg): # add common timestamp = guess_timezone(timestamp) - msg['timestamp'] = timestamp.isoformat() + msg['timestamp'] = timestamp.isoformat(timespec='milliseconds') msg['messageId'] = stringify_id(message_id) msg['context']['library'] = { 'name': 'analytics-python', From 5d87a9085c18ee25660c8196dd1233100f9b61b6 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Tue, 29 Nov 2022 12:00:03 -0500 Subject: [PATCH 265/323] Release 2.2.2. --- HISTORY.md | 3 +++ RELEASING.md | 5 +++-- segment/analytics/version.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 77e74e57..56723743 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +# 2.2.2 / 2022-11-29 +- Specifying milliseconds as the isoformat rather than the default microseconds in timestamp + # 2.2.1 / 2022-06-23 - Empty Catch fix #217 - Build Isolation fix #216 diff --git a/RELEASING.md b/RELEASING.md index 9ae22f90..e141a4ae 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -1,8 +1,9 @@ Releasing ========= -1. Update `VERSION` in `analytics/version.py` to the new version. +1. Update `VERSION` in `segment/analytics/version.py` to the new version. 2. Update the `HISTORY.md` for the impending release. 3. `git commit -am "Release X.Y.Z."` (where X.Y.Z is the new version) 4. `git tag -a X.Y.Z -m "Version X.Y.Z"` (where X.Y.Z is the new version). -5. `make release`. +5. `git push && git push --tags` +6. `make release`. diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 8ec4737c..5a099d96 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.2.1' +VERSION = '2.2.2' From fd2c7dd441d33f335054007affd98da286ae0df7 Mon Sep 17 00:00:00 2001 From: Jeppe Fihl-Pearson Date: Tue, 21 Feb 2023 15:31:42 +0000 Subject: [PATCH 266/323] Return values for function calls via the proxy (#244) There shouldn't be any reason why you don't get any return value when the proxy is used, it might still be nice to know if you have maxed out the consumer queue or not. --- segment/analytics/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/segment/analytics/__init__.py b/segment/analytics/__init__.py index eda9deb4..230769b5 100644 --- a/segment/analytics/__init__.py +++ b/segment/analytics/__init__.py @@ -21,32 +21,32 @@ def track(*args, **kwargs): """Send a track call.""" - _proxy('track', *args, **kwargs) + return _proxy('track', *args, **kwargs) def identify(*args, **kwargs): """Send a identify call.""" - _proxy('identify', *args, **kwargs) + return _proxy('identify', *args, **kwargs) def group(*args, **kwargs): """Send a group call.""" - _proxy('group', *args, **kwargs) + return _proxy('group', *args, **kwargs) def alias(*args, **kwargs): """Send a alias call.""" - _proxy('alias', *args, **kwargs) + return _proxy('alias', *args, **kwargs) def page(*args, **kwargs): """Send a page call.""" - _proxy('page', *args, **kwargs) + return _proxy('page', *args, **kwargs) def screen(*args, **kwargs): """Send a screen call.""" - _proxy('screen', *args, **kwargs) + return _proxy('screen', *args, **kwargs) def flush(): @@ -76,4 +76,4 @@ def _proxy(method, *args, **kwargs): sync_mode=sync_mode, timeout=timeout) fn = getattr(default_client, method) - fn(*args, **kwargs) + return fn(*args, **kwargs) From 66cf312b428f742a3cecedf727177028ee5406b7 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 13 Mar 2023 12:07:00 -0400 Subject: [PATCH 267/323] Updating package name in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 409b5b04..69cefe7c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Analytics helps you measure your users, product, and business. It unlocks insigh ## 👨‍💻 Getting Started -Install `analytics-python` using pip: +Install `segment-analytics-python` using pip: ```bash pip3 install segment-analytics-python From 234a17c566a1cbbe89c6bbcfcb6319a419ed4269 Mon Sep 17 00:00:00 2001 From: Michael Vinogradov <100864943+mvinogradov-wavefin@users.noreply.github.com> Date: Mon, 12 Jun 2023 13:23:27 -0400 Subject: [PATCH 268/323] adding Python 3.10 and 3.11 classifiers (#254) --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 11f8ab47..fbc1d066 100644 --- a/setup.py +++ b/setup.py @@ -60,5 +60,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], ) From f881dbe64db4713d194c9e0540707f3473deb79d Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 12 Jun 2023 14:23:39 -0400 Subject: [PATCH 269/323] Release 2.2.3. --- HISTORY.md | 4 ++++ segment/analytics/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 56723743..04fa9c19 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,7 @@ +# 2.2.3 / 2023-06-12 +- Support for Python 3.10 and 3.11 +- Return values for function calls via the proxy + # 2.2.2 / 2022-11-29 - Specifying milliseconds as the isoformat rather than the default microseconds in timestamp diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 5a099d96..2914e9b7 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.2.2' +VERSION = '2.2.3' From 086606c112b2a9114f4c126f0ed448404f055d2f Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 21 Jun 2023 07:07:09 -0500 Subject: [PATCH 270/323] Issue #250 GitHub actions + Python 3.10 & 3.11 (#255) * Initial push to run new tests * Update filename to match github defaults * Updated requirements * update reqs * Update reqs * Updated workflow * Procedure updates * Update tests.yml * Testing * update syntax * update action * updating action * update pip command * Add new python version classifiers * Update readme for release --- .github/workflows/main.yml | 63 ++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 59 ++++++++++++++++++++++++++++ HISTORY.md | 3 +- requirements.txt | 76 +++++++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/tests.yml create mode 100644 requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..3d54fd2a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,63 @@ +name: Run Python Tests +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Python 3 + uses: actions/setup-python@v3 + with: + python-version: 3.7 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install python-dateutil backoff monotonic + pip install --user . + sudo pip install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil + - name: Run tests + run: make e2e_test + +# snyk: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# #- attach_workspace: { at: . } +# - run: pip3 install pipreqs +# - run: pip3 install --user appdirs +# - run: pipreqs . +# - run: pip3 install --user -r requirements.txt +# - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh# + +# test: +# #needs: ['coding-standard', 'lint'] +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python: ['3.7', '3.8', '3.9', '3.10', '3.11'] +# coverage: [false] +# experimental: [false] +# include: +# # Run code coverage. +# - python: '3.7' +# coverage: true +# experimental: false +# - python: '3.8' +# coverage: true +# experimental: false +# - python: '3.9' +# coverage: true +# experimental: false +# - python: '3.10' +# coverage: true +# experimental: false +# - python: '3.11' +# coverage: true +# experimental: false \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..7d1c621d --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,59 @@ +name: e2e tests + +on: + push: + branches: + - master + paths-ignore: + - '**.md' + pull_request: + paths-ignore: + - '**.md' + +jobs: + test-setup-python: + name: Test setup-python + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run with setup-python 3.7 + uses: ./ + with: + python-version: 3.7 + - name: Run tests + run: make test + run: make e2e_test + + - name: Run with setup-python 3.8 + uses: ./ + with: + python-version: 3.8 + - name: Run tests + run: make test + run: make e2e_test + + - name: Run with setup-python 3.9 + uses: ./ + with: + python-version: 3.9 + - name: Run tests + run: make test + run: make e2e_test + + - name: Run with setup-python 3.10 + uses: ./ + with: + python-version: 3.10 + - name: Run tests + run: make test + run: make e2e_test + + - name: Run with setup-python 3.11 + uses: ./ + with: + python-version: 3.11 + - name: Run tests + run: make test + run: make e2e_test \ No newline at end of file diff --git a/HISTORY.md b/HISTORY.md index 04fa9c19..ec09d0bc 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,4 +1,5 @@ -# 2.2.3 / 2023-06-12 +# 2.3.2 / 2023-06-12 +- Update project to use github actions - Support for Python 3.10 and 3.11 - Return values for function calls via the proxy diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a380b190 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,76 @@ +aiohttp==3.7.4.post0 +appdirs==1.4.4 +astroid==1.6.6 +async-timeout==3.0.1 +attrs==21.2.0 +backoff==1.10.0 +bleach==4.1.0 +botocore==1.29.40 +Cerberus==1.3.4 +certifi==2021.5.30 +chardet==4.0.0 +charset-normalizer==2.0.6 +colorama==0.4.4 +dephell==0.8.3 +dephell-archive==0.1.7 +dephell-argparse==0.1.3 +dephell-changelogs==0.0.1 +dephell-discover==0.2.10 +dephell-licenses==0.1.7 +dephell-links==0.1.5 +dephell-markers==1.0.3 +dephell-pythons==0.1.15 +dephell-setuptools==0.2.4 +dephell-shells==0.1.5 +dephell-specifier==0.2.2 +dephell-venvs==0.1.18 +dephell-versioning==0.1.2 +docutils==0.17.1 +entrypoints==0.3 +flake8==3.7.9 +git-remote-codecommit==1.16 +idna==3.2 +importlib-metadata==4.11.2 +isort==5.9.3 +Jinja2==3.0.1 +jmespath==1.0.1 +keyring==23.5.0 +lazy-object-proxy==1.6.0 +m2r==0.2.1 +MarkupSafe==2.0.1 +mccabe==0.6.1 +mistune==0.8.4 +mock==2.0.0 +monotonic==1.6 +multidict==5.1.0 +packaging==21.0 +pbr==5.6.0 +pexpect==4.8.0 +pkginfo==1.8.2 +protobuf==3.15.7 +ptyprocess==0.7.0 +pycodestyle==2.5.0 +pyflakes==2.1.1 +Pygments==2.11.2 +pylint==1.9.3 +pyparsing==2.4.7 +python-dateutil==2.8.2 +readme-renderer==32.0 +requests==2.26.0 +requests-toolbelt==0.9.1 +rfc3986==2.0.0 +ruamel.yaml==0.17.16 +ruamel.yaml.clib==0.2.6 +shellingham==1.4.0 +six==1.15.0 +termcolor==1.1.0 +tomlkit==0.7.2 +tqdm==4.63.0 +twine==3.8.0 +typing-extensions==3.10.0.2 +urllib3==1.26.6 +webencodings==0.5.1 +wrapt==1.12.1 +yarl==1.6.3 +yaspin==2.1.0 +zipp==3.7.0 From 4c0edc0207dd1db9ad00d8f800465739aba79f69 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:48:54 -0700 Subject: [PATCH 271/323] Bump protobuf from 3.15.7 to 4.23.3 (#267) Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 3.15.7 to 4.23.3. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/generate_changelog.py) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v3.15.7...v4.23.3) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a380b190..0413bf78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ packaging==21.0 pbr==5.6.0 pexpect==4.8.0 pkginfo==1.8.2 -protobuf==3.15.7 +protobuf==4.23.3 ptyprocess==0.7.0 pycodestyle==2.5.0 pyflakes==2.1.1 From ca0a9309bd95c2e7a938158e0808c648ee37f9b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:49:38 -0700 Subject: [PATCH 272/323] Bump certifi from 2021.5.30 to 2023.5.7 (#266) Bumps [certifi](https://github.com/certifi/python-certifi) from 2021.5.30 to 2023.5.7. - [Commits](https://github.com/certifi/python-certifi/compare/2021.05.30...2023.05.07) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0413bf78..320eb03c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ backoff==1.10.0 bleach==4.1.0 botocore==1.29.40 Cerberus==1.3.4 -certifi==2021.5.30 +certifi==2023.5.7 chardet==4.0.0 charset-normalizer==2.0.6 colorama==0.4.4 From 79582260071bfa662b7f50fbd8b838518fe32142 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:50:12 -0700 Subject: [PATCH 273/323] Bump requests-toolbelt from 0.9.1 to 1.0.0 (#265) Bumps [requests-toolbelt](https://github.com/requests/toolbelt) from 0.9.1 to 1.0.0. - [Changelog](https://github.com/requests/toolbelt/blob/master/HISTORY.rst) - [Commits](https://github.com/requests/toolbelt/compare/0.9.1...1.0.0) --- updated-dependencies: - dependency-name: requests-toolbelt dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 320eb03c..1418e2dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ pyparsing==2.4.7 python-dateutil==2.8.2 readme-renderer==32.0 requests==2.26.0 -requests-toolbelt==0.9.1 +requests-toolbelt==1.0.0 rfc3986==2.0.0 ruamel.yaml==0.17.16 ruamel.yaml.clib==0.2.6 From 4a5407190cea727e14d9521265eff012d028a2d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:50:33 -0700 Subject: [PATCH 274/323] Bump pygments from 2.11.2 to 2.15.1 (#263) Bumps [pygments](https://github.com/pygments/pygments) from 2.11.2 to 2.15.1. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.11.2...2.15.1) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1418e2dd..ea17f66f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ protobuf==4.23.3 ptyprocess==0.7.0 pycodestyle==2.5.0 pyflakes==2.1.1 -Pygments==2.11.2 +Pygments==2.15.1 pylint==1.9.3 pyparsing==2.4.7 python-dateutil==2.8.2 From 38e6040807036f0d56b48ed38dbb40216e39ddac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:50:56 -0700 Subject: [PATCH 275/323] Bump dephell-setuptools from 0.2.4 to 0.2.5 (#260) Bumps [dephell-setuptools](https://github.com/dephell/dephell_setuptools) from 0.2.4 to 0.2.5. - [Release notes](https://github.com/dephell/dephell_setuptools/releases) - [Commits](https://github.com/dephell/dephell_setuptools/compare/v.0.2.4...v.0.2.5) --- updated-dependencies: - dependency-name: dephell-setuptools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ea17f66f..852c2468 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ dephell-licenses==0.1.7 dephell-links==0.1.5 dephell-markers==1.0.3 dephell-pythons==0.1.15 -dephell-setuptools==0.2.4 +dephell-setuptools==0.2.5 dephell-shells==0.1.5 dephell-specifier==0.2.2 dephell-venvs==0.1.18 From accef2cecad72b15f28503acecd316873292696e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:51:27 -0700 Subject: [PATCH 276/323] Bump pkginfo from 1.8.2 to 1.9.6 (#259) Bumps [pkginfo](https://code.launchpad.net/~tseaver/pkginfo/trunk) from 1.8.2 to 1.9.6. --- updated-dependencies: - dependency-name: pkginfo dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 852c2468..3c6eb338 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,7 @@ multidict==5.1.0 packaging==21.0 pbr==5.6.0 pexpect==4.8.0 -pkginfo==1.8.2 +pkginfo==1.9.6 protobuf==4.23.3 ptyprocess==0.7.0 pycodestyle==2.5.0 From 3c7f33cdd10082e73fafff0668dbfe00f0153fd2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:56:08 -0700 Subject: [PATCH 277/323] Bump requests from 2.26.0 to 2.31.0 (#258) Bumps [requests](https://github.com/psf/requests) from 2.26.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.26.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c6eb338..d6bb4a23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,7 +56,7 @@ pylint==1.9.3 pyparsing==2.4.7 python-dateutil==2.8.2 readme-renderer==32.0 -requests==2.26.0 +requests==2.31.0 requests-toolbelt==1.0.0 rfc3986==2.0.0 ruamel.yaml==0.17.16 From 353f315e15949bf8807af4774b899eb5ae150fad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:51:09 -0700 Subject: [PATCH 278/323] Bump pyparsing from 2.4.7 to 3.1.1 (#280) Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 2.4.7 to 3.1.1. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_2.4.7...3.1.1) --- updated-dependencies: - dependency-name: pyparsing dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d6bb4a23..920a54d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,7 @@ pycodestyle==2.5.0 pyflakes==2.1.1 Pygments==2.15.1 pylint==1.9.3 -pyparsing==2.4.7 +pyparsing==3.1.1 python-dateutil==2.8.2 readme-renderer==32.0 requests==2.31.0 From 119d9cbc8b4b8ed3149c02d949764a8c329c4b02 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:51:40 -0700 Subject: [PATCH 279/323] Bump certifi from 2023.5.7 to 2023.7.22 (#278) Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.5.7 to 2023.7.22. - [Commits](https://github.com/certifi/python-certifi/compare/2023.05.07...2023.07.22) --- updated-dependencies: - dependency-name: certifi dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 920a54d7..8adf5692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ backoff==1.10.0 bleach==4.1.0 botocore==1.29.40 Cerberus==1.3.4 -certifi==2023.5.7 +certifi==2023.7.22 chardet==4.0.0 charset-normalizer==2.0.6 colorama==0.4.4 From 937bd8b20de764cf70f97587eeaeb7c77e418585 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:52:10 -0700 Subject: [PATCH 280/323] Bump charset-normalizer from 2.0.6 to 3.2.0 (#275) Bumps [charset-normalizer](https://github.com/Ousret/charset_normalizer) from 2.0.6 to 3.2.0. - [Release notes](https://github.com/Ousret/charset_normalizer/releases) - [Changelog](https://github.com/Ousret/charset_normalizer/blob/master/CHANGELOG.md) - [Upgrade guide](https://github.com/Ousret/charset_normalizer/blob/master/UPGRADE.md) - [Commits](https://github.com/Ousret/charset_normalizer/compare/2.0.6...3.2.0) --- updated-dependencies: - dependency-name: charset-normalizer dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8adf5692..b922d26d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ botocore==1.29.40 Cerberus==1.3.4 certifi==2023.7.22 chardet==4.0.0 -charset-normalizer==2.0.6 +charset-normalizer==3.2.0 colorama==0.4.4 dephell==0.8.3 dephell-archive==0.1.7 From a90e28ed6bcf0591a61d4c4280deaf5f62f0eea6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:54:35 -0700 Subject: [PATCH 281/323] Bump bleach from 4.1.0 to 6.0.0 (#273) Bumps [bleach](https://github.com/mozilla/bleach) from 4.1.0 to 6.0.0. - [Changelog](https://github.com/mozilla/bleach/blob/main/CHANGES) - [Commits](https://github.com/mozilla/bleach/compare/v4.1.0...v6.0.0) --- updated-dependencies: - dependency-name: bleach dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b922d26d..80b20922 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ astroid==1.6.6 async-timeout==3.0.1 attrs==21.2.0 backoff==1.10.0 -bleach==4.1.0 +bleach==6.0.0 botocore==1.29.40 Cerberus==1.3.4 certifi==2023.7.22 From f1f1e6c3a4040b38d9aa235dbd67992f95c0e2bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:54:59 -0700 Subject: [PATCH 282/323] Bump ruamel-yaml-clib from 0.2.6 to 0.2.7 (#271) Bumps [ruamel-yaml-clib](https://sourceforge.net/p/ruamel-yaml-clib/code/ci/default/tree) from 0.2.6 to 0.2.7. --- updated-dependencies: - dependency-name: ruamel-yaml-clib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 80b20922..405dfba9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,7 +60,7 @@ requests==2.31.0 requests-toolbelt==1.0.0 rfc3986==2.0.0 ruamel.yaml==0.17.16 -ruamel.yaml.clib==0.2.6 +ruamel.yaml.clib==0.2.7 shellingham==1.4.0 six==1.15.0 termcolor==1.1.0 From f01f7bb04c436eb93d839d4d09a9e2c0058b694e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:55:33 -0700 Subject: [PATCH 283/323] Bump pbr from 5.6.0 to 5.11.1 (#269) Bumps [pbr](https://docs.openstack.org/pbr/latest/) from 5.6.0 to 5.11.1. --- updated-dependencies: - dependency-name: pbr dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 405dfba9..2207a238 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,7 +44,7 @@ mock==2.0.0 monotonic==1.6 multidict==5.1.0 packaging==21.0 -pbr==5.6.0 +pbr==5.11.1 pexpect==4.8.0 pkginfo==1.9.6 protobuf==4.23.3 From d90ab20b801e0de6602dee91f31f7c0c2389f386 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 08:57:02 -0700 Subject: [PATCH 284/323] Bump docutils from 0.17.1 to 0.20.1 (#268) Bumps [docutils](https://docutils.sourceforge.io/) from 0.17.1 to 0.20.1. --- updated-dependencies: - dependency-name: docutils dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2207a238..6890f1f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ dephell-shells==0.1.5 dephell-specifier==0.2.2 dephell-venvs==0.1.18 dephell-versioning==0.1.2 -docutils==0.17.1 +docutils==0.20.1 entrypoints==0.3 flake8==3.7.9 git-remote-codecommit==1.16 From cf8b97ca095aa83d63531bc0b315e3fcc7c08c85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:41:35 -0700 Subject: [PATCH 285/323] Bump protobuf from 4.23.3 to 4.24.2 (#293) Bumps [protobuf](https://github.com/protocolbuffers/protobuf) from 4.23.3 to 4.24.2. - [Release notes](https://github.com/protocolbuffers/protobuf/releases) - [Changelog](https://github.com/protocolbuffers/protobuf/blob/main/protobuf_release.bzl) - [Commits](https://github.com/protocolbuffers/protobuf/compare/v4.23.3...v4.24.2) --- updated-dependencies: - dependency-name: protobuf dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6890f1f4..46b18351 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,7 +47,7 @@ packaging==21.0 pbr==5.11.1 pexpect==4.8.0 pkginfo==1.9.6 -protobuf==4.23.3 +protobuf==4.24.2 ptyprocess==0.7.0 pycodestyle==2.5.0 pyflakes==2.1.1 From b7a802db0c908a857ff7670c671eb213b89ea749 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:42:02 -0700 Subject: [PATCH 286/323] Bump yarl from 1.6.3 to 1.9.2 (#290) Bumps [yarl](https://github.com/aio-libs/yarl) from 1.6.3 to 1.9.2. - [Release notes](https://github.com/aio-libs/yarl/releases) - [Changelog](https://github.com/aio-libs/yarl/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/yarl/compare/v1.6.3...v1.9.2) --- updated-dependencies: - dependency-name: yarl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 46b18351..2b9d59af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -71,6 +71,6 @@ typing-extensions==3.10.0.2 urllib3==1.26.6 webencodings==0.5.1 wrapt==1.12.1 -yarl==1.6.3 +yarl==1.9.2 yaspin==2.1.0 zipp==3.7.0 From 4d9fbbb5cf797c110d570f948cdc6360be01c964 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:42:48 -0700 Subject: [PATCH 287/323] Bump idna from 3.2 to 3.4 (#289) Bumps [idna](https://github.com/kjd/idna) from 3.2 to 3.4. - [Changelog](https://github.com/kjd/idna/blob/master/HISTORY.rst) - [Commits](https://github.com/kjd/idna/compare/v3.2...v3.4) --- updated-dependencies: - dependency-name: idna dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2b9d59af..5196a570 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ docutils==0.20.1 entrypoints==0.3 flake8==3.7.9 git-remote-codecommit==1.16 -idna==3.2 +idna==3.4 importlib-metadata==4.11.2 isort==5.9.3 Jinja2==3.0.1 From a77b5d8bbb07969bc1ae2ab507b45e728a084bd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:43:25 -0700 Subject: [PATCH 288/323] Bump lazy-object-proxy from 1.6.0 to 1.9.0 (#288) Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.6.0 to 1.9.0. - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.6.0...v1.9.0) --- updated-dependencies: - dependency-name: lazy-object-proxy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5196a570..a094a121 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,7 @@ isort==5.9.3 Jinja2==3.0.1 jmespath==1.0.1 keyring==23.5.0 -lazy-object-proxy==1.6.0 +lazy-object-proxy==1.9.0 m2r==0.2.1 MarkupSafe==2.0.1 mccabe==0.6.1 From a2ed84e4f365f1c662bfee771e8e1918fcd048d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:44:20 -0700 Subject: [PATCH 289/323] Bump cerberus from 1.3.4 to 1.3.5 (#287) Bumps [cerberus](https://github.com/pyeve/cerberus) from 1.3.4 to 1.3.5. - [Changelog](https://github.com/pyeve/cerberus/blob/1.3.x/CHANGES.rst) - [Commits](https://github.com/pyeve/cerberus/compare/1.3.4...1.3.5) --- updated-dependencies: - dependency-name: cerberus dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a094a121..ad5b4f93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ attrs==21.2.0 backoff==1.10.0 bleach==6.0.0 botocore==1.29.40 -Cerberus==1.3.4 +Cerberus==1.3.5 certifi==2023.7.22 chardet==4.0.0 charset-normalizer==3.2.0 From 5e57c9b9684dcfe48cb5ff2778073581d606c526 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:45:53 -0700 Subject: [PATCH 290/323] Bump markupsafe from 2.0.1 to 2.1.3 (#285) Bumps [markupsafe](https://github.com/pallets/markupsafe) from 2.0.1 to 2.1.3. - [Release notes](https://github.com/pallets/markupsafe/releases) - [Changelog](https://github.com/pallets/markupsafe/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/markupsafe/compare/2.0.1...2.1.3) --- updated-dependencies: - dependency-name: markupsafe dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ad5b4f93..ffc7c556 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ jmespath==1.0.1 keyring==23.5.0 lazy-object-proxy==1.9.0 m2r==0.2.1 -MarkupSafe==2.0.1 +MarkupSafe==2.1.3 mccabe==0.6.1 mistune==0.8.4 mock==2.0.0 From 9b904d3651bb07ec46504b09560040f7d7c504a7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:46:17 -0700 Subject: [PATCH 291/323] Bump attrs from 21.2.0 to 23.1.0 (#284) Bumps [attrs](https://github.com/python-attrs/attrs) from 21.2.0 to 23.1.0. - [Release notes](https://github.com/python-attrs/attrs/releases) - [Changelog](https://github.com/python-attrs/attrs/blob/main/CHANGELOG.md) - [Commits](https://github.com/python-attrs/attrs/compare/21.2.0...23.1.0) --- updated-dependencies: - dependency-name: attrs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ffc7c556..b97f9a04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiohttp==3.7.4.post0 appdirs==1.4.4 astroid==1.6.6 async-timeout==3.0.1 -attrs==21.2.0 +attrs==23.1.0 backoff==1.10.0 bleach==6.0.0 botocore==1.29.40 From 91f23b92780d6fdfd9a46a00746a319a094c4460 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:47:07 -0700 Subject: [PATCH 292/323] Bump ruamel-yaml from 0.17.16 to 0.17.32 (#283) Bumps [ruamel-yaml](https://sourceforge.net/p/ruamel-yaml/code/ci/default/tree) from 0.17.16 to 0.17.32. --- updated-dependencies: - dependency-name: ruamel-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b97f9a04..6fe251c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -59,7 +59,7 @@ readme-renderer==32.0 requests==2.31.0 requests-toolbelt==1.0.0 rfc3986==2.0.0 -ruamel.yaml==0.17.16 +ruamel.yaml==0.17.32 ruamel.yaml.clib==0.2.7 shellingham==1.4.0 six==1.15.0 From 1c44a93da67825e5ea2e0115819d908f6a0b26d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 07:47:42 -0700 Subject: [PATCH 293/323] Bump pygments from 2.15.1 to 2.16.1 (#282) Bumps [pygments](https://github.com/pygments/pygments) from 2.15.1 to 2.16.1. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.15.1...2.16.1) --- updated-dependencies: - dependency-name: pygments dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6fe251c2..4888d8ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -51,7 +51,7 @@ protobuf==4.24.2 ptyprocess==0.7.0 pycodestyle==2.5.0 pyflakes==2.1.1 -Pygments==2.15.1 +Pygments==2.16.1 pylint==1.9.3 pyparsing==3.1.1 python-dateutil==2.8.2 From b74ed1824cfd2b6a3bf1c42fba1a042da21f234f Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 6 Oct 2023 14:35:28 -0400 Subject: [PATCH 294/323] Python OAuth implementation (#320) * Working OAuth implementation * Logging improvements and change for 429 handling * Cleanup and restore unit tests to functioning order --- analytics/consumer.py | 133 ----------- analytics/test/client.py | 342 ----------------------------- requirements.txt | 1 + samples/oauth.py | 52 +++++ segment/analytics/__init__.py | 16 +- segment/analytics/client.py | 31 ++- segment/analytics/consumer.py | 15 +- segment/analytics/oauth_manager.py | 208 ++++++++++++++++++ segment/analytics/request.py | 29 ++- segment/analytics/test/__init__.py | 17 +- segment/analytics/test/client.py | 9 +- segment/analytics/test/consumer.py | 12 +- segment/analytics/test/module.py | 2 +- segment/analytics/test/oauth.py | 155 +++++++++++++ segment/analytics/test/request.py | 2 +- segment/analytics/test/utils.py | 2 +- setup.py | 4 +- 17 files changed, 520 insertions(+), 510 deletions(-) delete mode 100644 analytics/consumer.py delete mode 100644 analytics/test/client.py create mode 100644 samples/oauth.py create mode 100644 segment/analytics/oauth_manager.py create mode 100644 segment/analytics/test/oauth.py diff --git a/analytics/consumer.py b/analytics/consumer.py deleted file mode 100644 index 33b9c26c..00000000 --- a/analytics/consumer.py +++ /dev/null @@ -1,133 +0,0 @@ -import logging -from threading import Thread -import json -import monotonic -import backoff - - -from analytics.request import post, APIError, DatetimeSerializer - -from queue import Empty - -MAX_MSG_SIZE = 32 << 10 - -# Our servers only accept batches less than 500KB. Here limit is set slightly -# lower to leave space for extra data that will be added later, eg. "sentAt". -BATCH_SIZE_LIMIT = 475000 - - -class Consumer(Thread): - """Consumes the messages from the client's queue.""" - log = logging.getLogger('segment') - - def __init__(self, queue, write_key, upload_size=100, host=None, - on_error=None, upload_interval=0.5, gzip=False, retries=10, - timeout=15, proxies=None): - """Create a consumer thread.""" - Thread.__init__(self) - # Make consumer a daemon thread so that it doesn't block program exit - self.daemon = True - self.upload_size = upload_size - self.upload_interval = upload_interval - self.write_key = write_key - self.host = host - self.on_error = on_error - self.queue = queue - self.gzip = gzip - # It's important to set running in the constructor: if we are asked to - # pause immediately after construction, we might set running to True in - # run() *after* we set it to False in pause... and keep running - # forever. - self.running = True - self.retries = retries - self.timeout = timeout - self.proxies = proxies - - def run(self): - """Runs the consumer.""" - self.log.debug('consumer is running...') - while self.running: - self.upload() - - self.log.debug('consumer exited.') - - def pause(self): - """Pause the consumer.""" - self.running = False - - def upload(self): - """Upload the next batch of items, return whether successful.""" - success = False - batch = self.next() - if len(batch) == 0: - return False - - try: - self.request(batch) - success = True - except Exception as e: - self.log.error('error uploading: %s', e) - success = False - if self.on_error: - self.on_error(e, batch) - finally: - # mark items as acknowledged from queue - for _ in batch: - self.queue.task_done() - return success - - def next(self): - """Return the next batch of items to upload.""" - queue = self.queue - items = [] - - start_time = monotonic.monotonic() - total_size = 0 - - while len(items) < self.upload_size: - elapsed = monotonic.monotonic() - start_time - if elapsed >= self.upload_interval: - break - try: - item = queue.get( - block=True, timeout=self.upload_interval - elapsed) - item_size = len(json.dumps( - item, cls=DatetimeSerializer).encode()) - if item_size > MAX_MSG_SIZE: - self.log.error( - 'Item exceeds 32kb limit, dropping. (%s)', str(item)) - continue - items.append(item) - total_size += item_size - if total_size >= BATCH_SIZE_LIMIT: - self.log.debug( - 'hit batch size limit (size: %d)', total_size) - break - except Empty: - break - - return items - - def request(self, batch): - """Attempt to upload the batch and retry before raising an error """ - - def fatal_exception(exc): - if isinstance(exc, APIError): - # retry on server errors and client errors - # with 429 status code (rate limited), - # don't retry on other client errors - return (400 <= exc.status < 500) and exc.status != 429 - else: - # retry on all other errors (eg. network) - return False - - @backoff.on_exception( - backoff.expo, - Exception, - max_tries=self.retries + 1, - giveup=fatal_exception) - def send_request(): - post(self.write_key, self.host, gzip=self.gzip, - timeout=self.timeout, batch=batch, proxies=self.proxies) - - send_request() diff --git a/analytics/test/client.py b/analytics/test/client.py deleted file mode 100644 index fcfbc0eb..00000000 --- a/analytics/test/client.py +++ /dev/null @@ -1,342 +0,0 @@ -from datetime import date, datetime -import unittest -import time -import mock - -from analytics.version import VERSION -from analytics.client import Client - - -class TestClient(unittest.TestCase): - - def fail(self): - """Mark the failure handler""" - self.failed = True - - def setUp(self): - self.failed = False - self.client = Client('testsecret', on_error=self.fail) - - def test_requires_write_key(self): - self.assertRaises(AssertionError, Client) - - def test_empty_flush(self): - self.client.flush() - - def test_basic_track(self): - client = self.client - success, msg = client.track('userId', 'python test event') - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - - self.assertEqual(msg['event'], 'python test event') - self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertTrue(isinstance(msg['messageId'], str)) - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['properties'], {}) - self.assertEqual(msg['type'], 'track') - - def test_stringifies_user_id(self): - # A large number that loses precision in node: - # node -e "console.log(157963456373623802 + 1)" > 157963456373623800 - client = self.client - success, msg = client.track( - user_id=157963456373623802, event='python test event') - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - - self.assertEqual(msg['userId'], '157963456373623802') - self.assertEqual(msg['anonymousId'], None) - - def test_stringifies_anonymous_id(self): - # A large number that loses precision in node: - # node -e "console.log(157963456373623803 + 1)" > 157963456373623800 - client = self.client - success, msg = client.track( - anonymous_id=157963456373623803, event='python test event') - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - - self.assertEqual(msg['userId'], None) - self.assertEqual(msg['anonymousId'], '157963456373623803') - - def test_advanced_track(self): - client = self.client - success, msg = client.track( - 'userId', 'python test event', {'property': 'value'}, - {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', - {'Amplitude': True}, 'messageId') - - self.assertTrue(success) - - self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['properties'], {'property': 'value'}) - self.assertEqual(msg['integrations'], {'Amplitude': True}) - self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['event'], 'python test event') - self.assertEqual(msg['anonymousId'], 'anonymousId') - self.assertEqual(msg['context']['library'], { - 'name': 'analytics-python', - 'version': VERSION - }) - self.assertEqual(msg['messageId'], 'messageId') - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'track') - - def test_basic_identify(self): - client = self.client - success, msg = client.identify('userId', {'trait': 'value'}) - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - - self.assertEqual(msg['traits'], {'trait': 'value'}) - self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertTrue(isinstance(msg['messageId'], str)) - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'identify') - - def test_advanced_identify(self): - client = self.client - success, msg = client.identify( - 'userId', {'trait': 'value'}, {'ip': '192.168.0.1'}, - datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, - 'messageId') - - self.assertTrue(success) - - self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], {'Amplitude': True}) - self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['traits'], {'trait': 'value'}) - self.assertEqual(msg['anonymousId'], 'anonymousId') - self.assertEqual(msg['context']['library'], { - 'name': 'analytics-python', - 'version': VERSION - }) - self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertEqual(msg['messageId'], 'messageId') - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'identify') - - def test_basic_group(self): - client = self.client - success, msg = client.group('userId', 'groupId') - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - - self.assertEqual(msg['groupId'], 'groupId') - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'group') - - def test_advanced_group(self): - client = self.client - success, msg = client.group( - 'userId', 'groupId', {'trait': 'value'}, {'ip': '192.168.0.1'}, - datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, - 'messageId') - - self.assertTrue(success) - - self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], {'Amplitude': True}) - self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['traits'], {'trait': 'value'}) - self.assertEqual(msg['anonymousId'], 'anonymousId') - self.assertEqual(msg['context']['library'], { - 'name': 'analytics-python', - 'version': VERSION - }) - self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertEqual(msg['messageId'], 'messageId') - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'group') - - def test_basic_alias(self): - client = self.client - success, msg = client.alias('previousId', 'userId') - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - self.assertEqual(msg['previousId'], 'previousId') - self.assertEqual(msg['userId'], 'userId') - - def test_basic_page(self): - client = self.client - success, msg = client.page('userId', name='name') - self.assertFalse(self.failed) - client.flush() - self.assertTrue(success) - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'page') - self.assertEqual(msg['name'], 'name') - - def test_advanced_page(self): - client = self.client - success, msg = client.page( - 'userId', 'category', 'name', {'property': 'value'}, - {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', - {'Amplitude': True}, 'messageId') - - self.assertTrue(success) - - self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], {'Amplitude': True}) - self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['properties'], {'property': 'value'}) - self.assertEqual(msg['anonymousId'], 'anonymousId') - self.assertEqual(msg['context']['library'], { - 'name': 'analytics-python', - 'version': VERSION - }) - self.assertEqual(msg['category'], 'category') - self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertEqual(msg['messageId'], 'messageId') - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'page') - self.assertEqual(msg['name'], 'name') - - def test_basic_screen(self): - client = self.client - success, msg = client.screen('userId', name='name') - client.flush() - self.assertTrue(success) - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'screen') - self.assertEqual(msg['name'], 'name') - - def test_advanced_screen(self): - client = self.client - success, msg = client.screen( - 'userId', 'category', 'name', {'property': 'value'}, - {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', - {'Amplitude': True}, 'messageId') - - self.assertTrue(success) - - self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') - self.assertEqual(msg['integrations'], {'Amplitude': True}) - self.assertEqual(msg['context']['ip'], '192.168.0.1') - self.assertEqual(msg['properties'], {'property': 'value'}) - self.assertEqual(msg['anonymousId'], 'anonymousId') - self.assertEqual(msg['context']['library'], { - 'name': 'analytics-python', - 'version': VERSION - }) - self.assertTrue(isinstance(msg['timestamp'], str)) - self.assertEqual(msg['messageId'], 'messageId') - self.assertEqual(msg['category'], 'category') - self.assertEqual(msg['userId'], 'userId') - self.assertEqual(msg['type'], 'screen') - self.assertEqual(msg['name'], 'name') - - def test_flush(self): - client = self.client - # set up the consumer with more requests than a single batch will allow - for _ in range(1000): - _, _ = client.identify('userId', {'trait': 'value'}) - # We can't reliably assert that the queue is non-empty here; that's - # a race condition. We do our best to load it up though. - client.flush() - # Make sure that the client queue is empty after flushing - self.assertTrue(client.queue.empty()) - - def test_shutdown(self): - client = self.client - # set up the consumer with more requests than a single batch will allow - for _ in range(1000): - _, _ = client.identify('userId', {'trait': 'value'}) - client.shutdown() - # we expect two things after shutdown: - # 1. client queue is empty - # 2. consumer thread has stopped - self.assertTrue(client.queue.empty()) - for consumer in client.consumers: - self.assertFalse(consumer.is_alive()) - - def test_synchronous(self): - client = Client('testsecret', sync_mode=True) - - success, _ = client.identify('userId') - self.assertFalse(client.consumers) - self.assertTrue(client.queue.empty()) - self.assertTrue(success) - - def test_overflow(self): - client = Client('testsecret', max_queue_size=1) - # Ensure consumer thread is no longer uploading - client.join() - - for _ in range(10): - client.identify('userId') - - success, _ = client.identify('userId') - # Make sure we are informed that the queue is at capacity - self.assertFalse(success) - - def test_success_on_invalid_write_key(self): - client = Client('bad_key', on_error=self.fail) - client.track('userId', 'event') - client.flush() - self.assertFalse(self.failed) - - def test_numeric_user_id(self): - self.client.track(1234, 'python event') - self.client.flush() - self.assertFalse(self.failed) - - def test_identify_with_date_object(self): - client = self.client - success, msg = client.identify( - 'userId', - { - 'birthdate': date(1981, 2, 2), - }, - ) - client.flush() - self.assertTrue(success) - self.assertFalse(self.failed) - - self.assertEqual(msg['traits'], {'birthdate': date(1981, 2, 2)}) - - def test_gzip(self): - client = Client('testsecret', on_error=self.fail, gzip=True) - for _ in range(10): - client.identify('userId', {'trait': 'value'}) - client.flush() - self.assertFalse(self.failed) - - def test_user_defined_upload_size(self): - client = Client('testsecret', on_error=self.fail, - upload_size=10, upload_interval=3) - - def mock_post_fn(**kwargs): - self.assertEqual(len(kwargs['batch']), 10) - - # the post function should be called 2 times, with a batch size of 10 - # each time. - with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) \ - as mock_post: - for _ in range(20): - client.identify('userId', {'trait': 'value'}) - time.sleep(1) - self.assertEqual(mock_post.call_count, 2) - - def test_user_defined_timeout(self): - client = Client('testsecret', timeout=10) - for consumer in client.consumers: - self.assertEqual(consumer.timeout, 10) - - def test_default_timeout_15(self): - client = Client('testsecret') - for consumer in client.consumers: - self.assertEqual(consumer.timeout, 15) - - def test_proxies(self): - client = Client('testsecret', proxies='203.243.63.16:80') - success, msg = client.identify('userId', {'trait': 'value'}) - self.assertTrue(success) diff --git a/requirements.txt b/requirements.txt index 4888d8ed..5eaf3fcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,6 +52,7 @@ ptyprocess==0.7.0 pycodestyle==2.5.0 pyflakes==2.1.1 Pygments==2.16.1 +PyJWT==2.8.0 pylint==1.9.3 pyparsing==3.1.1 python-dateutil==2.8.2 diff --git a/samples/oauth.py b/samples/oauth.py new file mode 100644 index 00000000..bf1cc70d --- /dev/null +++ b/samples/oauth.py @@ -0,0 +1,52 @@ +import sys +import os +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +import time +import segment.analytics as analytics + +privatekey = '''-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDVll7uJaH322IN +PQsH2aOXZJ2r1q+6hpVK1R5JV1p41PUzn8pOxyXFHWB+53dUd4B8qywKS36XQjp0 +VmhR1tQ22znQ9ZCM6y4LGeOJBjAZiFZLcGQNNrDFC0WGWTrK1ZTS2K7p5qy4fIXG +laNkMXiGGCawkgcHAdOvPTy8m1d9a6YSetYVmBP/tEYN95jPyZFIoHQfkQPBPr9W +cWPpdEBzasHV5d957akjurPpleDiD5as66UW4dkWXvS7Wu7teCLCyDApcyJKTb2Z +SXybmWjhIZuctZMAx3wT/GgW3FbkGaW5KLQgBUMzjpL0fCtMatlqckMD92ll1FuK +R+HnXu05AgMBAAECggEBAK4o2il4GDUh9zbyQo9ZIPLuwT6AZXRED3Igi3ykNQp4 +I6S/s9g+vQaY6LkyBnSiqOt/K/8NBiFSiJWaa5/n+8zrP56qzf6KOlYk+wsdN5Vq +PWtwLrUzljpl8YAWPEFunNa8hwwE42vfZbnDBKNLT4qQIOQzfnVxQOoQlfj49gM2 +iSrblvsnQTyucFy3UyTeioHbh8q2Xqcxry5WUCOrFDd3IIwATTpLZGw0IPeuFJbJ +NfBizLEcyJaM9hujQU8PRCRd16MWX+bbYM6Mh4dkT40QXWsVnHBHwsgPdQgDgseF +Na4ajtHoC0DlwYCXpCm3IzJfKfq/LR2q8NDUgKeF4AECgYEA9nD4czza3SRbzhpZ +bBoK77CSNqCcMAqyuHB0hp/XX3yB7flF9PIPb2ReO8wwmjbxn+bm7PPz2Uwd2SzO +pU+FXmyKJr53Jxw/hmDWZCoh42gsGDlVqpmytzsj74KlaYiMyZmEGbD7t/FGfNGV +LdLDJaHIYxEimFviOTXKCeKvPAECgYEA3d8tv4jdp1uAuRZiU9Z/tfw5mJOi3oXF +8AdFFDwaPzcTorEAxjrt9X6IjPbLIDJNJtuXYpe+dG6720KyuNnhLhWW9oZEJTwT +dUgqZ2fTCOS9uH0jSn+ZFlgTWI6UDQXRwE7z8avlhMIrQVmPsttGTo7V6sQVtGRx +bNj2RSVekTkCgYAJvy4UYLPHS0jWPfSLcfw8vp8JyhBjVgj7gncZW/kIrcP1xYYe +yfQSU8XmV40UjFfCGz/G318lmP0VOdByeVKtCV3talsMEPHyPqI8E+6DL/uOebYJ +qUqINK6XKnOgWOY4kvnGillqTQCcry1XQp61PlDOmj7kB75KxPXYrj6AAQKBgQDa ++ixCv6hURuEyy77cE/YT/Q4zYnL6wHjtP5+UKwWUop1EkwG6o+q7wtiul90+t6ah +1VUCP9X/QFM0Qg32l0PBohlO0pFrVnG17TW8vSHxwyDkds1f97N19BOT8ZR5jebI +sKPfP9LVRnY+l1BWLEilvB+xBzqMwh2YWkIlWI6PMQKBgGi6TBnxp81lOYrxVRDj +/3ycRnVDmBdlQKFunvfzUBmG1mG/G0YHeVSUKZJGX7w2l+jnDwIA383FcUeA8X6A +l9q+amhtkwD/6fbkAu/xoWNl+11IFoxd88y2ByBFoEKB6UVLuCTSKwXDqzEZet7x +mDyRxq7ohIzLkw8b8buDeuXZ +-----END PRIVATE KEY----- +''' # Should be read from a file on disk which can be rotated out + +analytics.write_key = '' + +analytics.oauth_client_id = 'CLIENT_ID' # OAuth application ID from segment dashboard +analytics.oauth_client_key = privatekey # generated as a public/private key pair in PEM format from OpenSSL +analytics.oauth_key_id = 'KEY_ID' # From segment dashboard after uploading public key +analytics.oauth_scope = 'tracking_api:write' #'public_api:read_write' + +def on_error(error, items): + print("An error occurred: ", error) +analytics.debug = True +analytics.on_error = on_error + +analytics.track('AUser', 'track') +analytics.flush() + +time.sleep(3) \ No newline at end of file diff --git a/segment/analytics/__init__.py b/segment/analytics/__init__.py index 230769b5..ef35f67f 100644 --- a/segment/analytics/__init__.py +++ b/segment/analytics/__init__.py @@ -9,6 +9,7 @@ host = Client.DefaultConfig.host on_error = Client.DefaultConfig.on_error debug = Client.DefaultConfig.debug +log_handler = Client.DefaultConfig.log_handler send = Client.DefaultConfig.send sync_mode = Client.DefaultConfig.sync_mode max_queue_size = Client.DefaultConfig.max_queue_size @@ -16,6 +17,13 @@ timeout = Client.DefaultConfig.timeout max_retries = Client.DefaultConfig.max_retries +"""Oauth Settings.""" +oauth_client_id = Client.DefaultConfig.oauth_client_id +oauth_client_key = Client.DefaultConfig.oauth_client_key +oauth_key_id = Client.DefaultConfig.oauth_key_id +oauth_auth_server = Client.DefaultConfig.oauth_auth_server +oauth_scope = Client.DefaultConfig.oauth_scope + default_client = None @@ -73,7 +81,13 @@ def _proxy(method, *args, **kwargs): max_queue_size=max_queue_size, send=send, on_error=on_error, gzip=gzip, max_retries=max_retries, - sync_mode=sync_mode, timeout=timeout) + sync_mode=sync_mode, timeout=timeout, + oauth_client_id=oauth_client_id, + oauth_client_key=oauth_client_key, + oauth_key_id=oauth_key_id, + oauth_auth_server=oauth_auth_server, + oauth_scope=oauth_scope, + ) fn = getattr(default_client, method) return fn(*args, **kwargs) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 515da899..1d4e35da 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -6,6 +6,7 @@ import json from dateutil.tz import tzutc +from segment.analytics.oauth_manager import OauthManager from segment.analytics.utils import guess_timezone, clean from segment.analytics.consumer import Consumer, MAX_MSG_SIZE @@ -23,6 +24,7 @@ class DefaultConfig(object): host = None on_error = None debug = False + log_handler = None send = True sync_mode = False max_queue_size = 10000 @@ -33,6 +35,12 @@ class DefaultConfig(object): thread = 1 upload_interval = 0.5 upload_size = 100 + oauth_client_id = None + oauth_client_key = None + oauth_key_id = None + oauth_auth_server = 'https://oauth2.segment.io' + oauth_scope = 'tracking_api:write' + """Create a new Segment client.""" log = logging.getLogger('segment') @@ -51,7 +59,13 @@ def __init__(self, proxies=DefaultConfig.proxies, thread=DefaultConfig.thread, upload_size=DefaultConfig.upload_size, - upload_interval=DefaultConfig.upload_interval,): + upload_interval=DefaultConfig.upload_interval, + log_handler=DefaultConfig.log_handler, + oauth_client_id=DefaultConfig.oauth_client_id, + oauth_client_key=DefaultConfig.oauth_client_key, + oauth_key_id=DefaultConfig.oauth_key_id, + oauth_auth_server=DefaultConfig.oauth_auth_server, + oauth_scope=DefaultConfig.oauth_scope,): require('write_key', write_key, str) self.queue = queue.Queue(max_queue_size) @@ -64,9 +78,19 @@ def __init__(self, self.gzip = gzip self.timeout = timeout self.proxies = proxies + self.oauth_manager = None + if(oauth_client_id and oauth_client_key and oauth_key_id): + self.oauth_manager = OauthManager(oauth_client_id, oauth_client_key, oauth_key_id, + oauth_auth_server, oauth_scope, timeout, max_retries) + + if log_handler: + self.log.addHandler(log_handler) if debug: self.log.setLevel(logging.DEBUG) + if not log_handler: + # default log handler does not print debug or info + self.log.addHandler(logging.StreamHandler()) if sync_mode: self.consumers = None @@ -85,7 +109,7 @@ def __init__(self, self.queue, write_key, host=host, on_error=on_error, upload_size=upload_size, upload_interval=upload_interval, gzip=gzip, retries=max_retries, timeout=timeout, - proxies=proxies, + proxies=proxies, oauth_manager=self.oauth_manager, ) self.consumers.append(consumer) @@ -280,7 +304,8 @@ def _enqueue(self, msg): if self.sync_mode: self.log.debug('enqueued with blocking %s.', msg['type']) post(self.write_key, self.host, gzip=self.gzip, - timeout=self.timeout, proxies=self.proxies, batch=[msg]) + timeout=self.timeout, proxies=self.proxies, + oauth_manager=self.oauth_manager, batch=[msg]) return True, msg diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index 27586284..a78f2d34 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -14,6 +14,13 @@ # lower to leave space for extra data that will be added later, eg. "sentAt". BATCH_SIZE_LIMIT = 475000 +class FatalError(Exception): + def __init__(self, message): + self.message = message + def __str__(self): + msg = "[Segment] {0})" + return msg.format(self.message) + class Consumer(Thread): """Consumes the messages from the client's queue.""" @@ -21,7 +28,7 @@ class Consumer(Thread): def __init__(self, queue, write_key, upload_size=100, host=None, on_error=None, upload_interval=0.5, gzip=False, retries=10, - timeout=15, proxies=None): + timeout=15, proxies=None, oauth_manager=None): """Create a consumer thread.""" Thread.__init__(self) # Make consumer a daemon thread so that it doesn't block program exit @@ -41,6 +48,7 @@ def __init__(self, queue, write_key, upload_size=100, host=None, self.retries = retries self.timeout = timeout self.proxies = proxies + self.oauth_manager = oauth_manager def run(self): """Runs the consumer.""" @@ -118,6 +126,8 @@ def fatal_exception(exc): # with 429 status code (rate limited), # don't retry on other client errors return (400 <= exc.status < 500) and exc.status != 429 + elif isinstance(exc, FatalError): + return True else: # retry on all other errors (eg. network) return False @@ -129,6 +139,7 @@ def fatal_exception(exc): giveup=fatal_exception) def send_request(): post(self.write_key, self.host, gzip=self.gzip, - timeout=self.timeout, batch=batch, proxies=self.proxies) + timeout=self.timeout, batch=batch, proxies=self.proxies, + oauth_manager=self.oauth_manager) send_request() diff --git a/segment/analytics/oauth_manager.py b/segment/analytics/oauth_manager.py new file mode 100644 index 00000000..453a23a0 --- /dev/null +++ b/segment/analytics/oauth_manager.py @@ -0,0 +1,208 @@ +from datetime import date, datetime +import logging +import threading +import time +import uuid +from requests import sessions +import jwt + +from segment.analytics import utils +from segment.analytics.request import APIError +from segment.analytics.consumer import FatalError + +_session = sessions.Session() + +class OauthManager(object): + def __init__(self, + client_id, + client_key, + key_id, + auth_server='https://oauth2.segment.io', + scope='tracking_api:write', + timeout=15, + max_retries=3): + self.client_id = client_id + self.client_key = client_key + self.key_id = key_id + self.auth_server = auth_server + self.scope = scope + self.timeout = timeout + self.max_retries = max_retries + self.retry_count = 0 + self.clock_skew = 0 + + self.log = logging.getLogger('segment') + self.thread = None + self.token_mutex = threading.Lock() + self.token = None + self.error = None + + def get_token(self): + with self.token_mutex: + if self.token: + return self.token + # No good token, start the loop + self.log.debug("OAuth is enabled. No cached access token.") + # Make sure we're not waiting an excessively long time (this will not cancel 429 waits) + if self.thread and self.thread.is_alive(): + self.thread.cancel() + self.thread = threading.Timer(0,self._poller_loop) + self.thread.daemon = True + self.thread.start() + + while True: + # Wait for a token or error + with self.token_mutex: + if self.token: + return self.token + if self.error: + error = self.error + self.error = None + raise error + if self.thread: + # Wait for a cycle, may not have an answer immediately + self.thread.join(1) + + def clear_token(self): + self.log.debug("OAuth Token invalidated. Poller for new token is {}".format( + "active" if self.thread and self.thread.is_alive() else "stopped" )) + with self.token_mutex: + self.token = None + + def _request_token(self): + jwt_body = { + "iss": self.client_id, + "sub": self.client_id, + "aud": utils.remove_trailing_slash(self.auth_server), + "iat": int(time.time())-5 - self.clock_skew, + "exp": int(time.time()) + 55 - self.clock_skew, + "jti": str(uuid.uuid4()) + } + + signed_jwt = jwt.encode( + jwt_body, + self.client_key, + algorithm="RS256", + headers={"kid": self.key_id}, + ) + + request_body = 'grant_type=client_credentials&client_assertion_type='\ + 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer&'\ + 'client_assertion={}&scope={}'.format(signed_jwt, self.scope) + + token_endpoint = f'{utils.remove_trailing_slash(self.auth_server)}/token' + + self.log.debug("OAuth token requested from {} with size {}".format(token_endpoint, len(request_body))) + + res = _session.post(url=token_endpoint, data=request_body, timeout=self.timeout, + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + return res + + def _poller_loop(self): + refresh_timer_ms = 25 + response = None + + try: + response = self._request_token() + except Exception as e: + self.retry_count += 1 + if self.retry_count < self.max_retries: + self.log.debug("OAuth token request encountered an error on attempt {}: {}".format(self.retry_count ,e)) + self.thread = threading.Timer(refresh_timer_ms / 1000.0, self._poller_loop) + self.thread.daemon = True + self.thread.start() + return + # Too many retries, giving up + self.log.error("OAuth token request encountered an error after {} attempts: {}".format(self.retry_count ,e)) + self.error = FatalError(str(e)) + return + if response.headers.get("Date"): + try: + server_time = datetime.strptime(response.headers.get("Date"), "%a, %d %b %Y %H:%M:%S %Z") + self.clock_skew = int((datetime.utcnow() - server_time).total_seconds()) + except Exception as e: + self.log.error("OAuth token request received a response with an invalid Date header: {} | {}".format(response, e)) + + if response.status_code == 200: + data = None + try: + data = response.json() + except Exception as e: + self.retry_count += 1 + if self.retry_count < self.max_retries: + self.thread = threading.Timer(refresh_timer_ms / 1000.0, self._poller_loop) + self.thread.daemon = True + self.thread.start() + return + # Too many retries, giving up + self.error = e + return + try: + with self.token_mutex: + self.token = data['access_token'] + # success! + self.retry_count = 0 + except Exception as e: + # No access token in response? + self.log.error("OAuth token request received a successful response with a missing token: {}".format(response)) + try: + refresh_timer_ms = int(data['expires_in']) / 2 * 1000 + except Exception as e: + refresh_timer_ms = 60 * 1000 + + elif response.status_code == 429: + self.retry_count += 1 + rate_limit_reset_timestamp = None + try: + rate_limit_reset_timestamp = int(response.headers.get("X-RateLimit-Reset")) + except Exception as e: + self.log.error("OAuth rate limit response did not have a valid rest time: {} | {}".format(response, e)) + if rate_limit_reset_timestamp: + refresh_timer_ms = rate_limit_reset_timestamp - time.time() * 1000 + else: + refresh_timer_ms = 5 * 1000 + + self.log.debug("OAuth token request encountered a rate limit response, waiting {} ms".format(refresh_timer_ms)) + # We want subsequent calls to get_token to be able to interrupt our + # Timeout when it's waiting for e.g. a long normal expiration, but + # not when we're waiting for a rate limit reset. Sleep instead. + time.sleep(refresh_timer_ms / 1000.0) + refresh_timer_ms = 0 + elif response.status_code in [400, 401, 415]: + # unrecoverable errors (except for skew). APIError will be handled by request.py + self.retry_count = 0 + try: + payload = response.json() + + if (payload.get('error') == 'invalid_request' and + (payload.get('error_description') == 'Token is expired' or + payload.get('error_description') == 'Token used before issued')): + refresh_timer_ms = 0 # Retry immediately hopefully with a good skew value + self.thread = threading.Timer(refresh_timer_ms / 1000.0, self._poller_loop) + self.thread.daemon = True + self.thread.start() + return + + self.error = APIError(response.status_code, payload['error'], payload['error_description']) + except ValueError: + self.error = APIError(response.status_code, 'unknown', response.text) + self.log.error("OAuth token request error was unrecoverable, possibly due to configuration: {}".format(self.error)) + return + else: + # any other error + self.log.debug("OAuth token request error, attempt {}: [{}] {}".format(self.retry_count, response.status_code, response.reason)) + self.retry_count += 1 + + if self.retry_count > 0 and self.retry_count % self.max_retries == 0: + # every time we pass the retry count, put up an error to release any waiting token requests + try: + payload = response.json() + self.error = APIError(response.status_code, payload['error'], payload['error_description']) + except ValueError: + self.error = APIError(response.status_code, 'unknown', response.text) + self.log.error("OAuth token request error after {} attempts: {}".format(self.retry_count, self.error)) + + # loop + self.thread = threading.Timer(refresh_timer_ms / 1000.0, self._poller_loop) + self.thread.daemon = True + self.thread.start() diff --git a/segment/analytics/request.py b/segment/analytics/request.py index d1901f79..273247ce 100644 --- a/segment/analytics/request.py +++ b/segment/analytics/request.py @@ -13,19 +13,26 @@ _session = sessions.Session() -def post(write_key, host=None, gzip=False, timeout=15, proxies=None, **kwargs): +def post(write_key, host=None, gzip=False, timeout=15, proxies=None, oauth_manager=None, **kwargs): """Post the `kwargs` to the API""" log = logging.getLogger('segment') body = kwargs - body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat() + if not "sentAt" in body.keys(): + body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat() + body["writeKey"] = write_key url = remove_trailing_slash(host or 'https://api.segment.io') + '/v1/batch' - auth = HTTPBasicAuth(write_key, '') + auth = None + if oauth_manager: + auth = oauth_manager.get_token() data = json.dumps(body, cls=DatetimeSerializer) log.debug('making request: %s', data) headers = { 'Content-Type': 'application/json', 'User-Agent': 'analytics-python/' + VERSION } + if auth: + headers['Authorization'] = 'Bearer {}'.format(auth) + if gzip: headers['Content-Encoding'] = 'gzip' buf = BytesIO() @@ -37,26 +44,32 @@ def post(write_key, host=None, gzip=False, timeout=15, proxies=None, **kwargs): kwargs = { "data": data, - "auth": auth, "headers": headers, "timeout": 15, } if proxies: kwargs['proxies'] = proxies - - res = _session.post(url, data=data, auth=auth, - headers=headers, timeout=timeout) - + res = None + try: + res = _session.post(url, data=data, headers=headers, timeout=timeout) + except Exception as e: + log.error(e) + raise e + if res.status_code == 200: log.debug('data uploaded successfully') return res + if oauth_manager and res.status_code in [400, 401, 403]: + oauth_manager.clear_token() + try: payload = res.json() log.debug('received response: %s', payload) raise APIError(res.status_code, payload['code'], payload['message']) except ValueError: + log.error('Unknown error: [%s] %s', res.status_code, res.reason) raise APIError(res.status_code, 'unknown', res.text) diff --git a/segment/analytics/test/__init__.py b/segment/analytics/test/__init__.py index 09bf9b63..98ad6aa3 100644 --- a/segment/analytics/test/__init__.py +++ b/segment/analytics/test/__init__.py @@ -2,14 +2,13 @@ import pkgutil import logging import sys -import analytics - -from analytics.client import Client +import segment.analytics as analytics +from segment.analytics.client import Client def all_names(): for _, modname, _ in pkgutil.iter_modules(__path__): - yield 'analytics.test.' + modname + yield 'segment.analytics.test.' + modname def all(): @@ -32,6 +31,7 @@ def test_debug(self): analytics.debug = False analytics.flush() self.assertFalse(analytics.default_client.debug) + analytics.default_client.log.setLevel(0) # reset log level after debug enable def test_gzip(self): self.assertIsNone(analytics.default_client) @@ -45,9 +45,11 @@ def test_gzip(self): def test_host(self): self.assertIsNone(analytics.default_client) - analytics.host = 'test-host' + analytics.host = 'http://test-host' analytics.flush() - self.assertEqual(analytics.default_client.host, 'test-host') + self.assertEqual(analytics.default_client.host, 'http://test-host') + analytics.host = None + analytics.default_client = None def test_max_queue_size(self): self.assertIsNone(analytics.default_client) @@ -80,3 +82,6 @@ def test_timeout(self): def setUp(self): analytics.write_key = 'test-init' analytics.default_client = None + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/segment/analytics/test/client.py b/segment/analytics/test/client.py index f01dc2ce..bffdb1e5 100644 --- a/segment/analytics/test/client.py +++ b/segment/analytics/test/client.py @@ -3,13 +3,13 @@ import time import mock -from analytics.version import VERSION -from analytics.client import Client +from segment.analytics.version import VERSION +from segment.analytics.client import Client class TestClient(unittest.TestCase): - def fail(self, e, batch): + def fail(self, e, batch=[]): """Mark the failure handler""" self.failed = True @@ -294,6 +294,7 @@ def test_numeric_user_id(self): def test_debug(self): Client('bad_key', debug=True) + self.client.log.setLevel(0) # reset log level after debug enable def test_identify_with_date_object(self): client = self.client @@ -325,7 +326,7 @@ def mock_post_fn(*args, **kwargs): # the post function should be called 2 times, with a batch size of 10 # each time. - with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) \ + with mock.patch('segment.analytics.consumer.post', side_effect=mock_post_fn) \ as mock_post: for _ in range(20): client.identify('userId', {'trait': 'value'}) diff --git a/segment/analytics/test/consumer.py b/segment/analytics/test/consumer.py index 16d0b213..4a2a1e24 100644 --- a/segment/analytics/test/consumer.py +++ b/segment/analytics/test/consumer.py @@ -8,8 +8,8 @@ except ImportError: from Queue import Queue -from analytics.consumer import Consumer, MAX_MSG_SIZE -from analytics.request import APIError +from segment.analytics.consumer import Consumer, MAX_MSG_SIZE +from segment.analytics.request import APIError class TestConsumer(unittest.TestCase): @@ -59,7 +59,7 @@ def test_upload_interval(self): upload_interval = 0.3 consumer = Consumer(q, 'testsecret', upload_size=10, upload_interval=upload_interval) - with mock.patch('analytics.consumer.post') as mock_post: + with mock.patch('segment.analytics.consumer.post') as mock_post: consumer.start() for i in range(0, 3): track = { @@ -79,7 +79,7 @@ def test_multiple_uploads_per_interval(self): upload_size = 10 consumer = Consumer(q, 'testsecret', upload_size=upload_size, upload_interval=upload_interval) - with mock.patch('analytics.consumer.post') as mock_post: + with mock.patch('segment.analytics.consumer.post') as mock_post: consumer.start() for i in range(0, upload_size * 2): track = { @@ -110,7 +110,7 @@ def mock_post(*args, **kwargs): raise expected_exception mock_post.call_count = 0 - with mock.patch('analytics.consumer.post', + with mock.patch('segment.analytics.consumer.post', mock.Mock(side_effect=mock_post)): track = { 'type': 'track', @@ -190,7 +190,7 @@ def mock_post_fn(_, data, **kwargs): % len(data.encode())) return res - with mock.patch('analytics.request._session.post', + with mock.patch('segment.analytics.request._session.post', side_effect=mock_post_fn) as mock_post: consumer.start() for _ in range(0, n_msgs + 2): diff --git a/segment/analytics/test/module.py b/segment/analytics/test/module.py index 3901b1c7..e5fe598c 100644 --- a/segment/analytics/test/module.py +++ b/segment/analytics/test/module.py @@ -1,6 +1,6 @@ import unittest -import analytics +import segment.analytics as analytics class TestModule(unittest.TestCase): diff --git a/segment/analytics/test/oauth.py b/segment/analytics/test/oauth.py new file mode 100644 index 00000000..259342bc --- /dev/null +++ b/segment/analytics/test/oauth.py @@ -0,0 +1,155 @@ +from datetime import datetime +import threading +import time +import unittest +import mock +import sys +import os +sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../..")) +from segment.analytics.client import Client +import segment.analytics.oauth_manager +import requests + +privatekey = '''-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDVll7uJaH322IN +PQsH2aOXZJ2r1q+6hpVK1R5JV1p41PUzn8pOxyXFHWB+53dUd4B8qywKS36XQjp0 +VmhR1tQ22znQ9ZCM6y4LGeOJBjAZiFZLcGQNNrDFC0WGWTrK1ZTS2K7p5qy4fIXG +laNkMXiGGCawkgcHAdOvPTy8m1d9a6YSetYVmBP/tEYN95jPyZFIoHQfkQPBPr9W +cWPpdEBzasHV5d957akjurPpleDiD5as66UW4dkWXvS7Wu7teCLCyDApcyJKTb2Z +SXybmWjhIZuctZMAx3wT/GgW3FbkGaW5KLQgBUMzjpL0fCtMatlqckMD92ll1FuK +R+HnXu05AgMBAAECggEBAK4o2il4GDUh9zbyQo9ZIPLuwT6AZXRED3Igi3ykNQp4 +I6S/s9g+vQaY6LkyBnSiqOt/K/8NBiFSiJWaa5/n+8zrP56qzf6KOlYk+wsdN5Vq +PWtwLrUzljpl8YAWPEFunNa8hwwE42vfZbnDBKNLT4qQIOQzfnVxQOoQlfj49gM2 +iSrblvsnQTyucFy3UyTeioHbh8q2Xqcxry5WUCOrFDd3IIwATTpLZGw0IPeuFJbJ +NfBizLEcyJaM9hujQU8PRCRd16MWX+bbYM6Mh4dkT40QXWsVnHBHwsgPdQgDgseF +Na4ajtHoC0DlwYCXpCm3IzJfKfq/LR2q8NDUgKeF4AECgYEA9nD4czza3SRbzhpZ +bBoK77CSNqCcMAqyuHB0hp/XX3yB7flF9PIPb2ReO8wwmjbxn+bm7PPz2Uwd2SzO +pU+FXmyKJr53Jxw/hmDWZCoh42gsGDlVqpmytzsj74KlaYiMyZmEGbD7t/FGfNGV +LdLDJaHIYxEimFviOTXKCeKvPAECgYEA3d8tv4jdp1uAuRZiU9Z/tfw5mJOi3oXF +8AdFFDwaPzcTorEAxjrt9X6IjPbLIDJNJtuXYpe+dG6720KyuNnhLhWW9oZEJTwT +dUgqZ2fTCOS9uH0jSn+ZFlgTWI6UDQXRwE7z8avlhMIrQVmPsttGTo7V6sQVtGRx +bNj2RSVekTkCgYAJvy4UYLPHS0jWPfSLcfw8vp8JyhBjVgj7gncZW/kIrcP1xYYe +yfQSU8XmV40UjFfCGz/G318lmP0VOdByeVKtCV3talsMEPHyPqI8E+6DL/uOebYJ +qUqINK6XKnOgWOY4kvnGillqTQCcry1XQp61PlDOmj7kB75KxPXYrj6AAQKBgQDa ++ixCv6hURuEyy77cE/YT/Q4zYnL6wHjtP5+UKwWUop1EkwG6o+q7wtiul90+t6ah +1VUCP9X/QFM0Qg32l0PBohlO0pFrVnG17TW8vSHxwyDkds1f97N19BOT8ZR5jebI +sKPfP9LVRnY+l1BWLEilvB+xBzqMwh2YWkIlWI6PMQKBgGi6TBnxp81lOYrxVRDj +/3ycRnVDmBdlQKFunvfzUBmG1mG/G0YHeVSUKZJGX7w2l+jnDwIA383FcUeA8X6A +l9q+amhtkwD/6fbkAu/xoWNl+11IFoxd88y2ByBFoEKB6UVLuCTSKwXDqzEZet7x +mDyRxq7ohIzLkw8b8buDeuXZ +-----END PRIVATE KEY-----''' + +def mocked_requests_get(*args, **kwargs): + class MockResponse: + def __init__(self, data, status_code): + self.__dict__['headers'] = {'date': datetime.now().strftime("%a, %d %b %Y %H:%M:%S GMT")} + self.__dict__.update(data) + self.status_code = status_code + + def json(self): + return self.json_data + if 'url' not in kwargs: + kwargs['url'] = args[0] + if kwargs['url'] == 'http://127.0.0.1:80/token': + return MockResponse({"json_data" : {"access_token": "test_token", "expires_in": 4000}}, 200) + elif kwargs['url'] == 'http://127.0.0.1:400/token': + return MockResponse({"reason": "test_reason", "json_data" : {"error":"unrecoverable", "error_description":"nah"}}, 400) + elif kwargs['url'] == 'http://127.0.0.1:429/token': + return MockResponse({"reason": "test_reason", "headers" : {"X-RateLimit-Reset": time.time()*1000 + 2000}}, 429) + elif kwargs['url'] == 'http://127.0.0.1:500/token': + return MockResponse({"reason": "test_reason", "json_data" : {"error":"recoverable", "error_description":"nah"}}, 500) + elif kwargs['url'] == 'http://127.0.0.1:501/token': + if mocked_requests_get.error_count < 0 or mocked_requests_get.error_count > 0: + if mocked_requests_get.error_count > 0: + mocked_requests_get.error_count -= 1 + return MockResponse({"reason": "test_reason", "json_data" : {"error":"recoverable", "message":"nah"}}, 500) + else: # return the number of errors if set above 0 + mocked_requests_get.error_count = -1 + return MockResponse({"json_data" : {"access_token": "test_token", "expires_in": 4000}}, 200) + elif kwargs['url'] == 'https://api.segment.io/v1/batch': + return MockResponse({}, 200) + print("Unhandled mock URL") + return MockResponse({'text':'Unhandled mock URL error'}, 404) +mocked_requests_get.error_count = -1 + +class TestOauthManager(unittest.TestCase): + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_success(self, mock_post): + manager = segment.analytics.oauth_manager.OauthManager("id", privatekey, "keyid", "http://127.0.0.1:80") + self.assertEqual(manager.get_token(), "test_token") + self.assertEqual(manager.max_retries, 3) + self.assertEqual(manager.scope, "tracking_api:write") + self.assertEqual(manager.auth_server, "http://127.0.0.1:80") + self.assertEqual(manager.timeout, 15) + self.assertTrue(manager.thread.is_alive) + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_fail_unrecoverably(self, mock_post): + manager = segment.analytics.oauth_manager.OauthManager("id", privatekey, "keyid", "http://127.0.0.1:400") + with self.assertRaises(Exception) as context: + manager.get_token() + self.assertTrue(manager.thread.is_alive) + self.assertEqual(mock_post.call_count, 1) + manager.thread.cancel() + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_fail_with_retries(self, mock_post): + manager = segment.analytics.oauth_manager.OauthManager("id", privatekey, "keyid", "http://127.0.0.1:500") + with self.assertRaises(Exception) as context: + manager.get_token() + self.assertTrue(manager.thread.is_alive) + self.assertEqual(mock_post.call_count, 3) + manager.thread.cancel() + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + @mock.patch('time.sleep', spec=time.sleep) # 429 uses sleep so it won't be interrupted + def test_oauth_rate_limit_delay(self, mock_sleep, mock_post): + manager = segment.analytics.oauth_manager.OauthManager("id", privatekey, "keyid", "http://127.0.0.1:429") + manager._poller_loop() + self.assertTrue(mock_sleep.call_args[0][0] > 1.9 and mock_sleep.call_args[0][0] <= 2.0) + +class TestOauthIntegration(unittest.TestCase): + def fail(self, e, batch=[]): + self.failed = True + + def setUp(self): + self.failed = False + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_integration_success(self, mock_post): + client = Client("write_key", on_error=self.fail, oauth_auth_server="http://127.0.0.1:80", + oauth_client_id="id",oauth_client_key=privatekey, oauth_key_id="keyid") + client.track("user", "event") + client.flush() + self.assertFalse(self.failed) + self.assertEqual(mock_post.call_count, 2) + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_integration_failure(self, mock_post): + client = Client("write_key", on_error=self.fail, oauth_auth_server="http://127.0.0.1:400", + oauth_client_id="id",oauth_client_key=privatekey, oauth_key_id="keyid") + client.track("user", "event") + client.flush() + self.assertTrue(self.failed) + self.assertEqual(mock_post.call_count, 1) + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_integration_recovery(self, mock_post): + mocked_requests_get.error_count = 2 # 2 errors and then success + client = Client("write_key", on_error=self.fail, oauth_auth_server="http://127.0.0.1:501", + oauth_client_id="id",oauth_client_key=privatekey, oauth_key_id="keyid") + client.track("user", "event") + client.flush() + self.assertFalse(self.failed) + self.assertEqual(mock_post.call_count, 4) + + @mock.patch.object(requests.Session, 'post', side_effect=mocked_requests_get) + def test_oauth_integration_fail_bad_key(self, mock_post): + client = Client("write_key", on_error=self.fail, oauth_auth_server="http://127.0.0.1:80", + oauth_client_id="id",oauth_client_key="badkey", oauth_key_id="keyid") + client.track("user", "event") + client.flush() + self.assertTrue(self.failed) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/segment/analytics/test/request.py b/segment/analytics/test/request.py index 3420deca..3f40c497 100644 --- a/segment/analytics/test/request.py +++ b/segment/analytics/test/request.py @@ -3,7 +3,7 @@ import json import requests -from analytics.request import post, DatetimeSerializer +from segment.analytics.request import post, DatetimeSerializer class TestRequests(unittest.TestCase): diff --git a/segment/analytics/test/utils.py b/segment/analytics/test/utils.py index 6995e799..43a6fd4b 100644 --- a/segment/analytics/test/utils.py +++ b/segment/analytics/test/utils.py @@ -4,7 +4,7 @@ from dateutil.tz import tzutc -from analytics import utils +from segment.analytics import utils class TestUtils(unittest.TestCase): diff --git a/setup.py b/setup.py index fbc1d066..0f3da9ed 100644 --- a/setup.py +++ b/setup.py @@ -40,8 +40,8 @@ author_email='friends@segment.com', maintainer='Segment', maintainer_email='friends@segment.com', - test_suite='analytics.test.all', - packages=['segment.analytics', 'analytics.test'], + test_suite='segment.analytics.test.all', + packages=['segment.analytics', 'segment.analytics.test'], python_requires='>=3.6.0', license='MIT License', install_requires=install_requires, From e83dc0d005a94b5c971ea5a0b079a5aefc88f97b Mon Sep 17 00:00:00 2001 From: Rudyard Richter Date: Wed, 25 Oct 2023 11:38:43 -0600 Subject: [PATCH 295/323] Update from monotonic to time module (#310) --- requirements.txt | 1 - segment/analytics/consumer.py | 6 +++--- setup.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5eaf3fcf..bc5a7159 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,6 @@ MarkupSafe==2.1.3 mccabe==0.6.1 mistune==0.8.4 mock==2.0.0 -monotonic==1.6 multidict==5.1.0 packaging==21.0 pbr==5.11.1 diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index a78f2d34..1c471573 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -1,6 +1,6 @@ import logging +import time from threading import Thread -import monotonic import backoff import json @@ -88,11 +88,11 @@ def next(self): queue = self.queue items = [] - start_time = monotonic.monotonic() + start_time = time.monotonic() total_size = 0 while len(items) < self.upload_size: - elapsed = monotonic.monotonic() - start_time + elapsed = time.monotonic() - start_time if elapsed >= self.upload_interval: break try: diff --git a/setup.py b/setup.py index 0f3da9ed..8f8edfe2 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,6 @@ install_requires = [ "requests~=2.7", - "monotonic~=1.5", "backoff~=2.1", "python-dateutil~=2.2" ] From 9423077c901a447de035772d52686a31c4ee18d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:47:29 -0800 Subject: [PATCH 296/323] Bump m2r from 0.2.1 to 0.3.1 (#301) Bumps [m2r](https://github.com/miyakogi/m2r) from 0.2.1 to 0.3.1. - [Changelog](https://github.com/miyakogi/m2r/blob/dev/CHANGES.md) - [Commits](https://github.com/miyakogi/m2r/compare/v0.2.1...v0.3.1) --- updated-dependencies: - dependency-name: m2r dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bc5a7159..c29f55af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ Jinja2==3.0.1 jmespath==1.0.1 keyring==23.5.0 lazy-object-proxy==1.9.0 -m2r==0.2.1 +m2r==0.3.1 MarkupSafe==2.1.3 mccabe==0.6.1 mistune==0.8.4 From 1510fa9e0a40fd20b88b54b66c94085ba0470271 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 09:09:22 -0800 Subject: [PATCH 297/323] Bump tqdm from 4.63.0 to 4.66.1 (#296) Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.63.0 to 4.66.1. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.63.0...v4.66.1) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c29f55af..fb7a1e1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -65,7 +65,7 @@ shellingham==1.4.0 six==1.15.0 termcolor==1.1.0 tomlkit==0.7.2 -tqdm==4.63.0 +tqdm==4.66.1 twine==3.8.0 typing-extensions==3.10.0.2 urllib3==1.26.6 From 211a8f306ab1e5b10d97650b457d5ff58b3d43d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 09:09:53 -0800 Subject: [PATCH 298/323] Bump backoff from 1.10.0 to 2.2.1 (#297) Bumps [backoff](https://github.com/litl/backoff) from 1.10.0 to 2.2.1. - [Release notes](https://github.com/litl/backoff/releases) - [Changelog](https://github.com/litl/backoff/blob/master/CHANGELOG.md) - [Commits](https://github.com/litl/backoff/compare/v1.10.0...v2.2.1) --- updated-dependencies: - dependency-name: backoff dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fb7a1e1e..799a7986 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ appdirs==1.4.4 astroid==1.6.6 async-timeout==3.0.1 attrs==23.1.0 -backoff==1.10.0 +backoff==2.2.1 bleach==6.0.0 botocore==1.29.40 Cerberus==1.3.5 From 72612edf42200ee44746aa78536b5cde3a8bcb2c Mon Sep 17 00:00:00 2001 From: David Cain Date: Fri, 26 Jan 2024 10:11:33 -0700 Subject: [PATCH 299/323] Correct HISTORY header for most recent release (#365) The most recent release was 2.2.3, not 2.3.2 -- we can fix the log. While we're at it, can fix a few typos in this file too. --- HISTORY.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index ec09d0bc..77bd1672 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,5 @@ -# 2.3.2 / 2023-06-12 -- Update project to use github actions +# 2.2.3 / 2023-06-12 +- Update project to use GitHub Actions - Support for Python 3.10 and 3.11 - Return values for function calls via the proxy @@ -52,7 +52,7 @@ - Add `shutdown` function - Add gzip support - Add exponential backoff with jitter when retrying -- Add a paramater in Client to configure max retries +- Add a parameter in Client to configure max retries - Limit batch upload size to 500KB - Drop messages greater than 32kb - Allow user-defined upload size @@ -73,7 +73,7 @@ # 1.2.6 / 2016-12-07 -- dont add messages to the queue if send is false +- don't add messages to the queue if send is false - drop py32 support # 1.2.5 / 2016-07-02 @@ -129,7 +129,7 @@ # 1.0.1 / 2014-09-08 -- fixing unicode handling, for write_key and events +- fixing Unicode handling, for write_key and events - adding six to requirements.txt and install scripts # 1.0.0 / 2014-09-05 @@ -139,8 +139,8 @@ - moving to analytics.write_key API - moving consumer to a separate thread - adding request retries -- making analytics.flush() syncrhonous -- adding full travis tests +- making analytics.flush() synchronous +- adding full Travis tests # 0.4.4 / 2013-11-21 From dedc53d976978557ffefda67948920d1ee5c63b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 09:12:19 -0800 Subject: [PATCH 300/323] Bump jinja2 from 3.0.1 to 3.1.3 (#406) Bumps [jinja2](https://github.com/pallets/jinja) from 3.0.1 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.0.1...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 799a7986..512f8a54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ git-remote-codecommit==1.16 idna==3.4 importlib-metadata==4.11.2 isort==5.9.3 -Jinja2==3.0.1 +Jinja2==3.1.3 jmespath==1.0.1 keyring==23.5.0 lazy-object-proxy==1.9.0 From f3274e6cbdf90de8485b52d5b0cda3c8f9663ca4 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Wed, 31 Jan 2024 14:43:52 -0500 Subject: [PATCH 301/323] Release 2.3.0. --- HISTORY.md | 6 ++++++ README.md | 3 --- segment/analytics/version.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 77bd1672..ff9d88c8 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,9 @@ +# 2.3.0 / 2024-01-29 +- OAuth 2.0 support +- Adding Python 3.10 and 3.11 classifiers by @mvinogradov-wavefin +- Update from monotonic to time module by @rudyardrichter +- Correct HISTORY header for most recent release by @DavidCain + # 2.2.3 / 2023-06-12 - Update project to use GitHub Actions - Support for Python 3.10 and 3.11 diff --git a/README.md b/README.md index 69cefe7c..12dd2fb5 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ analytics-python ============== -[![CircleCI](https://circleci.com/gh/segmentio/analytics-python/tree/master.svg?style=svg&circle-token=c0b411a3e21943918294714ad1d75a1cfc718f79)](https://circleci.com/gh/segmentio/analytics-python/tree/master) - - analytics-python is a python client for [Segment](https://segment.com)
diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 2914e9b7..0f572f9f 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.2.3' +VERSION = '2.3.0' From d3fe3abeb3811e3783814e38069818b2fc16fb70 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Wed, 7 Feb 2024 15:13:15 -0500 Subject: [PATCH 302/323] Release 2.3.1 --- HISTORY.md | 3 +++ segment/analytics/version.py | 2 +- setup.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index ff9d88c8..9dee9377 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +# 2.3.1 / 2024-02-07 +- Fixing dependency for JWT + # 2.3.0 / 2024-01-29 - OAuth 2.0 support - Adding Python 3.10 and 3.11 classifiers by @mvinogradov-wavefin diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 0f572f9f..7e696978 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.3.0' +VERSION = '2.3.1' diff --git a/setup.py b/setup.py index 8f8edfe2..2cec9ff5 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,8 @@ install_requires = [ "requests~=2.7", "backoff~=2.1", - "python-dateutil~=2.2" + "python-dateutil~=2.2", + "PyJWT~=2.8.0" ] tests_require = [ From fd028a2bacd28fbb6d60ce305340592200e84f00 Mon Sep 17 00:00:00 2001 From: Alan Charles <50601149+alanjcharles@users.noreply.github.com> Date: Tue, 13 Feb 2024 09:50:09 -0700 Subject: [PATCH 303/323] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 12dd2fb5..11a75110 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ analytics-python is a python client for [Segment](https://segment.com) +### ⚠️ Maintenance ⚠️ +This library is in maintenance mode. It will send data as intended, but receive no new feature support and only critical maintenance updates from Segment. +

You can't fix what you can't measure

From 2c7cf031631315c10217a9e9bcefeff7c4968a3f Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 15 Feb 2024 10:43:33 -0500 Subject: [PATCH 304/323] Release 2.3.2. --- HISTORY.md | 3 +++ segment/analytics/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 9dee9377..fa7009c4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +# 2.3.2 / 2024-02-15 +- Updating version to create a release wheel without the outdated /analytics files + # 2.3.1 / 2024-02-07 - Fixing dependency for JWT diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 7e696978..a5ac6164 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.3.1' +VERSION = '2.3.2' From acbaa9f3d26fbc5538eba4cc1c7cd16267dfa117 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Mon, 26 Feb 2024 11:04:58 -0600 Subject: [PATCH 305/323] Issue #344 fix tests (#425) * Update syntax and test * Push to rerun tests * Syntax update * Push to test * made a change to force action * Update Readme to reflect new workflows * Remove CI folder and files * Update syntax * Fix issues with run rows * Adding workflows for manual execution * Update run on push for tests * Removing workflow * Remove version suffix * Trying a suggestion from user * removed * Reset * Update syntax * Update per docs * Update make statement * Trying to fix some errors * Forgot only one run per statement * Update pylint * enable e2e ENV variable * Updating test script * Update node version requirements * Remove e2e test for now and update test command * adding dateutil to test * set up required modules * Update scripts for testing * Update syntax and test * Push to rerun tests * Syntax update * Push to test * made a change to force action * Update Readme to reflect new workflows * Remove CI folder and files * Update syntax * Fix issues with run rows * Adding workflows for manual execution * Update run on push for tests * Removing workflow * Remove version suffix * Trying a suggestion from user * removed * Reset * Update syntax * Update per docs * Update make statement * Trying to fix some errors * Forgot only one run per statement * Update pylint * enable e2e ENV variable * Updating test script * Update node version requirements * Remove e2e test for now and update test command * adding dateutil to test * set up required modules * Update scripts for testing * Moving tests to new file * Update test Suite for python 3 requirements * Updating requirements.txt to include only necessary packages * Update label * Adding in requirements * Remove dephell from requirements --- .circleci/config.yml | 110 --- .github/workflows/main.yml | 4 +- .github/workflows/tests.yml | 52 +- .pylintrc | 711 +++++++++--------- README.md | 7 + requirements.txt | 72 +- segment/analytics/test/__init__.py | 87 --- .../test/{client.py => test_client.py} | 0 .../test/{consumer.py => test_consumer.py} | 0 segment/analytics/test/test_init.py | 87 +++ .../test/{module.py => test_module.py} | 0 .../test/{oauth.py => test_oauth.py} | 0 .../test/{request.py => test_request.py} | 0 .../test/{utils.py => test_utils.py} | 0 14 files changed, 481 insertions(+), 649 deletions(-) delete mode 100644 .circleci/config.yml rename segment/analytics/test/{client.py => test_client.py} (100%) rename segment/analytics/test/{consumer.py => test_consumer.py} (100%) create mode 100644 segment/analytics/test/test_init.py rename segment/analytics/test/{module.py => test_module.py} (100%) rename segment/analytics/test/{oauth.py => test_oauth.py} (100%) rename segment/analytics/test/{request.py => test_request.py} (100%) rename segment/analytics/test/{utils.py => test_utils.py} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index bb80c74e..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,110 +0,0 @@ -version: 2 -defaults: - taggedReleasesFilter: &taggedReleasesFilter - tags: - only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. -jobs: - build: - docker: - - image: circleci/python:3.8 - steps: - - checkout - - run: pip3 install python-dateutil backoff monotonic - - run: pip3 install --user . - - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil - - run: make test - - store_artifacts: - path: pylint.out - - store_artifacts: - path: flake8.out - - snyk: - docker: - - image: circleci/python:3.9 - steps: - - checkout - - attach_workspace: { at: . } - - run: pip3 install pipreqs - - run: pip3 install --user appdirs - - run: pipreqs . - - run: pip3 install --user -r requirements.txt - - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh - - test_37: &test - docker: - - image: circleci/python:3.7 - steps: - - checkout - - run: pip3 install python-dateutil backoff monotonic - - run: pip3 install --user .[test] - - run: - name: Linting with Flake8 - command: | - git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics - - run: make test - - run: make e2e_test - - test_38: - <<: *test - docker: - - image: circleci/python:3.8 - - test_39: - <<: *test - docker: - - image: circleci/python:3.9 - - publish: - docker: - - image: circleci/python:3.9 - steps: - - checkout - - run: sudo pip install twine - - run: make release - -workflows: - version: 2 - build_test_release: - jobs: - - build: - filters: - <<: *taggedReleasesFilter - - test_37: - filters: - <<: *taggedReleasesFilter - - test_38: - filters: - <<: *taggedReleasesFilter - - test_39: - filters: - <<: *taggedReleasesFilter - - publish: - requires: - - build - - test_37 - - test_38 - - test_39 - filters: - <<: *taggedReleasesFilter - branches: - ignore: /.*/ - static_analysis: - jobs: - - build - - snyk: - context: snyk - requires: - - build - scheduled_e2e_test: - triggers: - - schedule: - cron: "0 * * * *" - filters: - branches: - only: - - master - - scheduled_e2e_testing - jobs: - - test_37 - - test_38 - - test_39 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3d54fd2a..a17f8887 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,9 +21,9 @@ jobs: pip install -r requirements.txt pip install python-dateutil backoff monotonic pip install --user . - sudo pip install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil + sudo pip install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil aiohttp==3.9.1 - name: Run tests - run: make e2e_test + run: python -m unittest discover -s segment # snyk: # runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d1c621d..257b7077 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,14 +1,15 @@ -name: e2e tests +name: analytics test suite on: push: branches: - master + - '**Tests**' paths-ignore: - '**.md' pull_request: paths-ignore: - - '**.md' + - '**.md' jobs: test-setup-python: @@ -16,44 +17,49 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run with setup-python 3.7 - uses: ./ + uses: actions/setup-python@v5 with: - python-version: 3.7 + python-version: '3.7' + - name: Setup required modules + run: python -m pip install -r requirements.txt - name: Run tests - run: make test - run: make e2e_test + run: python -m unittest discover -s segment - name: Run with setup-python 3.8 - uses: ./ + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: '3.8' + - name: Setup required modules + run: python -m pip install -r requirements.txt - name: Run tests - run: make test - run: make e2e_test + run: python -m unittest discover -s segment - name: Run with setup-python 3.9 - uses: ./ + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.9' + - name: Setup required modules + run: python -m pip install -r requirements.txt - name: Run tests - run: make test - run: make e2e_test + run: python -m unittest discover -s segment - name: Run with setup-python 3.10 - uses: ./ + uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' + - name: Setup required modules + run: python -m pip install -r requirements.txt - name: Run tests - run: make test - run: make e2e_test + run: python -m unittest discover -s segment - name: Run with setup-python 3.11 - uses: ./ + uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: '3.11' + - name: Setup required modules + run: python -m pip install -r requirements.txt - name: Run tests - run: make test - run: make e2e_test \ No newline at end of file + run: python -m unittest discover -s segment \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 568c4cc2..4712a015 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,184 +1,139 @@ -[MASTER] - -# Add files or directories to the ignore list. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the denylist. The -# regex matches against base names, not paths. -ignore-patterns= +[MAIN] # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= -# Use multiple processes to speed up Pylint. -jobs=1 +# Files or directories to be skipped. They should be base names, not +# paths. +ignore=CVS -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= +# Add files or directories matching the regex patterns to the ignore-list. The +# regex matches against paths and can be in Posix or Windows format. +ignore-paths= + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns=^\.# # Pickle collected data for later comparisons. persistent=yes -# Specify a configuration file. -#rcfile= +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + pylint.extensions.check_elif, + pylint.extensions.bad_builtin, + pylint.extensions.docparams, + pylint.extensions.for_any_all, + pylint.extensions.set_membership, + pylint.extensions.code_style, + pylint.extensions.overlapping_exceptions, + pylint.extensions.typing, + pylint.extensions.redefined_variable_type, + pylint.extensions.comparison_placement, + pylint.extensions.broad_try_clause, + pylint.extensions.dict_init_mutate, + pylint.extensions.consider_refactoring_into_while_condition, + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=1 # When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages +# user-friendly hints instead of false-positive error messages. suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-allow-list= + +# Minimum supported python version +py-version = 3.8.0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# Specify a score threshold under which the program will exit with error. +fail-under=10.0 + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint in +# a server-like mode. +clear-cache-post-run=no + [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= +# confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + use-symbolic-message-instead, + useless-suppression, # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if +# disable everything first and then re-enable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=too-many-public-methods, - no-else-return, - print-statement, - invalid-name, - global-statement, - too-many-arguments, - missing-docstring, - too-many-instance-attributes, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - invalid-unicode-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - locally-enabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member +disable= + attribute-defined-outside-init, + invalid-name, + missing-docstring, + protected-access, + too-few-public-methods, + # handled by black + format, + # We anticipate #3512 where it will become optional + fixme, + consider-using-assignment-expr, [REPORTS] -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. output-format=text # Tells whether to display a full report or only the messages reports=no -# Activate the evaluation score. -score=yes - - -[REFACTORING] +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention' +# and 'info', which contain the number of messages in each category, as +# well as 'statement', which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=optparse.Values,sys.exit +# Activate the evaluation score. +score=yes [LOGGING] @@ -187,37 +142,25 @@ never-returning-functions=optparse.Values,sys.exit # function parameter format logging-modules=logging - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO +notes=FIXME,XXX,TODO + +# Regular expression of note tags to take in consideration. +#notes-rgx= [SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=6 + # Ignore comments when computing similarities. ignore-comments=yes @@ -225,281 +168,274 @@ ignore-comments=yes ignore-docstrings=yes # Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 +ignore-imports=yes - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 +# Signatures are removed from the similarity computation +ignore-signatures=yes [VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=no + # List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. +# you should avoid defining new builtins when possible. additional-builtins= -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. -callbacks=cb_, - _cb +callbacks=cb_,_cb -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes -# Tells whether we should check for unused import in __init__ files. -init-import=no +# List of names allowed to shadow builtins +allowed-redefined-builtins= # List of qualified module names which can have objects that can redefine # builtins. -redefining-builtins-modules=past.builtins,future.builtins,io,builtins +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io [FORMAT] -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Maximum number of characters on a single line. +max-line-length=100 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Maximum number of lines in a module +max-module-lines=2000 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' -# Maximum number of characters on a single line. -max-line-length=100 +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 -# Maximum number of lines in a module -max-module-lines=1000 +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no +[BASIC] -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= -[BASIC] +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata -# Naming style matching correct argument names -argument-naming-style=snake_case +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct variable names. +variable-naming-style=snake_case -# Regular expression matching correct argument names. Overrides argument- -# naming-style -#argument-rgx= +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ -# Naming style matching correct attribute names +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming style matching correct attribute names. attr-naming-style=snake_case -# Regular expression matching correct attribute names. Overrides attr-naming- -# style -#attr-rgx= +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,}$ -# Bad variable names which should always be refused, separated by a comma -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming style matching correct class attribute names. class-attribute-naming-style=any -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style -#class-attribute-rgx= +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. +#class-const-rgx= -# Naming style matching correct class names +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming style matching correct class names. class-naming-style=PascalCase -# Regular expression matching correct class names. Overrides class-naming-style -#class-rgx= +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ -# Naming style matching correct constant names -const-naming-style=UPPER_CASE -# Regular expression matching correct constant names. Overrides const-naming- -# style -#const-rgx= +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,}$ + +# Regular expression matching correct type variable names +#typevar-rgx= + +# Regular expression which should only match function or class names that do +# not require a docstring. Use ^(?!__init__$)_ to also check __init__. +no-docstring-rgx=__.*__ # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 -# Naming style matching correct function names -function-naming-style=snake_case +# List of decorators that define properties, such as abc.abstractproperty. +property-classes=abc.abstractproperty -# Regular expression matching correct function names. Overrides function- -# naming-style -#function-rgx= -# Good variable names which should always be accepted, separated by a comma -good-names=i, - j, - k, - ex, - Run, - _ +[TYPECHECK] -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no +# Regex pattern to define which classes are considered mixins if ignore-mixin- +# members is set to 'yes' +mixin-class-rgx=.*MixIn -# Naming style matching correct inline iteration names -inlinevar-naming-style=any +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style -#inlinevar-rgx= +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local -# Naming style matching correct method names -method-naming-style=snake_case +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent,argparse.Namespace -# Regular expression matching correct method names. Overrides method-naming- -# style -#method-rgx= +# List of decorators that create context managers from functions, such as +# contextlib.contextmanager. +contextmanager-decorators=contextlib.contextmanager -# Naming style matching correct module names -module-naming-style=snake_case +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes -# Regular expression matching correct module names. Overrides module-naming- -# style -#module-rgx= +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 -# Naming style matching correct variable names -variable-naming-style=snake_case +[SPELLING] -# Regular expression matching correct variable names. Overrides variable- -# naming-style -#variable-rgx= +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= +# List of comma separated words that should not be checked. +spelling-ignore-words= -[DESIGN] +# List of comma separated words that should be considered directives if they +# appear and the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:,pragma:,# noinspection -# Maximum number of arguments for function / method -max-args=5 +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file=.pyenchant_pylint_custom_dict.txt -# Maximum number of attributes for a class (see R0902). -max-attributes=7 +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=2 -# Maximum number of branch for function / method body -max-branches=12 -# Maximum number of locals for function / method body -max-locals=20 +[DESIGN] -# Maximum number of parents for a class (see R0901). -max-parents=7 +# Maximum number of arguments for function / method +max-args = 9 -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 +# Maximum number of locals for function / method body +max-locals = 19 # Maximum number of return / yield for function / method body -max-returns=6 +max-returns=11 + +# Maximum number of branch for function / method body +max-branches = 20 # Maximum number of statements in function / method body -max-statements=50 +max-statements = 50 -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 +# Maximum number of attributes for a class (see R0902). +max-attributes=11 +# Maximum number of statements in a try-block +max-try-statements = 7 [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make +defining-attr-methods=__init__,__new__,setUp,__post_init__ # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls @@ -507,31 +443,41 @@ valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no [IMPORTS] +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + # Allow wildcard imports from modules that define __all__. allow-wildcard-with-all=no +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + # Analyse import fallback blocks. This can be used to support both Python 2 and # 3 compatible code, which means that the block might have code that exists # only in one or another interpreter, leading to false positives when analysed. analyse-fallback-blocks=no # Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= +deprecated-modules=regsub,TERMIOS,Bastion,rexec # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= @@ -543,9 +489,60 @@ known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception + + +[TYPING] + +# Set to ``no`` if the app / library does **NOT** need to support runtime +# introspection of type annotations. If you use type annotations +# **exclusively** for type checking of an application, you're probably fine. +# For libraries, evaluate if some users what to access the type hints at +# runtime first, e.g., through ``typing.get_type_hints``. Applies to Python +# versions 3.7 - 3.9 +runtime-typing = no + + +[DEPRECATED_BUILTINS] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,input + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[CODE_STYLE] + +# Max line length for which to sill emit suggestions. Used to prevent optional +# suggestions which would get split by a code formatter (e.g., black). Will +# default to the setting for ``max-line-length``. +#max-line-length-suggestions= \ No newline at end of file diff --git a/README.md b/README.md index 11a75110..fdcd72cf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ analytics-python ============== +======= +[![Run Python Tests](https://github.com/North-Two-Five/analytics-python/actions/workflows/main.yml/badge.svg)](https://github.com/North-Two-Five/analytics-python/actions/workflows/main.yml) +[![.github/workflows/tests.yml](https://github.com/North-Two-Five/analytics-python/actions/workflows/tests.yml/badge.svg)](https://github.com/North-Two-Five/analytics-python/actions/workflows/tests.yml) +======= + + + analytics-python is a python client for [Segment](https://segment.com) ### ⚠️ Maintenance ⚠️ diff --git a/requirements.txt b/requirements.txt index 512f8a54..cf0cbbee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,76 +1,8 @@ -aiohttp==3.7.4.post0 -appdirs==1.4.4 -astroid==1.6.6 -async-timeout==3.0.1 -attrs==23.1.0 backoff==2.2.1 -bleach==6.0.0 -botocore==1.29.40 -Cerberus==1.3.5 -certifi==2023.7.22 -chardet==4.0.0 -charset-normalizer==3.2.0 -colorama==0.4.4 -dephell==0.8.3 -dephell-archive==0.1.7 -dephell-argparse==0.1.3 -dephell-changelogs==0.0.1 -dephell-discover==0.2.10 -dephell-licenses==0.1.7 -dephell-links==0.1.5 -dephell-markers==1.0.3 -dephell-pythons==0.1.15 -dephell-setuptools==0.2.5 -dephell-shells==0.1.5 -dephell-specifier==0.2.2 -dephell-venvs==0.1.18 -dephell-versioning==0.1.2 -docutils==0.20.1 -entrypoints==0.3 +cryptography==41.0.3 flake8==3.7.9 -git-remote-codecommit==1.16 -idna==3.4 -importlib-metadata==4.11.2 -isort==5.9.3 -Jinja2==3.1.3 -jmespath==1.0.1 -keyring==23.5.0 -lazy-object-proxy==1.9.0 -m2r==0.3.1 -MarkupSafe==2.1.3 -mccabe==0.6.1 -mistune==0.8.4 mock==2.0.0 -multidict==5.1.0 -packaging==21.0 -pbr==5.11.1 -pexpect==4.8.0 -pkginfo==1.9.6 -protobuf==4.24.2 -ptyprocess==0.7.0 -pycodestyle==2.5.0 -pyflakes==2.1.1 -Pygments==2.16.1 PyJWT==2.8.0 -pylint==1.9.3 -pyparsing==3.1.1 +pylint==2.8.0 python-dateutil==2.8.2 -readme-renderer==32.0 requests==2.31.0 -requests-toolbelt==1.0.0 -rfc3986==2.0.0 -ruamel.yaml==0.17.32 -ruamel.yaml.clib==0.2.7 -shellingham==1.4.0 -six==1.15.0 -termcolor==1.1.0 -tomlkit==0.7.2 -tqdm==4.66.1 -twine==3.8.0 -typing-extensions==3.10.0.2 -urllib3==1.26.6 -webencodings==0.5.1 -wrapt==1.12.1 -yarl==1.9.2 -yaspin==2.1.0 -zipp==3.7.0 diff --git a/segment/analytics/test/__init__.py b/segment/analytics/test/__init__.py index 98ad6aa3..e69de29b 100644 --- a/segment/analytics/test/__init__.py +++ b/segment/analytics/test/__init__.py @@ -1,87 +0,0 @@ -import unittest -import pkgutil -import logging -import sys -import segment.analytics as analytics -from segment.analytics.client import Client - - -def all_names(): - for _, modname, _ in pkgutil.iter_modules(__path__): - yield 'segment.analytics.test.' + modname - - -def all(): - logging.basicConfig(stream=sys.stderr) - return unittest.defaultTestLoader.loadTestsFromNames(all_names()) - - -class TestInit(unittest.TestCase): - def test_writeKey(self): - self.assertIsNone(analytics.default_client) - analytics.flush() - self.assertEqual(analytics.default_client.write_key, 'test-init') - - def test_debug(self): - self.assertIsNone(analytics.default_client) - analytics.debug = True - analytics.flush() - self.assertTrue(analytics.default_client.debug) - analytics.default_client = None - analytics.debug = False - analytics.flush() - self.assertFalse(analytics.default_client.debug) - analytics.default_client.log.setLevel(0) # reset log level after debug enable - - def test_gzip(self): - self.assertIsNone(analytics.default_client) - analytics.gzip = True - analytics.flush() - self.assertTrue(analytics.default_client.gzip) - analytics.default_client = None - analytics.gzip = False - analytics.flush() - self.assertFalse(analytics.default_client.gzip) - - def test_host(self): - self.assertIsNone(analytics.default_client) - analytics.host = 'http://test-host' - analytics.flush() - self.assertEqual(analytics.default_client.host, 'http://test-host') - analytics.host = None - analytics.default_client = None - - def test_max_queue_size(self): - self.assertIsNone(analytics.default_client) - analytics.max_queue_size = 1337 - analytics.flush() - self.assertEqual(analytics.default_client.queue.maxsize, 1337) - - def test_max_retries(self): - self.assertIsNone(analytics.default_client) - client = Client('testsecret', max_retries=42) - for consumer in client.consumers: - self.assertEqual(consumer.retries, 42) - - def test_sync_mode(self): - self.assertIsNone(analytics.default_client) - analytics.sync_mode = True - analytics.flush() - self.assertTrue(analytics.default_client.sync_mode) - analytics.default_client = None - analytics.sync_mode = False - analytics.flush() - self.assertFalse(analytics.default_client.sync_mode) - - def test_timeout(self): - self.assertIsNone(analytics.default_client) - analytics.timeout = 1.234 - analytics.flush() - self.assertEqual(analytics.default_client.timeout, 1.234) - - def setUp(self): - analytics.write_key = 'test-init' - analytics.default_client = None - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/segment/analytics/test/client.py b/segment/analytics/test/test_client.py similarity index 100% rename from segment/analytics/test/client.py rename to segment/analytics/test/test_client.py diff --git a/segment/analytics/test/consumer.py b/segment/analytics/test/test_consumer.py similarity index 100% rename from segment/analytics/test/consumer.py rename to segment/analytics/test/test_consumer.py diff --git a/segment/analytics/test/test_init.py b/segment/analytics/test/test_init.py new file mode 100644 index 00000000..98ad6aa3 --- /dev/null +++ b/segment/analytics/test/test_init.py @@ -0,0 +1,87 @@ +import unittest +import pkgutil +import logging +import sys +import segment.analytics as analytics +from segment.analytics.client import Client + + +def all_names(): + for _, modname, _ in pkgutil.iter_modules(__path__): + yield 'segment.analytics.test.' + modname + + +def all(): + logging.basicConfig(stream=sys.stderr) + return unittest.defaultTestLoader.loadTestsFromNames(all_names()) + + +class TestInit(unittest.TestCase): + def test_writeKey(self): + self.assertIsNone(analytics.default_client) + analytics.flush() + self.assertEqual(analytics.default_client.write_key, 'test-init') + + def test_debug(self): + self.assertIsNone(analytics.default_client) + analytics.debug = True + analytics.flush() + self.assertTrue(analytics.default_client.debug) + analytics.default_client = None + analytics.debug = False + analytics.flush() + self.assertFalse(analytics.default_client.debug) + analytics.default_client.log.setLevel(0) # reset log level after debug enable + + def test_gzip(self): + self.assertIsNone(analytics.default_client) + analytics.gzip = True + analytics.flush() + self.assertTrue(analytics.default_client.gzip) + analytics.default_client = None + analytics.gzip = False + analytics.flush() + self.assertFalse(analytics.default_client.gzip) + + def test_host(self): + self.assertIsNone(analytics.default_client) + analytics.host = 'http://test-host' + analytics.flush() + self.assertEqual(analytics.default_client.host, 'http://test-host') + analytics.host = None + analytics.default_client = None + + def test_max_queue_size(self): + self.assertIsNone(analytics.default_client) + analytics.max_queue_size = 1337 + analytics.flush() + self.assertEqual(analytics.default_client.queue.maxsize, 1337) + + def test_max_retries(self): + self.assertIsNone(analytics.default_client) + client = Client('testsecret', max_retries=42) + for consumer in client.consumers: + self.assertEqual(consumer.retries, 42) + + def test_sync_mode(self): + self.assertIsNone(analytics.default_client) + analytics.sync_mode = True + analytics.flush() + self.assertTrue(analytics.default_client.sync_mode) + analytics.default_client = None + analytics.sync_mode = False + analytics.flush() + self.assertFalse(analytics.default_client.sync_mode) + + def test_timeout(self): + self.assertIsNone(analytics.default_client) + analytics.timeout = 1.234 + analytics.flush() + self.assertEqual(analytics.default_client.timeout, 1.234) + + def setUp(self): + analytics.write_key = 'test-init' + analytics.default_client = None + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/segment/analytics/test/module.py b/segment/analytics/test/test_module.py similarity index 100% rename from segment/analytics/test/module.py rename to segment/analytics/test/test_module.py diff --git a/segment/analytics/test/oauth.py b/segment/analytics/test/test_oauth.py similarity index 100% rename from segment/analytics/test/oauth.py rename to segment/analytics/test/test_oauth.py diff --git a/segment/analytics/test/request.py b/segment/analytics/test/test_request.py similarity index 100% rename from segment/analytics/test/request.py rename to segment/analytics/test/test_request.py diff --git a/segment/analytics/test/utils.py b/segment/analytics/test/test_utils.py similarity index 100% rename from segment/analytics/test/utils.py rename to segment/analytics/test/test_utils.py From ceff3b7f738e6fd6bc7cc28141ab193d10dafd5a Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Thu, 16 May 2024 07:48:26 -0700 Subject: [PATCH 306/323] Adding automation of jira tickets from github issues (#460) --- .github/workflows/create_jira.yml | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/create_jira.yml diff --git a/.github/workflows/create_jira.yml b/.github/workflows/create_jira.yml new file mode 100644 index 00000000..8180ac0f --- /dev/null +++ b/.github/workflows/create_jira.yml @@ -0,0 +1,39 @@ +name: Create Jira Ticket + +on: + issues: + types: + - opened + +jobs: + create_jira: + name: Create Jira Ticket + runs-on: ubuntu-latest + environment: IssueTracker + steps: + - name: Checkout + uses: actions/checkout@master + - name: Login + uses: atlassian/gajira-login@master + env: + JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} + JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} + JIRA_API_TOKEN: ${{ secrets.JIRA_TOKEN }} + JIRA_EPIC_KEY: ${{ secrets.JIRA_EPIC_KEY }} + JIRA_PROJECT: ${{ secrets.JIRA_PROJECT }} + + - name: Create + id: create + uses: atlassian/gajira-create@master + with: + project: ${{ secrets.JIRA_PROJECT }} + issuetype: Bug + summary: | + [${{ github.event.repository.name }}] (${{ github.event.issue.number }}): ${{ github.event.issue.title }} + description: | + Github Link: ${{ github.event.issue.html_url }} + ${{ github.event.issue.body }} + fields: '{"parent": {"key": "${{ secrets.JIRA_EPIC_KEY }}"}}' + + - name: Log created issue + run: echo "Issue ${{ steps.create.outputs.issue }} was created" \ No newline at end of file From 6c0cce7b0b0c1b8a13765cca5d1aebae6b9f2e90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 May 2024 06:21:22 -0700 Subject: [PATCH 307/323] Bump cryptography from 41.0.3 to 42.0.7 (#458) Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 42.0.7. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...42.0.7) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cf0cbbee..884e1d0d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ backoff==2.2.1 -cryptography==41.0.3 +cryptography==42.0.7 flake8==3.7.9 mock==2.0.0 PyJWT==2.8.0 From 75838879c5b2859f6305bea0970c424198ee278a Mon Sep 17 00:00:00 2001 From: Pascal Corpet Date: Tue, 9 Jul 2024 16:43:00 +0200 Subject: [PATCH 308/323] Use now(tz=utc) instead of deprecated utcnow (#470) --- segment/analytics/client.py | 2 +- segment/analytics/request.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/segment/analytics/client.py b/segment/analytics/client.py index 1d4e35da..0f8015cd 100644 --- a/segment/analytics/client.py +++ b/segment/analytics/client.py @@ -267,7 +267,7 @@ def _enqueue(self, msg): """Push a new `msg` onto the queue, return `(success, msg)`""" timestamp = msg['timestamp'] if timestamp is None: - timestamp = datetime.utcnow().replace(tzinfo=tzutc()) + timestamp = datetime.now(tz=tzutc()) message_id = msg.get('messageId') if message_id is None: message_id = uuid4() diff --git a/segment/analytics/request.py b/segment/analytics/request.py index 273247ce..1f5a56ec 100644 --- a/segment/analytics/request.py +++ b/segment/analytics/request.py @@ -18,7 +18,7 @@ def post(write_key, host=None, gzip=False, timeout=15, proxies=None, oauth_manag log = logging.getLogger('segment') body = kwargs if not "sentAt" in body.keys(): - body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat() + body["sentAt"] = datetime.now(tz=tzutc()).isoformat() body["writeKey"] = write_key url = remove_trailing_slash(host or 'https://api.segment.io') + '/v1/batch' auth = None From b1854ccaf9cd62ca94e0aedafe3f2dbdc004c071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yannick=20P=C3=89ROUX?= Date: Wed, 18 Sep 2024 01:15:35 +0200 Subject: [PATCH 309/323] Update PyJWT & loosen dependency contraint (#477) * Update PyJWT to 2.9.0 * Loosen dependency constraint for PyJWT --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 884e1d0d..656f3b20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backoff==2.2.1 cryptography==42.0.7 flake8==3.7.9 mock==2.0.0 -PyJWT==2.8.0 +PyJWT==2.9.0 pylint==2.8.0 python-dateutil==2.8.2 requests==2.31.0 diff --git a/setup.py b/setup.py index 2cec9ff5..4bfdd978 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ "requests~=2.7", "backoff~=2.1", "python-dateutil~=2.2", - "PyJWT~=2.8.0" + "PyJWT~=2.8" ] tests_require = [ From e0452976aec9bc5535cb008e33c974e9bad0760a Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 18 Sep 2024 12:21:30 -0500 Subject: [PATCH 310/323] Clean up dependabot test failures (#482) * Clean up dependabot test failures * Remove python 3.7 test due to EOL --- .github/workflows/main.yml | 2 +- .github/workflows/tests.yml | 9 --------- requirements.txt | 8 ++++---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a17f8887..3fb68af8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,7 +14,7 @@ jobs: - name: Install Python 3 uses: actions/setup-python@v3 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 257b7077..792a2200 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,15 +19,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Run with setup-python 3.7 - uses: actions/setup-python@v5 - with: - python-version: '3.7' - - name: Setup required modules - run: python -m pip install -r requirements.txt - - name: Run tests - run: python -m unittest discover -s segment - - name: Run with setup-python 3.8 uses: actions/setup-python@v5 with: diff --git a/requirements.txt b/requirements.txt index 656f3b20..18c9d174 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ backoff==2.2.1 -cryptography==42.0.7 -flake8==3.7.9 +cryptography==43.0.1 +flake8==7.1.1 mock==2.0.0 +pylint==3.2.7 PyJWT==2.9.0 -pylint==2.8.0 python-dateutil==2.8.2 -requests==2.31.0 +requests==2.32.3 From b133fff466fa38c8212ce35a0356ba45710af621 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 4 Oct 2024 15:25:03 -0400 Subject: [PATCH 311/323] Update oauth to use a delay instead of timestamp * Update oauth test * Pulling in changes and fixing test --- segment/analytics/oauth_manager.py | 8 ++++---- segment/analytics/test/test_oauth.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/segment/analytics/oauth_manager.py b/segment/analytics/oauth_manager.py index 453a23a0..61a54ea5 100644 --- a/segment/analytics/oauth_manager.py +++ b/segment/analytics/oauth_manager.py @@ -152,13 +152,13 @@ def _poller_loop(self): elif response.status_code == 429: self.retry_count += 1 - rate_limit_reset_timestamp = None + rate_limit_reset_time = None try: - rate_limit_reset_timestamp = int(response.headers.get("X-RateLimit-Reset")) + rate_limit_reset_time = int(response.headers.get("X-RateLimit-Reset")) except Exception as e: self.log.error("OAuth rate limit response did not have a valid rest time: {} | {}".format(response, e)) - if rate_limit_reset_timestamp: - refresh_timer_ms = rate_limit_reset_timestamp - time.time() * 1000 + if rate_limit_reset_time: + refresh_timer_ms = rate_limit_reset_time * 1000 else: refresh_timer_ms = 5 * 1000 diff --git a/segment/analytics/test/test_oauth.py b/segment/analytics/test/test_oauth.py index 259342bc..a2269507 100644 --- a/segment/analytics/test/test_oauth.py +++ b/segment/analytics/test/test_oauth.py @@ -55,7 +55,7 @@ def json(self): elif kwargs['url'] == 'http://127.0.0.1:400/token': return MockResponse({"reason": "test_reason", "json_data" : {"error":"unrecoverable", "error_description":"nah"}}, 400) elif kwargs['url'] == 'http://127.0.0.1:429/token': - return MockResponse({"reason": "test_reason", "headers" : {"X-RateLimit-Reset": time.time()*1000 + 2000}}, 429) + return MockResponse({"reason": "test_reason", "headers" : {"X-RateLimit-Reset": 234}}, 429) elif kwargs['url'] == 'http://127.0.0.1:500/token': return MockResponse({"reason": "test_reason", "json_data" : {"error":"recoverable", "error_description":"nah"}}, 500) elif kwargs['url'] == 'http://127.0.0.1:501/token': @@ -106,7 +106,7 @@ def test_oauth_fail_with_retries(self, mock_post): def test_oauth_rate_limit_delay(self, mock_sleep, mock_post): manager = segment.analytics.oauth_manager.OauthManager("id", privatekey, "keyid", "http://127.0.0.1:429") manager._poller_loop() - self.assertTrue(mock_sleep.call_args[0][0] > 1.9 and mock_sleep.call_args[0][0] <= 2.0) + mock_sleep.assert_called_with(234) class TestOauthIntegration(unittest.TestCase): def fail(self, e, batch=[]): @@ -152,4 +152,4 @@ def test_oauth_integration_fail_bad_key(self, mock_post): self.assertTrue(self.failed) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From ec1c05ae566d7a32f0b8b5f58752a3fe79319087 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 7 Oct 2024 12:18:15 -0400 Subject: [PATCH 312/323] Release 2.3.3. (#490) --- HISTORY.md | 3 +++ segment/analytics/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index fa7009c4..41f21e60 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +# 2.3.3 / 2024-10-07 +- Update time handling and OAuth + # 2.3.2 / 2024-02-15 - Updating version to create a release wheel without the outdated /analytics files diff --git a/segment/analytics/version.py b/segment/analytics/version.py index a5ac6164..47cb28fc 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.3.2' +VERSION = '2.3.3' From 3b545facc844a23975f534e1af04b655cf24267f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:06:22 -0400 Subject: [PATCH 313/323] Bump cryptography from 43.0.1 to 43.0.3 (#492) Bumps [cryptography](https://github.com/pyca/cryptography) from 43.0.1 to 43.0.3. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/43.0.1...43.0.3) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 18c9d174..3ff2ac90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ backoff==2.2.1 -cryptography==43.0.1 +cryptography==43.0.3 flake8==7.1.1 mock==2.0.0 pylint==3.2.7 From 1eae85f5506ec64adb329080d7e896c68ccc0599 Mon Sep 17 00:00:00 2001 From: "Shane L. Duvall" Date: Wed, 11 Dec 2024 12:25:16 -0600 Subject: [PATCH 314/323] require python >= 3.9 (#497) * Update tests to require python >= 3.9 * PyJWT version update * setup update * Update workflow to meet minimum requirements --- .github/workflows/main.yml | 12 +++--------- .github/workflows/tests.yml | 9 --------- setup.py | 9 +++------ 3 files changed, 6 insertions(+), 24 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3fb68af8..65d6119b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,14 +14,14 @@ jobs: - name: Install Python 3 uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install python-dateutil backoff monotonic pip install --user . - sudo pip install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil aiohttp==3.9.1 + sudo pip install pylint==3.3.1 flake8 mock==3.0.5 python-dateutil aiohttp==3.9.1 - name: Run tests run: python -m unittest discover -s segment @@ -41,17 +41,11 @@ jobs: # runs-on: ubuntu-latest # strategy: # matrix: -# python: ['3.7', '3.8', '3.9', '3.10', '3.11'] +# python: ['3.9', '3.10', '3.11'] # coverage: [false] # experimental: [false] # include: # # Run code coverage. -# - python: '3.7' -# coverage: true -# experimental: false -# - python: '3.8' -# coverage: true -# experimental: false # - python: '3.9' # coverage: true # experimental: false diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 792a2200..97b401c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,15 +19,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Run with setup-python 3.8 - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Setup required modules - run: python -m pip install -r requirements.txt - - name: Run tests - run: python -m unittest discover -s segment - - name: Run with setup-python 3.9 uses: actions/setup-python@v5 with: diff --git a/setup.py b/setup.py index 4bfdd978..0443d7b2 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,12 @@ "requests~=2.7", "backoff~=2.1", "python-dateutil~=2.2", - "PyJWT~=2.8" + "PyJWT~=2.10.1" ] tests_require = [ "mock==2.0.0", - "pylint==2.8.0", + "pylint==3.3.1", "flake8==3.7.9", ] @@ -42,7 +42,7 @@ maintainer_email='friends@segment.com', test_suite='segment.analytics.test.all', packages=['segment.analytics', 'segment.analytics.test'], - python_requires='>=3.6.0', + python_requires='>=3.9.0', license='MIT License', install_requires=install_requires, extras_require={ @@ -56,9 +56,6 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From a300a1aabfc93ab45ab8c23640c659d01fde7f34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:39:05 -0500 Subject: [PATCH 315/323] Bump pyjwt from 2.9.0 to 2.10.1 (#496) Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.9.0 to 2.10.1. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.9.0...2.10.1) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michael Grosse Huelsewiesche --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3ff2ac90..712f46e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,6 @@ cryptography==43.0.3 flake8==7.1.1 mock==2.0.0 pylint==3.2.7 -PyJWT==2.9.0 +PyJWT==2.10.1 python-dateutil==2.8.2 requests==2.32.3 From 603632f3aa57965c694209c3e8470d935e443b60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:45:04 -0500 Subject: [PATCH 316/323] Bump pylint from 3.2.7 to 3.3.2 (#498) Bumps [pylint](https://github.com/pylint-dev/pylint) from 3.2.7 to 3.3.2. - [Release notes](https://github.com/pylint-dev/pylint/releases) - [Commits](https://github.com/pylint-dev/pylint/compare/v3.2.7...v3.3.2) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michael Grosse Huelsewiesche --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 712f46e7..602f7a40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backoff==2.2.1 cryptography==43.0.3 flake8==7.1.1 mock==2.0.0 -pylint==3.2.7 +pylint==3.3.2 PyJWT==2.10.1 python-dateutil==2.8.2 requests==2.32.3 From b8d7370d6d66ef002798d205f4a8b411dba6bba4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:16:46 -0500 Subject: [PATCH 317/323] Bump cryptography from 43.0.3 to 44.0.0 (#495) Bumps [cryptography](https://github.com/pyca/cryptography) from 43.0.3 to 44.0.0. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/43.0.3...44.0.0) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 602f7a40..02321eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ backoff==2.2.1 -cryptography==43.0.3 +cryptography==44.0.0 flake8==7.1.1 mock==2.0.0 pylint==3.3.2 From c8e4c56e18e981df6c1d5eb67211e67f2108c391 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:22:03 -0500 Subject: [PATCH 318/323] Bump pylint from 3.3.2 to 3.3.3 (#499) Bumps [pylint](https://github.com/pylint-dev/pylint) from 3.3.2 to 3.3.3. - [Release notes](https://github.com/pylint-dev/pylint/releases) - [Commits](https://github.com/pylint-dev/pylint/compare/v3.3.2...v3.3.3) --- updated-dependencies: - dependency-name: pylint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 02321eb1..308a4e87 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ backoff==2.2.1 cryptography==44.0.0 flake8==7.1.1 mock==2.0.0 -pylint==3.3.2 +pylint==3.3.3 PyJWT==2.10.1 python-dateutil==2.8.2 requests==2.32.3 From a2db8d8d81582156c59bbd026e2a26390bb33708 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Fri, 21 Feb 2025 14:16:29 -0500 Subject: [PATCH 319/323] Proxy datatype was wrong in test (#504) --- segment/analytics/test/test_client.py | 2 +- segment/analytics/test/test_consumer.py | 2 +- segment/analytics/test/test_request.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/segment/analytics/test/test_client.py b/segment/analytics/test/test_client.py index bffdb1e5..87dc63a9 100644 --- a/segment/analytics/test/test_client.py +++ b/segment/analytics/test/test_client.py @@ -344,6 +344,6 @@ def test_default_timeout_15(self): self.assertEqual(consumer.timeout, 15) def test_proxies(self): - client = Client('testsecret', proxies='203.243.63.16:80') + client = Client('testsecret', proxies={'http':'203.243.63.16:80','https':'203.243.63.16:80'}) success, msg = client.identify('userId', {'trait': 'value'}) self.assertTrue(success) diff --git a/segment/analytics/test/test_consumer.py b/segment/analytics/test/test_consumer.py index 4a2a1e24..f60f20ec 100644 --- a/segment/analytics/test/test_consumer.py +++ b/segment/analytics/test/test_consumer.py @@ -200,7 +200,7 @@ def mock_post_fn(_, data, **kwargs): @classmethod def test_proxies(cls): - consumer = Consumer(None, 'testsecret', proxies='203.243.63.16:80') + consumer = Consumer(None, 'testsecret', proxies={'http':'203.243.63.16:80','https':'203.243.63.16:80'}) track = { 'type': 'track', 'event': 'python event', diff --git a/segment/analytics/test/test_request.py b/segment/analytics/test/test_request.py index 3f40c497..9a496667 100644 --- a/segment/analytics/test/test_request.py +++ b/segment/analytics/test/test_request.py @@ -57,6 +57,6 @@ def test_proxies(self): 'userId': 'userId', 'event': 'python event', 'type': 'track', - 'proxies': '203.243.63.16:80' + 'proxies': {'http':'203.243.63.16:80','https':'203.243.63.16:80'} }]) self.assertEqual(res.status_code, 200) From 3cb32c9d982aacfed4d87ffe5ec195ad1ce1f1fa Mon Sep 17 00:00:00 2001 From: Aleksandr <32888481+aldev12@users.noreply.github.com> Date: Sat, 22 Feb 2025 02:04:19 +0300 Subject: [PATCH 320/323] fix proxies (#488) --- segment/analytics/request.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/segment/analytics/request.py b/segment/analytics/request.py index 1f5a56ec..4118e2cf 100644 --- a/segment/analytics/request.py +++ b/segment/analytics/request.py @@ -45,14 +45,14 @@ def post(write_key, host=None, gzip=False, timeout=15, proxies=None, oauth_manag kwargs = { "data": data, "headers": headers, - "timeout": 15, + "timeout": timeout, } if proxies: kwargs['proxies'] = proxies - res = None + try: - res = _session.post(url, data=data, headers=headers, timeout=timeout) + res = _session.post(url, **kwargs) except Exception as e: log.error(e) raise e From 631ebaf1e9b8524e97ae60c83cc77ffacdf82a9a Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 24 Feb 2025 11:19:15 -0500 Subject: [PATCH 321/323] Fix proxy tests for real this time, and actually test the proxy values make it to post (#505) --- segment/analytics/test/test_client.py | 19 +++++++++++++++--- segment/analytics/test/test_consumer.py | 17 ++++++++++++++-- segment/analytics/test/test_request.py | 26 ++++++++++++++++++------- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/segment/analytics/test/test_client.py b/segment/analytics/test/test_client.py index 87dc63a9..eb68400c 100644 --- a/segment/analytics/test/test_client.py +++ b/segment/analytics/test/test_client.py @@ -344,6 +344,19 @@ def test_default_timeout_15(self): self.assertEqual(consumer.timeout, 15) def test_proxies(self): - client = Client('testsecret', proxies={'http':'203.243.63.16:80','https':'203.243.63.16:80'}) - success, msg = client.identify('userId', {'trait': 'value'}) - self.assertTrue(success) + proxies={'http':'203.243.63.16:80','https':'203.243.63.16:80'} + client = Client('testsecret', proxies=proxies) + def mock_post_fn(*args, **kwargs): + res = mock.Mock() + res.status_code = 200 + res.json.return_value = {'code': 'success', 'message': 'success'} + return res + + with mock.patch('segment.analytics.request._session.post', side_effect=mock_post_fn) as mock_post: + success, msg = client.identify('userId', {'trait': 'value'}) + client.flush() + self.assertTrue(success) + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + self.assertIn('proxies', kwargs) + self.assertEqual(kwargs['proxies'], proxies) \ No newline at end of file diff --git a/segment/analytics/test/test_consumer.py b/segment/analytics/test/test_consumer.py index f60f20ec..83717266 100644 --- a/segment/analytics/test/test_consumer.py +++ b/segment/analytics/test/test_consumer.py @@ -200,10 +200,23 @@ def mock_post_fn(_, data, **kwargs): @classmethod def test_proxies(cls): - consumer = Consumer(None, 'testsecret', proxies={'http':'203.243.63.16:80','https':'203.243.63.16:80'}) + proxies = {'http': '203.243.63.16:80', 'https': '203.243.63.16:80'} + consumer = Consumer(None, 'testsecret', proxies=proxies) track = { 'type': 'track', 'event': 'python event', 'userId': 'userId' } - consumer.request([track]) + + def mock_post_fn(*args, **kwargs): + res = mock.Mock() + res.status_code = 200 + res.json.return_value = {'code': 'success', 'message': 'success'} + return res + + with mock.patch('segment.analytics.request._session.post', side_effect=mock_post_fn) as mock_post: + consumer.request([track]) + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + cls().assertIn('proxies', kwargs) + cls().assertEqual(kwargs['proxies'], proxies) diff --git a/segment/analytics/test/test_request.py b/segment/analytics/test/test_request.py index 9a496667..5ffca009 100644 --- a/segment/analytics/test/test_request.py +++ b/segment/analytics/test/test_request.py @@ -2,6 +2,7 @@ import unittest import json import requests +from unittest import mock from segment.analytics.request import post, DatetimeSerializer @@ -53,10 +54,21 @@ def test_should_timeout(self): }], timeout=0.0001) def test_proxies(self): - res = post('testsecret', batch=[{ - 'userId': 'userId', - 'event': 'python event', - 'type': 'track', - 'proxies': {'http':'203.243.63.16:80','https':'203.243.63.16:80'} - }]) - self.assertEqual(res.status_code, 200) + proxies = {'http': '203.243.63.16:80', 'https': '203.243.63.16:80'} + def mock_post_fn(*args, **kwargs): + res = mock.Mock() + res.status_code = 200 + res.json.return_value = {'code': 'success', 'message': 'success'} + return res + + with mock.patch('segment.analytics.request._session.post', side_effect=mock_post_fn) as mock_post: + res = post('testsecret', proxies= proxies, batch=[{ + 'userId': 'userId', + 'event': 'python event', + 'type': 'track' + }]) + self.assertEqual(res.status_code, 200) + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + self.assertIn('proxies', kwargs) + self.assertEqual(kwargs['proxies'], proxies) From cb3e7ff0ea7c9c0e25158ab5badaa2267c4681c8 Mon Sep 17 00:00:00 2001 From: Michael Grosse Huelsewiesche Date: Mon, 24 Feb 2025 11:37:01 -0500 Subject: [PATCH 322/323] Release 2.3.4. (#507) --- HISTORY.md | 3 +++ segment/analytics/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 41f21e60..c730b6d1 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +# 2.3.4 / 2025-2-24 +- Fix for proxy values not being used + # 2.3.3 / 2024-10-07 - Update time handling and OAuth diff --git a/segment/analytics/version.py b/segment/analytics/version.py index 47cb28fc..f82ed75c 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.3.3' +VERSION = '2.3.4' From bc40fba94fbe280105812e4f9b94a25422ff4920 Mon Sep 17 00:00:00 2001 From: Neelkanth Kaushik Date: Tue, 2 Dec 2025 23:17:42 +0530 Subject: [PATCH 323/323] Fixes for Github Issue #516, #515 and #493 (#517) * Fixed Github Issue #516 * Fixed Github Issue 515 * Minor changes and conditional fix --- HISTORY.md | 9 +++++++++ segment/analytics/consumer.py | 24 +++++++++++++++++++----- segment/analytics/request.py | 3 +-- segment/analytics/version.py | 2 +- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index c730b6d1..613a38c3 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,12 @@ +# 2.3.5 / 2025-11-18 +- Fix for Github Issue #516 +- Fix for Github Issue #515 +- Fix for Github Issue #493 +- Updated the version to 2.3.5 +- Enhanced retry logic in segment/analytics/consumer.py by adding detailed debug logs for each retry attempt and logging an error when all retries are exhausted. +- Moved the return success out of the finally block in segment/analytics/consumer.py at Line 84. +- Removed redundant error logging in segment/analytics/request.py to avoid duplicate logs when exceptions are raised. + # 2.3.4 / 2025-2-24 - Fix for proxy values not being used diff --git a/segment/analytics/consumer.py b/segment/analytics/consumer.py index 1c471573..157e3c93 100644 --- a/segment/analytics/consumer.py +++ b/segment/analytics/consumer.py @@ -14,9 +14,11 @@ # lower to leave space for extra data that will be added later, eg. "sentAt". BATCH_SIZE_LIMIT = 475000 + class FatalError(Exception): def __init__(self, message): self.message = message + def __str__(self): msg = "[Segment] {0})" return msg.format(self.message) @@ -81,7 +83,7 @@ def upload(self): # mark items as acknowledged from queue for _ in batch: self.queue.task_done() - return success + return success def next(self): """Return the next batch of items to upload.""" @@ -132,14 +134,26 @@ def fatal_exception(exc): # retry on all other errors (eg. network) return False + attempt_count = 0 + @backoff.on_exception( backoff.expo, Exception, max_tries=self.retries + 1, - giveup=fatal_exception) + giveup=fatal_exception, + on_backoff=lambda details: self.log.debug( + f"Retry attempt {details['tries']}/{self.retries + 1} after {details['elapsed']:.2f}s" + )) def send_request(): - post(self.write_key, self.host, gzip=self.gzip, - timeout=self.timeout, batch=batch, proxies=self.proxies, - oauth_manager=self.oauth_manager) + nonlocal attempt_count + attempt_count += 1 + try: + return post(self.write_key, self.host, gzip=self.gzip, + timeout=self.timeout, batch=batch, proxies=self.proxies, + oauth_manager=self.oauth_manager) + except Exception as e: + if attempt_count >= self.retries + 1: + self.log.error(f"All {self.retries} retries exhausted. Final error: {e}") + raise send_request() diff --git a/segment/analytics/request.py b/segment/analytics/request.py index 4118e2cf..ab92b807 100644 --- a/segment/analytics/request.py +++ b/segment/analytics/request.py @@ -54,9 +54,8 @@ def post(write_key, host=None, gzip=False, timeout=15, proxies=None, oauth_manag try: res = _session.post(url, **kwargs) except Exception as e: - log.error(e) raise e - + if res.status_code == 200: log.debug('data uploaded successfully') return res diff --git a/segment/analytics/version.py b/segment/analytics/version.py index f82ed75c..b4ab188b 100644 --- a/segment/analytics/version.py +++ b/segment/analytics/version.py @@ -1 +1 @@ -VERSION = '2.3.4' +VERSION = '2.3.5'