diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 391359e..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,103 +0,0 @@ -version: 2.1 -tagged_build_filters: &tagged_build_filters - branches: - ignore: /.*/ - tags: - only: /[0-9]+\.[0-9]+\.[0-9]+/ -test_build_filters: &test_build_filters - branches: - only: /.*/ - tags: - ignore: /[0-9]+\.[0-9]+\.[0-9]+/ -jobs: - test: - docker: - - image: circleci/python:3 - steps: - - checkout - - run: poetry install - - run: poetry run lint - - run: poetry run test - build: - docker: - - image: circleci/python:3 - steps: - - checkout - - run: - name: Check Tagged Push - command: | - PKG_VERSION=$(cat logdna/VERSION) - if [[ "${CIRCLE_TAG}" != "${PKG_VERSION}" ]]; then - echo "There is mismatch:" - echo " TAG_VERSION: ${CIRCLE_TAG}" - echo " PKG_VERSION: ${PKG_VERSION}" - exit 1 - fi - - run: python setup.py sdist - - persist_to_workspace: - root: . - paths: - - ./dist/logdna-*.tar.gz - release: - docker: - - image: circleci/golang:1.12 - steps: - - attach_workspace: - at: . - - run: go get -u github.com/tcnksm/ghr - - run: - name: Create a Release - command: | - ghr \ - -n "LogDNA Python Logger v${CIRCLE_TAG}" \ - -t ${GITHUB_TOKEN} \ - -u ${CIRCLE_PROJECT_USERNAME} \ - -r ${CIRCLE_PROJECT_REPONAME} \ - -draft ${CIRCLE_TAG} ./dist/ - - persist_to_workspace: - root: . - paths: - - ./dist/logdna-*.tar.gz - approve: - machine: true - steps: - - attach_workspace: - at: . - - persist_to_workspace: - root: . - paths: - - ./dist/logdna-*.tar.gz - publish: - docker: - - image: circleci/python:3 - steps: - - attach_workspace: - at: . - - run: sudo pip install twine - - run: twine upload dist/logdna-*.tar.gz -workflows: - update: - jobs: - - test: - filters: *test_build_filters - - build: - requires: - - test - filters: *tagged_build_filters - - release: - requires: - - build - filters: *tagged_build_filters - - approve: - type: approval - requires: - - release - filters: *tagged_build_filters - - publish: - requires: - - approve - filters: *tagged_build_filters -test: - jobs: - - test: - filters: *test_build_filters diff --git a/.config.mk b/.config.mk new file mode 100644 index 0000000..65038c5 --- /dev/null +++ b/.config.mk @@ -0,0 +1,9 @@ +# Below is an example of pulling the current version of a node app. + +#VERSION is being deprecated by APP_VERSION - no changes necessary - see Makefile +#APP_VERSION=$(shell awk '/version/ {gsub(/[",]/,""); print $$2}' package.json) + +GIT_AUTHOR_NAME ?= $(shell git config --get user.name) +GIT_AUTHOR_EMAIL ?= $(shell git config --get user.email) +GIT_COMMITTER_NAME ?= $(GIT_AUTHOR_NAME) +GIT_COMMITTER_EMAIL ?= $(GIT_AUTHOR_EMAIL) diff --git a/.gitignore b/.gitignore index 5d0eed0..dc6e372 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -**/*.pyc +*.pyc **/*cache* build/ develop-eggs @@ -11,3 +11,6 @@ bin coverage/ .coverage Session.vim +.cache +pypoetry/ +pip/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b52389a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,127 @@ +# Changelog + + + +## v1.18.12 (2023-07-27) +### Fix +* Don't overwrite base logging Handler class lock var ([#108](https://github.com/logdna/python/issues/108)) ([`5b04d72`](https://github.com/logdna/python/commit/5b04d72b42686d926fb5e73dadb2d7a1ba16d9c3)) + +## v1.18.11 (2023-07-26) +### Fix +* Remove Thread/event to fix Django regression ([#107](https://github.com/logdna/python/issues/107)) ([`0337a09`](https://github.com/logdna/python/commit/0337a09433953227dabb0b65da8583e8f9273986)) + +## v1.18.10 (2023-07-26) +### Fix +* Utilize apikey header for auth to make compatible with pipelines ([#104](https://github.com/logdna/python/issues/104)) ([`5394e97`](https://github.com/logdna/python/commit/5394e9714779878cd415a4566cd44d9183e150b9)) + +## v1.18.9 (2023-07-21) +### Fix +* Make flush thread a daemon thread to prevent shutdown hang ([#102](https://github.com/logdna/python/issues/102)) ([`17a69b0`](https://github.com/logdna/python/commit/17a69b044de43a7a9d7d3e6eb65a0c60f1fa23f0)) + +## v1.18.8 (2023-07-18) +### Fix +* Bump semver ([#101](https://github.com/logdna/python/issues/101)) ([`913e5f4`](https://github.com/logdna/python/commit/913e5f4f35f2920a6b2162022b93495b4774654c)) + +## v1.18.7 (2023-05-05) +### Fix +* Gate buils from non maintainers ([`97803b5`](https://github.com/logdna/python/commit/97803b55a102539a75b0763891c1d27757460067)) + +## v1.18.6 (2023-01-27) +### Fix +* Added retries for http 429 504 ([#93](https://github.com/logdna/python/issues/93)) ([`712d81d`](https://github.com/logdna/python/commit/712d81d4ab2bfbf95d65898fb365a5bcb1396199)) + +## v1.18.5 (2023-01-27) +### Fix +* **chore:** Upgraded pytest to resolve security vulnerability in py <= 1.11.0 ([#92](https://github.com/logdna/python/issues/92)) ([`905b06a`](https://github.com/logdna/python/commit/905b06a648e19896087b0c51ba1e055212727560)) + +## v1.18.4 (2023-01-06) +### Fix +* Dependabot -> Vulnerabilities -> cryptography >= 37.0.0 < 38.0.3 ([#91](https://github.com/logdna/python/issues/91)) ([`7ccde50`](https://github.com/logdna/python/commit/7ccde50ba50c0abbec1a7efe4dd665e8b35511c0)) + +## v1.18.3 (2022-12-07) +### Fix +* Add documentation for log_error_response option ([#88](https://github.com/logdna/python/issues/88)) ([`d5bf85c`](https://github.com/logdna/python/commit/d5bf85ca26579e0186e1abdbcd1b6c44c60c9eca)) + +## v1.18.2 (2022-05-10) +### Fix +* Requests error handling ([#82](https://github.com/logdna/python/issues/82)) ([`e412859`](https://github.com/logdna/python/commit/e4128592aa9c2301c6467115148f2f23f88da9d3)) + +## v1.18.1 (2021-11-18) +### Fix +* **threading:** Account for secondary buffer flush and deadlock ([`d00b952`](https://github.com/logdna/python/commit/d00b9529d116ddf9ba454462d4d804dc54423c83)) + +## v1.18.0 (2021-07-26) +### Fix +* **opts:** Repair logging options for each call ([`e326e4c`](https://github.com/logdna/python/commit/e326e4c2461b808b5d3a885b37555f8e610615e4)) + +## v1.17.0 (2021-07-15) +### Feature +* **meta:** Enable adding custom meta fields ([`f250237`](https://github.com/logdna/python/commit/f250237dbde932e99ab199023516f66a248c5e80)) +* **threadWorkerPools:** Introduce extra threads ([`9bfe479`](https://github.com/logdna/python/commit/9bfe479132acb0aa8e5784f2aa31298606e49789)) + +### Documentation +* Update the README as requested ([`5dd754f`](https://github.com/logdna/python/commit/5dd754f177675eb25cd5d5449bd3bcc8286f8739)) + +## v1.16.0 (2021-04-15) + + +## v1.15.0 (2021-04-15) + + +## v1.14.0 (2021-04-15) + + +## v1.13.0 (2021-04-15) + + +## v1.12.0 (2021-04-15) + + +## v1.11.0 (2021-04-15) + + +## v1.10.0 (2021-04-15) + + +## v1.9.0 (2021-04-15) + + +## v1.8.0 (2021-04-15) + + +## v1.7.0 (2021-04-07) + + +## v1.6.0 (2021-04-07) +### Feature +* **ci:** Enable releases through semantic release ([`9e4b1c0`](https://github.com/logdna/python/commit/9e4b1c0a43bc0941ba4fb336ea12a3f497622ee6)) +* **ci:** Include jenkins setup ([`1c5be37`](https://github.com/logdna/python/commit/1c5be37ef32776f2d0ba2b68b67cebb58b1f0177)) +* **tasks:** Convert run scripts to tasks ([`f5a8b18`](https://github.com/logdna/python/commit/f5a8b182941a0c514594cbaa7a3ac5952173d5a8)) +* **lint:** Setup linting + test harness ([`0a7763a`](https://github.com/logdna/python/commit/0a7763a2befbf598b59d7ab595b19e22b173fda5)) +* **tags:** Allow tags to be configured ([`6adc21e`](https://github.com/logdna/python/commit/6adc21e872fa521be1aaf08309f7f3d0ba3dc5c5)) +* **meta:** Python log info in meta object ([`cf9b505`](https://github.com/logdna/python/commit/cf9b505734df12918a665a8a8c74d4fd74e5bc47)) +* **handlers:** Make available via config file ([`39c5dec`](https://github.com/logdna/python/commit/39c5decd98e8d4feb6c1bbfa487faf35396c8b12)) + +### Fix +* **ci:** Correct invalid environment variable ([`fce4d59`](https://github.com/logdna/python/commit/fce4d5995b31c426f2b66992b57f129f22f9f18f)) + +### Documentation +* Add @matthiasfru as a contributor ([`3727bc3`](https://github.com/logdna/python/commit/3727bc3386d3dd6465e0b0d15675a091a3743c24)) +* Add @rudyryk as a contributor ([`9ab0e39`](https://github.com/logdna/python/commit/9ab0e3932180c41bff1cd0944c24fb8e208f4391)) +* Add @kurtiss as a contributor ([`1476085`](https://github.com/logdna/python/commit/14760857649b56207240bae907386a33f7f1666b)) +* Update @inkrement as a contributor ([`6a2cede`](https://github.com/logdna/python/commit/6a2cedef6695e18a981a3605176c9cffdd159827)) +* Add @inkrement as a contributor ([`7ba6992`](https://github.com/logdna/python/commit/7ba6992287f98c478da72f5982a0869b5d835df7)) +* Add @btashton as a contributor ([`c1e69bf`](https://github.com/logdna/python/commit/c1e69bfc965e1e1e5717ec7af3271dc0bf6b502d)) +* Add @SpainTrain as a contributor ([`7b9f04b`](https://github.com/logdna/python/commit/7b9f04b86a1e813bccea29ea8db1554808ea48b6)) +* Add @sataloger as a contributor ([`04081f0`](https://github.com/logdna/python/commit/04081f039d6136a66b259f7245ad7845ef1c080a)) +* Add @jdemaeyer as a contributor ([`69e8f7c`](https://github.com/logdna/python/commit/69e8f7cc0782a778d0da230ff1a8feb5f8cee352)) +* Add @dchai76 as a contributor ([`42e9c6b`](https://github.com/logdna/python/commit/42e9c6bddc2bce29714072bba37d85d9a734eac2)) +* Add @danmaas as a contributor ([`23bbab4`](https://github.com/logdna/python/commit/23bbab48cdef4aaef6813459dbdb23dbd9c60374)) +* Add @LYHuang as a contributor ([`3cc9e23`](https://github.com/logdna/python/commit/3cc9e232d0b12b9f0315efb1d618426b486a2604)) +* Add @baronomasia as a contributor ([`8971bd7`](https://github.com/logdna/python/commit/8971bd713d74cd9386c60a50cc75a7fc3591f544)) +* Add @utek as a contributor ([`ea495b4`](https://github.com/logdna/python/commit/ea495b479cc0549ff68a1e816c3f7a174c1c138c)) +* Add @esatterwhite as a contributor ([`0a334bd`](https://github.com/logdna/python/commit/0a334bdb5690049634c64eb0f9c6c1026b9ab001)) +* Add @mikehu as a contributor ([`e7aa7cb`](https://github.com/logdna/python/commit/e7aa7cb2624313c065f7bbc8a129c1a4841f9ec2)) +* Add @vilyapilya as a contributor ([`e3191f5`](https://github.com/logdna/python/commit/e3191f577fb7ddf7590ca1e1bf239f66d2f30fd0)) +* Add @smusali as a contributor ([`4d5022f`](https://github.com/logdna/python/commit/4d5022f93948cca239ebc104e34034c350956f65)) +* Add @respectus as a contributor ([`85a543c`](https://github.com/logdna/python/commit/85a543c9dc27a3c6064e790be0ba1c475187c38d)) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..db887bb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM condaforge/miniforge3:4.12.0-0 + +RUN conda install -y gcc pip poetry=1.1.7 git +RUN mkdir /workdir && chmod 777 /workdir +RUN git config --global --add safe.directory /workdir +WORKDIR /workdir + diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..a516452 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,102 @@ +library 'magic-butler-catalogue' +def PROJECT_NAME = 'logdna-python' +def TRIGGER_PATTERN = ".*@logdnabot.*" +def DEFAULT_BRANCH = 'master' +def CURRENT_BRANCH = [env.CHANGE_BRANCH, env.BRANCH_NAME]?.find{branch -> branch != null} + +pipeline { + agent { + node { + label 'ec2-fleet' + customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}" + } + } + + options { + timestamps() + ansiColor 'xterm' + } + + triggers { + issueCommentTrigger(TRIGGER_PATTERN) + } + + stages { + stage('Validate PR Source') { + when { + expression { env.CHANGE_FORK } + not { + triggeredBy 'issueCommentCause' + } + } + steps { + error("A maintainer needs to approve this PR for CI by commenting") + } + } + stage('Test') { + + steps { + sh 'make install lint test' + } + + post { + always { + junit 'coverage/test.xml' + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'coverage', + reportFiles: 'index.html', + reportName: "coverage-${BUILD_TAG}" + ] + } + } + } + + stage('Release') { + + stages { + stage('dry run') { + when { + not { + branch "${DEFAULT_BRANCH}" + } + } + + environment { + GH_TOKEN = credentials('github-api-token') + PYPI_TOKEN = credentials('pypi-token') + JENKINS_URL = "${JENKINS_URL}" + BRANCH_NAME = "${DEFAULT_BRANCH}" + GIT_BRANCH = "${DEFAULT_BRANCH}" + CHANGE_ID = '' + } + + steps { + sh "make release-dry" + } + } + + stage('publish') { + + environment { + GH_TOKEN = credentials('github-api-token') + PYPI_TOKEN = credentials('pypi-token') + JENKINS_URL = "${JENKINS_URL}" + } + + when { + branch "${DEFAULT_BRANCH}" + not { + changelog '\\[skip ci\\]' + } + } + steps { + sh 'make release' + } + } + } + } + } +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a7f8886 --- /dev/null +++ b/Makefile @@ -0,0 +1,101 @@ +# Makefile Version 2021032403 +# +# Source in repository specific environment variables +include .config.mk + +# Define commands via docker +DOCKER = docker +DOCKER_RUN := $(DOCKER) run --rm -i +WORKDIR :=/workdir +DOCKER_COMMAND := $(DOCKER_RUN) -u "$(shell id -u)":"$(shell id -g)" -v $(PWD):$(WORKDIR):Z -w $(WORKDIR) \ + -e XDG_CONFIG_HOME=$(WORKDIR) \ + -e XDG_CACHE_HOME=$(WORKDIR) \ + -e POETRY_CACHE_DIR=$(WORKDIR)/.cache \ + -e POETRY_VIRTUALENV_IN_PROJECT=true \ + -e PYPI_TOKEN \ + -e GH_TOKEN \ + -e JENKINS_URL \ + -e BRANCH_NAME \ + -e CHANGE_ID \ + -e GIT_AUTHOR_NAME \ + -e GIT_AUTHOR_EMAIL \ + -e GIT_COMMITTER_NAME \ + -e GIT_COMMITTER_EMAIL \ + logdna-poetry:local + + +POETRY_COMMAND := $(DOCKER_COMMAND) poetry + +# Exports the variables for shell use +export + +# build image +.PHONY:build-image +build-image: + DOCKER_BUILDKIT=1 $(DOCKER) build -t logdna-poetry:local . + +# This helper function makes debugging much easier. +.PHONY:debug-% +debug-%: ## Debug a variable by calling `make debug-VARIABLE` + @echo $(*) = $($(*)) + +.PHONY:help +.SILENT:help +help: ## Show this help, includes list of all actions. + @awk 'BEGIN {FS = ":.*?## "}; /^.+: .*?## / && !/awk/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' ${MAKEFILE_LIST} + +.PHONY:run +run: install ## purge build time artifacts + $(DOCKER_COMMAND) bash + +.PHONY:clean +clean: ## purge build time artifacts + rm -rf dist/ build/ coverage/ pypoetry/ pip/ **/__pycache__/ .pytest_cache/ .cache .coverage + +.PHONY:changelog +changelog: install ## print the next version of the change log to stdout + $(POETRY_COMMAND) run semantic-release changelog --unreleased + +.PHONY:install +install: build-image ## install development and build time dependencies + $(POETRY_COMMAND) install --no-interaction + +.PHONY:lint +lint: install ## run lint rules and print error report + $(POETRY_COMMAND) run task lint + +.PHONY:lint-fix +lint-fix: install ## attempt to auto fix linting error and report remaining errors + $(POETRY_COMMAND) run task lint:fix + +.PHONY:package +package: install ## Generate a python sdist and wheel + $(POETRY_COMMAND) build + +.PHONY:release +release: clean install fetch-tags ## run semantic release build and publish results to github + pypi based on unreleased commits + $(POETRY_COMMAND) run task release + +.PHONY: fetch-tags +fetch-tags: ## workaround for jenkins repo cloning behavior + git fetch origin --tags + +.PHONY:release-dry +release-dry: clean install fetch-tags changelog ## run semantic release in noop mode + $(POETRY_COMMAND) run semantic-release publish --noop --verbosity=DEBUG + +.PHONY:release-patch +release-patch: clean install ## run semantic release build and force a patch release + $(POETRY_COMMAND) run semantic-release publish --patch + +.PHONY:release-minor +release-minor: clean install ## run semantic release build and force a minor release + $(POETRY_COMMAND) run semantic-release publish --minor + +.PHONY:release-major +release-major: clean install ## run semantic release build and force a major release + $(POETRY_COMMAND) run semantic-release publish --major + +.PHONY:test +test: install ## run project test suite + $(POETRY_COMMAND) run task test diff --git a/README.md b/README.md index 431d348..8db2bf3 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,20 @@

- -[![All Contributors](https://img.shields.io/badge/all_contributors-18-orange.svg?style=flat-square)](#contributors-) -

Python package for logging to LogDNA

+ +[![All Contributors](https://img.shields.io/badge/all_contributors-18-orange.svg?style=flat-square)](#contributors-) + + --- * [Installation](#installation) * [Setup](#setup) * [Usage](#usage) - * [Usage with File Config](#usage-with-file-config) + * [Usage with fileConfig](#usage-with-fileconfig) * [API](#api) * [LogDNAHandler(key: string, [options: dict])](#logdnahandlerkey-string-options-dict) * [key](#key) @@ -53,12 +54,14 @@ options = { # Defaults to False; when True meta objects are searchable options['index_meta'] = True +options['custom_fields'] = 'meta' + test = LogDNAHandler(key, options) log.addHandler(test) -log.warning("Warning message", {'app': 'bloop'}) +log.warning("Warning message", extra={'app': 'bloop'}) log.info("Info message") ``` @@ -66,11 +69,10 @@ _**Required**_ * [LogDNA Ingestion Key](https://app.logdna.com/manage/profile) _**Optional**_ -* Hostname - ([string][]) - max length 32 chars +* Hostname - ([string][]) * MAC Address - ([string][]) * IP Address - ([string][]) -* Max Length - ([bool][]) - formatted as options['max_length'] -* Index Meta - ([bool][]) - formatted as options['index_meta'] +* Index Meta - ([bool][]) - formatted as `options['index_meta']` ## Usage @@ -80,10 +82,10 @@ After initial setup, logging is as easy as: log.info('My Sample Log Line') # Add a custom level -log.info('My Sample Log Line', { 'level': 'MyCustomLevel' }) +log.info('My Sample Log Line', extra={ 'level': 'MyCustomLevel' }) # Include an App name with this specific log -log.info('My Sample Log Line', { 'level': 'Warn', 'app': 'myAppName'}) +log.info('My Sample Log Line', extra={ 'level': 'Warn', 'app': 'myAppName'}) # Pass associated objects along as metadata meta = { @@ -98,10 +100,10 @@ opts = { 'meta': meta } -log.info('My Sample Log Line', opts) +log.info('My Sample Log Line', extra=opts) ``` -### Usage with File Config +### Usage with fileConfig To use [LogDNAHandler](#logdnahandlerkey-string-options-dict) with [fileConfig][] (e.g., in a Django `settings.py` file): @@ -116,7 +118,7 @@ LOGGING = { 'logdna': { 'level': logging.DEBUG, 'class': 'logging.handlers.LogDNAHandler', - 'key': os.environ.get('LOGDNA_INGEST_KEY'), + 'key': os.environ.get('LOGDNA_INGESTION_KEY'), 'options': { 'app': '', 'env': os.environ.get('ENVIRONMENT'), @@ -133,7 +135,7 @@ LOGGING = { } ``` -(This example assumes you have set environment variables for `ENVIRONMENT` and `LOGDNA_INGEST_KEY`) +(This example assumes you have set environment variables for `ENVIRONMENT` and `LOGDNA_INGESTION_KEY`.) ## API @@ -143,7 +145,7 @@ LOGGING = { * _**Required**_ * Type: [string][] -* Values: `YourAPIKey` +* Values: `` The [LogDNA API Key](https://app.logdna.com/manage/profile) associated with your account. @@ -154,18 +156,16 @@ The [LogDNA API Key](https://app.logdna.com/manage/profile) associated with your * _Optional_ * Type: [string][] * Default: `''` -* Values: `YourCustomApp` -* Max Length: `32` +* Values: `` -The default app passed along with every log sent through this instance. +The default app named that is included in every every log line sent through this instance. ##### env * _Optional_ * Type: [string][] * Default: `''` -* Values: `YourCustomEnv` -* Max Length: `32` +* Values: `` The default env passed along with every log sent through this instance. @@ -174,8 +174,7 @@ The default env passed along with every log sent through this instance. * _Optional_ * Type: [string][] * Default: `''` -* Values: `YourCustomHostname` -* Max Length: `32` +* Values: `` The default hostname passed along with every log sent through this instance. @@ -189,6 +188,7 @@ Python [LogRecord][] objects includes language-specific information that may be Setting `include_standard_meta` to `True` automatically populates meta objects with `name`, `pathname`, and `lineno` from the [LogRecord][]. +*WARNING* This option is deprecated and will be removed in the upcoming major release. ##### index_meta @@ -200,16 +200,14 @@ We allow meta objects to be passed with each line. By default these meta objects If this option is set to True then meta objects are parsed and searchable up to three levels deep. Any fields deeper than three levels are stringified and cannot be searched. -*WARNING* If this option is True, your metadata objects MUST have consistent types across all log messages or the metadata object may not be parsed properly. - +*WARNING* If this option is True, your metadata objects MUST have consistent types across all log messages or the metadata object might not be parsed properly. ##### level * _Optional_ * Type: [string][] * Default: `Info` -* Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `YourCustomLevel` -* Max Length: `32` +* Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `` The default level passed along with every log sent through this instance. @@ -246,6 +244,22 @@ List of tags used to dynamically group hosts. More information on tags is avail A custom ingestion endpoint to stream log lines into. +##### custom_fields + +* _Optional_ +* Type: [list][]<[string][]> +* Default: `['args', 'name', 'pathname', 'lineno']` + +List of fields out of `record` object to include in the `meta` object. By default, `args`, `name`, `pathname`, and `lineno` will be included. + +##### log_error_response + +* _Optional_ +* Type: [bool][] +* Default: `False` + +Enables logging of the API response when an HTTP error is encountered + ### log(line, [options]) #### line @@ -253,7 +267,6 @@ A custom ingestion endpoint to stream log lines into. * _Required_ * Type: [string][] * Default: `''` -* Max Length: `32000` The log line to be sent to LogDNA. @@ -264,8 +277,7 @@ The log line to be sent to LogDNA. * _Optional_ * Type: [string][] * Default: `Info` -* Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `YourCustomLevel` -* Max Length: `32` +* Values: `Debug`, `Trace`, `Info`, `Warn`, `Error`, `Fatal`, `` The level passed along with this log line. @@ -274,8 +286,7 @@ The level passed along with this log line. * _Optional_ * Type: [string][] * Default: `''` -* Values: `YourCustomApp` -* Max Length: `32` +* Values: `` The app passed along with this log line. @@ -284,8 +295,7 @@ The app passed along with this log line. * _Optional_ * Type: [string][] * Default: `''` -* Values: `YourCustomEnv` -* Max Length: `32` +* Values: `` The environment passed with this log line. @@ -325,13 +335,17 @@ The time in seconds since the epoch to use for the log timestamp. It must be wit This project makes use of the [poetry][] package manager for local development. +```shell +$ poetry install +``` + ### Scripts **lint** Run linting rules w/o attempting to fix them ```shell -$ poetry run lint +$ poetry run task lint ``` @@ -341,7 +355,7 @@ Run lint rules against all local python files and attempt to fix where possible. ```shell -$ poetry run lint:fix +$ poetry run task lint:fix ``` **test**: @@ -349,7 +363,7 @@ $ poetry run lint:fix Runs all unit tests and generates coverage reports ```shell -poetry run test +poetry run task test ``` ## Contributors ✨ diff --git a/logdna/VERSION b/logdna/VERSION index 94fe62c..43bae12 100644 --- a/logdna/VERSION +++ b/logdna/VERSION @@ -1 +1 @@ -1.5.4 +1.18.12 diff --git a/logdna/configs.py b/logdna/configs.py index 3c808c9..d790776 100644 --- a/logdna/configs.py +++ b/logdna/configs.py @@ -1,15 +1,19 @@ -from os import path +from os import path, sep -with open("%s/VERSION" % path.abspath(path.dirname(__file__))) as f: +with open("{p}{s}VERSION".format(p=path.abspath(path.dirname(__file__)), + s=sep)) as f: version = f.read().strip('\n') defaults = { 'DEFAULT_REQUEST_TIMEOUT': 30, - 'MAX_LINE_LENGTH': 32000, - 'FLUSH_INTERVAL_SECS': 5, - 'FLUSH_BYTE_LIMIT': 2 * 1024 * 1024, + 'FLUSH_INTERVAL_SECS': 0.25, + 'FLUSH_LIMIT': 2 * 1024 * 1024, + 'MAX_CONCURRENT_REQUESTS': 10, + 'MAX_RETRY_ATTEMPTS': 3, + 'MAX_RETRY_JITTER': 0.5, + 'META_FIELDS': ['args', 'name', 'pathname', 'lineno'], 'LOGDNA_URL': 'https://logs.logdna.com/logs/ingest', - 'BUF_RETENTION_BYTE_LIMIT': 4 * 1024 * 1024, - 'RETRY_INTERVAL_SECS': 8, + 'BUF_RETENTION_LIMIT': 4 * 1024 * 1024, + 'RETRY_INTERVAL_SECS': 5, 'USER_AGENT': 'python/%s' % version } diff --git a/logdna/logdna.py b/logdna/logdna.py index eeb6000..e9f6128 100644 --- a/logdna/logdna.py +++ b/logdna/logdna.py @@ -1,190 +1,355 @@ -import json import logging import requests import socket import sys import threading import time -from functools import reduce + +from concurrent.futures import ThreadPoolExecutor from .configs import defaults -from .utils import sanitize_meta, get_ip +from .utils import sanitize_meta, get_ip, normalize_list_option class LogDNAHandler(logging.Handler): def __init__(self, key, options={}): + # Setup Handler logging.Handler.__init__(self) + + # Set Internal Logger self.internal_handler = logging.StreamHandler(sys.stdout) self.internal_handler.setLevel(logging.DEBUG) self.internalLogger = logging.getLogger('internal') - self.internalLogger.setLevel(logging.DEBUG) self.internalLogger.addHandler(self.internal_handler) + self.internalLogger.setLevel(logging.DEBUG) + # Set the Custom Variables self.key = key - self.buf = [] - self.secondary = [] - self.exception_flag = False - self.flusher = None - self.max_length = defaults['MAX_LINE_LENGTH'] - self.hostname = options.get('hostname', socket.gethostname()) self.ip = options.get('ip', get_ip()) self.mac = options.get('mac', None) - self.level = options.get('level', 'info') - self.verbose = str(options.get('verbose', 'true')).lower() + self.loglevel = options.get('level', 'info') self.app = options.get('app', '') self.env = options.get('env', '') + self.tags = normalize_list_option(options, 'tags') + self.custom_fields = normalize_list_option(options, 'custom_fields') + self.custom_fields += defaults['META_FIELDS'] + self.log_error_response = options.get('log_error_response', False) + + # Set the Connection Variables self.url = options.get('url', defaults['LOGDNA_URL']) self.request_timeout = options.get('request_timeout', defaults['DEFAULT_REQUEST_TIMEOUT']) - self.include_standard_meta = options.get('include_standard_meta', - False) - self.index_meta = options.get('index_meta', False) - self.flush_limit = options.get('flush_limit', - defaults['FLUSH_BYTE_LIMIT']) - self.flush_interval_secs = options.get('flush_interval', - defaults['FLUSH_INTERVAL_SECS']) + self.user_agent = options.get('user_agent', defaults['USER_AGENT']) + self.max_retry_attempts = options.get('max_retry_attempts', + defaults['MAX_RETRY_ATTEMPTS']) + self.max_retry_jitter = options.get('max_retry_jitter', + defaults['MAX_RETRY_JITTER']) + self.max_concurrent_requests = options.get( + 'max_concurrent_requests', defaults['MAX_CONCURRENT_REQUESTS']) self.retry_interval_secs = options.get('retry_interval_secs', defaults['RETRY_INTERVAL_SECS']) - self.tags = options.get('tags', []) - self.buf_retention_byte_limit = options.get( - 'buf_retention_limit', defaults['BUF_RETENTION_BYTE_LIMIT']) - self.user_agent = options.get('user_agent', defaults['USER_AGENT']) - if isinstance(self.tags, str): - self.tags = [tag.strip() for tag in self.tags.split(',')] - elif not isinstance(self.tags, list): - self.tags = [] - self.setLevel(logging.DEBUG) - self.lock = threading.RLock() + # Set the Flush-related Variables + self.buf = [] + self.buf_size = 0 - # TODO(esatterwhite): complexity too high (8) - def buffer_log(self, message): - if message and message['line']: - if len(message['line']) > self.max_length: - message['line'] = message[ - 'line'][:self.max_length] + ' (cut off, too long...)' - if self.verbose in ['true', 'debug', 'd']: - self.internalLogger.debug( - 'Line was longer than %s chars and was truncated.', - self.max_length) + self.include_standard_meta = options.get('include_standard_meta', None) - # Attempt to acquire lock to write to buf - # otherwise write to secondary as flush occurs - if self.lock.acquire(blocking=False): - buf_size = reduce(lambda x, y: x + len(y['line']), self.buf, 0) + if self.include_standard_meta is not None: + self.internalLogger.debug( + '"include_standard_meta" option will be deprecated ' + + 'removed in the upcoming major release') - if buf_size + len(message['line']) < self.buf_retention_byte_limit: - self.buf.append(message) - else: - self.internalLogger.debug( - 'The buffer size exceeded the limit: %s', - self.buf_retention_byte_limit) - self.lock.release() + self.index_meta = options.get('index_meta', False) + self.flush_limit = options.get('flush_limit', defaults['FLUSH_LIMIT']) + self.flush_interval_secs = options.get('flush_interval', + defaults['FLUSH_INTERVAL_SECS']) + self.buf_retention_limit = options.get('buf_retention_limit', + defaults['BUF_RETENTION_LIMIT']) + + # Set up the Thread Pools + self.worker_thread_pool = ThreadPoolExecutor() + self.request_thread_pool = ThreadPoolExecutor( + max_workers=self.max_concurrent_requests) + + self.setLevel(logging.DEBUG) + self._lock = threading.RLock() - if buf_size >= self.flush_limit and not self.exception_flag: - self.flush() - else: - self.secondary.append(message) + self.flusher = None + def start_flusher(self): if not self.flusher: - interval = (self.retry_interval_secs - if self.exception_flag else self.flush_interval_secs) - self.flusher = threading.Timer(interval, self.flush) + self.flusher = threading.Timer( + self.flush_interval_secs, self.flush) self.flusher.start() - def clean_after_success(self): - del self.buf[:] - self.exception_flag = False + def close_flusher(self): if self.flusher: self.flusher.cancel() self.flusher = None - def handle_exception(self, exception): - if self.flusher: - self.flusher.cancel() - self.flusher = None - self.exception_flag = True - if self.verbose in ['true', 'error', 'err', 'e']: - self.internalLogger.debug('Error sending logs %s', exception) - - # do not call without acquiring the lock - def send_request(self): - self.buf.extend(self.secondary) - self.secondary = [] - data = {'e': 'ls', 'ls': self.buf} - try: - res = requests.post(url=self.url, - json=data, - auth=('user', self.key), - params={ - 'hostname': self.hostname, - 'ip': self.ip, - 'mac': self.mac if self.mac else None, - 'tags': self.tags if self.tags else None - }, - stream=True, - timeout=self.request_timeout, - headers={'user-agent': self.user_agent}) - res.raise_for_status() - # when no RequestException happened - self.clean_after_success() - except requests.exceptions.RequestException as e: - self.handle_exception(e) + def buffer_log(self, message): + if self.worker_thread_pool: + try: + self.worker_thread_pool.submit(self.buffer_log_sync, message) + except RuntimeError: + self.buffer_log_sync(message) + except Exception as e: + self.internalLogger.debug('Error in calling buffer_log: %s', e) + + def buffer_log_sync(self, message): + # Attempt to acquire lock to write to buffer + if self._lock.acquire(blocking=True): + try: + msglen = len(message['line']) + if self.buf_size + msglen < self.buf_retention_limit: + self.buf.append(message) + self.buf_size += msglen + else: + self.internalLogger.debug( + 'The buffer size exceeded the limit: %s', + self.buf_retention_limit) + + if self.buf_size >= self.flush_limit: + self.close_flusher() + self.flush() + else: + self.start_flusher() + except Exception as e: + self.internalLogger.exception(f'Error in buffer_log_sync: {e}') + finally: + self._lock.release() def flush(self): - if len(self.buf) == 0: - return - if self.lock.acquire(blocking=False): - self.send_request() - self.lock.release() - else: - if not self.flusher: - self.flusher = threading.Timer(1, self.flush) - self.flusher.start() - - # TODO(esatterwhite): complexity too high (14) - def emit(self, record): # noqa: C901 + self.schedule_flush_sync() + + def schedule_flush_sync(self, should_block=False): + if self.request_thread_pool: + try: + self.request_thread_pool.submit( + self.try_lock_and_do_flush_request, should_block) + except RuntimeError: + self.try_lock_and_do_flush_request(should_block) + except Exception as e: + self.internalLogger.debug( + 'Error in calling try_lock_and_do_flush_request: %s', e) + + def try_lock_and_do_flush_request(self, should_block=False): + local_buf = [] + if self._lock.acquire(blocking=should_block): + if not self.buf: + self.close_flusher() + self._lock.release() + return + + local_buf = self.buf.copy() + self.buf.clear() + self.buf_size = 0 + if local_buf: + self.close_flusher() + self._lock.release() + + if local_buf: + self.try_request(local_buf) + + def try_request(self, buf): + data = {'e': 'ls', 'ls': buf} + retries = 0 + while retries < self.max_retry_attempts: + retries += 1 + if self.send_request(data): + break + + sleep_time = self.retry_interval_secs * (1 << (retries - 1)) + sleep_time += self.max_retry_jitter + time.sleep(sleep_time) + + if retries >= self.max_retry_attempts: + self.internalLogger.debug( + 'Flush exceeded %s tries. Discarding flush buffer', + self.max_retry_attempts) + + def send_request(self, data): # noqa: max-complexity: 13 + """ + Send log data to LogDNA server + Returns: + True - discard flush buffer + False - retry, keep flush buffer + """ + try: + headers = { + 'user-agent': self.user_agent, + 'apikey': self.key + } + response = requests.post(url=self.url, + json=data, + params={ + 'hostname': self.hostname, + 'ip': self.ip, + 'mac': self.mac, + 'tags': self.tags, + 'now': int(time.time() * 1000) + }, + stream=True, + allow_redirects=True, + timeout=self.request_timeout, + headers=headers) + + status_code = response.status_code + ''' + response code: + 1XX unexpected status + 200 expected status, OK + 2XX unexpected status + 301 302 303 unexpected status, + per "allow_redirects=True" + 3XX unexpected status + 401, 403 expected client error, + invalid ingestion key + 429 expected server error, + "client error", transient + 4XX unexpected client error + 500 502 503 504 507 expected server error, transient + 5XX unexpected server error + handling: + expected status discard flush buffer + unexpected status log + discard flush buffer + expected client error log + discard flush buffer + unexpected client error log + discard flush buffer + expected server error log + retry + unexpected server error log + discard flush buffer + ''' + if status_code == 200: + return True # discard + + if isinstance(response.reason, bytes): + # We attempt to decode utf-8 first because some servers + # choose to localize their reason strings. If the string + # isn't utf-8, we fall back to iso-8859-1 for all other + # encodings. (See PR #3538) + try: + reason = response.reason.decode('utf-8') + except UnicodeDecodeError: + reason = response.reason.decode('iso-8859-1') + else: + reason = response.reason + + if 200 < status_code <= 399: + self.internalLogger.debug('Unexpected response: %s. ' + + 'Discarding flush buffer', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return True # discard + + if status_code in [401, 403]: + self.internalLogger.debug( + 'Please provide a valid ingestion key. ' + + 'Discarding flush buffer') + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return True # discard + + if status_code == 429: + self.internalLogger.debug('Client Error: %s. Retrying...', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return False # retry + + if 400 <= status_code <= 499: + self.internalLogger.debug('Client Error: %s. ' + + 'Discarding flush buffer', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return True # discard + + if status_code in [500, 502, 503, 504, 507]: + self.internalLogger.debug('Server Error: %s. Retrying...', + reason) + if self.log_error_response: + self.internalLogger.debug( + 'Error Response: %s', response.text) + return False # retry + + self.internalLogger.debug('The request failed: %s.' + + 'Discarding flush buffer', + reason) + + except requests.exceptions.Timeout as timeout: + self.internalLogger.debug('Timeout Error: %s. Retrying...', + timeout) + return False # retry + + except requests.exceptions.RequestException as exception: + self.internalLogger.debug( + 'Error sending logs %s. Discarding flush buffer', exception) + + return True # discard + + def emit(self, record): msg = self.format(record) record = record.__dict__ - opts = {} - if 'args' in record: - opts = record['args'] - if self.include_standard_meta: - if isinstance(opts, tuple): - opts = {} - if 'meta' not in opts: - opts['meta'] = {} - for key in ['name', 'pathname', 'lineno']: - opts['meta'][key] = record[key] - message = { 'hostname': self.hostname, 'timestamp': int(time.time() * 1000), 'line': msg, - 'level': record['levelname'] or self.level, + 'level': record['levelname'] or self.loglevel, 'app': self.app or record['module'], - 'env': self.env + 'env': self.env, + 'meta': {} } - if not isinstance(opts, tuple): - if 'level' in opts: - message['level'] = opts['level'] - if 'app' in opts: - message['app'] = opts['app'] - if 'hostname' in opts: - message['hostname'] = opts['hostname'] - if 'env' in opts: - message['env'] = opts['env'] - if 'timestamp' in opts: - message['timestamp'] = opts['timestamp'] - if 'meta' in opts: - if self.index_meta: - message['meta'] = sanitize_meta(opts['meta']) - else: - message['meta'] = json.dumps(opts['meta']) + + for key in self.custom_fields: + if key in record: + if isinstance(record[key], tuple): + message['meta'][key] = list(record[key]) + elif record[key] is not None: + message['meta'][key] = record[key] + + message['meta'] = sanitize_meta(message['meta'], self.index_meta) + + opts = {} + if 'args' in record and not isinstance(record['args'], tuple): + opts = record['args'] + + for key in ['app', 'env', 'hostname', 'level', 'timestamp']: + if key in opts: + message[key] = opts[key] + self.buffer_log(message) def close(self): - if len(self.buf) > 0: - self.flush() + # Close the flusher + self.close_flusher() + + # First gracefully shut down any threads that are still attempting + # to add log messages to the buffer. This ensures that we don't lose + # any log messages that are in the process of being added to the + # buffer. + if self.worker_thread_pool: + self.worker_thread_pool.shutdown(wait=True) + self.worker_thread_pool = None + + # Manually force a flush of any remaining log messages in the buffer. + # We block here to ensure that the flush completes prior to the + # application exiting and because the probability of this + # introducing a noticeable delay is very low because close() is only + # called when the logger and application are shutting down. + self.schedule_flush_sync(should_block=True) + + # Finally, shut down the thread pool that was used to send the log + # messages to the server. We can assume at this point that all log + # messages that were in the buffer prior to the worker threads + # shutting down have been sent to the server. + if self.request_thread_pool: + self.request_thread_pool.shutdown(wait=True) + self.request_thread_pool = None + logging.Handler.close(self) diff --git a/logdna/utils.py b/logdna/utils.py index 790ae0a..7278f12 100644 --- a/logdna/utils.py +++ b/logdna/utils.py @@ -10,7 +10,24 @@ def is_jsonable(obj): return False -def sanitize_meta(meta): +def normalize_list_option(options, key): + value = options.get(key, []) + if isinstance(value, str): + value = [val.strip() for val in value.split(',')] + elif not isinstance(value, list): + value = [] + return value + + +def sanitize_meta(meta, index_meta=False): + if not index_meta: + if is_jsonable(meta): + return json.dumps(meta) + + return { + '__errors': 'Meta cannot be serialized into JSON-formatted string' + } + keys_to_sanitize = [] for key, value in meta.items(): if not is_jsonable(value): diff --git a/poetry.lock b/poetry.lock index 93c6849..c7fb7c7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,32 +1,26 @@ [[package]] name = "appnope" -version = "0.1.2" +version = "0.1.3" description = "Disable App Nap on macOS >= 10.9" category = "dev" optional = false python-versions = "*" -[[package]] -name = "atomicwrites" -version = "1.4.0" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - [[package]] name = "attrs" -version = "20.3.0" +version = "22.2.0" description = "Classes Without Boilerplate" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] -docs = ["furo", "sphinx", "zope.interface"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"] +dev = ["attrs"] +docs = ["furo", "sphinx", "myst-parser", "zope.interface", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"] +tests = ["attrs", "zope.interface"] +tests-no-zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] +tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"] [[package]] name = "backcall" @@ -36,33 +30,82 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "bleach" +version = "6.0.0" +description = "An easy safelist-based HTML-sanitizing tool." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +six = ">=1.9.0" +webencodings = "*" + +[package.extras] +css = ["tinycss2 (>=1.1.0,<1.2)"] + [[package]] name = "certifi" -version = "2020.12.5" +version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false +python-versions = ">=3.6" + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "dev" +optional = false python-versions = "*" +[package.dependencies] +pycparser = "*" + [[package]] -name = "chardet" -version = "4.0.0" -description = "Universal encoding detector for Python 2 and 3" +name = "charset-normalizer" +version = "3.0.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "*" + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "click-log" +version = "0.4.0" +description = "Logging integration for Click" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +click = "*" [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.6" description = "Cross-platform colored terminal text." category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" [[package]] name = "coverage" -version = "5.4" +version = "5.5" description = "Code coverage measurement for Python" category = "dev" optional = false @@ -71,93 +114,189 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] toml = ["toml"] +[[package]] +name = "cryptography" +version = "39.0.0" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1,!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "ruff"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + [[package]] name = "decorator" -version = "4.4.2" +version = "5.1.1" description = "Decorators for Humans" category = "dev" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*" +python-versions = ">=3.5" + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "dotty-dict" +version = "1.3.1" +description = "Dictionary wrapper for quick access to deeply nested keys." +category = "dev" +optional = false +python-versions = ">=3.5,<4.0" + +[[package]] +name = "exceptiongroup" +version = "1.1.0" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] [[package]] name = "flake8" -version = "3.8.4" +version = "3.9.2" description = "the modular source code checker: pep8 pyflakes and co" category = "dev" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [package.dependencies] importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} mccabe = ">=0.6.0,<0.7.0" -pycodestyle = ">=2.6.0a1,<2.7.0" -pyflakes = ">=2.2.0,<2.3.0" +pycodestyle = ">=2.7.0,<2.8.0" +pyflakes = ">=2.3.0,<2.4.0" + +[[package]] +name = "gitdb" +version = "4.0.10" +description = "Git Object Database" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.30" +description = "GitPython is a python library used to interact with Git repositories" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.7.4.3", markers = "python_version < \"3.8\""} [[package]] name = "idna" -version = "2.10" +version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.5" [[package]] name = "importlib-metadata" -version = "3.4.0" +version = "6.0.0" description = "Read metadata from Python packages" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] + +[[package]] +name = "importlib-resources" +version = "5.10.2" +description = "Read resources from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] [[package]] name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "invoke" +version = "1.7.3" +description = "Pythonic task execution" category = "dev" optional = false python-versions = "*" [[package]] name = "ipdb" -version = "0.13.4" +version = "0.13.11" description = "IPython-enabled pdb" category = "dev" optional = false -python-versions = ">=2.7" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] -ipython = {version = ">=5.1.0", markers = "python_version >= \"3.4\""} +decorator = {version = "*", markers = "python_version > \"3.6\" and python_version < \"3.11\" or python_version >= \"3.11\""} +ipython = {version = ">=7.31.1", markers = "python_version > \"3.6\" and python_version < \"3.11\" or python_version >= \"3.11\""} +tomli = {version = "*", markers = "python_version > \"3.6\" and python_version < \"3.11\""} [[package]] name = "ipython" -version = "7.16.1" +version = "7.34.0" description = "IPython: Productive Interactive Computing" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] appnope = {version = "*", markers = "sys_platform == \"darwin\""} backcall = "*" colorama = {version = "*", markers = "sys_platform == \"win32\""} decorator = "*" -jedi = ">=0.10" -pexpect = {version = "*", markers = "sys_platform != \"win32\""} +jedi = ">=0.16" +matplotlib-inline = "*" +pexpect = {version = ">4.3", markers = "sys_platform != \"win32\""} pickleshare = "*" prompt-toolkit = ">=2.0.0,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.1.0" pygments = "*" traitlets = ">=4.2" [package.extras] -all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.14)", "pygments", "qtconsole", "requests", "testpath"] +all = ["Sphinx (>=1.3)", "ipykernel", "ipyparallel", "ipywidgets", "nbconvert", "nbformat", "nose (>=0.10.1)", "notebook", "numpy (>=1.17)", "pygments", "qtconsole", "requests", "testpath"] doc = ["Sphinx (>=1.3)"] kernel = ["ipykernel"] nbconvert = ["nbconvert"] @@ -165,19 +304,26 @@ nbformat = ["nbformat"] notebook = ["notebook", "ipywidgets"] parallel = ["ipyparallel"] qtconsole = ["qtconsole"] -test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.14)"] +test = ["nose (>=0.10.1)", "requests", "testpath", "pygments", "nbformat", "ipykernel", "numpy (>=1.17)"] [[package]] -name = "ipython-genutils" -version = "0.2.0" -description = "Vestigial utilities from IPython" +name = "jaraco.classes" +version = "3.2.3" +description = "Utility functions for Python class constructs" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] [[package]] name = "jedi" -version = "0.18.0" +version = "0.18.2" description = "An autocompletion tool for Python that can be used for text editors." category = "dev" optional = false @@ -187,8 +333,53 @@ python-versions = ">=3.6" parso = ">=0.8.0,<0.9.0" [package.extras] +docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx-rtd-theme (==0.4.3)", "sphinx (==1.8.5)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] -testing = ["Django (<3.1)", "colorama", "docopt", "pytest (<6.0.0)"] +testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] + +[[package]] +name = "jeepney" +version = "0.8.0" +description = "Low-level, pure Python DBus protocol wrapper." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest", "pytest-trio", "pytest-asyncio (>=0.17)", "testpath", "trio", "async-timeout"] +trio = ["trio", "async-generator"] + +[[package]] +name = "keyring" +version = "23.13.1" +description = "Store and access your passwords safely." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +importlib-metadata = {version = ">=4.11.4", markers = "python_version < \"3.12\""} +importlib-resources = {version = "*", markers = "python_version < \"3.9\""} +"jaraco.classes" = "*" +jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""} +pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} + +[package.extras] +completion = ["shtab"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + +[[package]] +name = "matplotlib-inline" +version = "0.1.6" +description = "Inline Matplotlib backend for Jupyter" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +traitlets = "*" [[package]] name = "mccabe" @@ -198,20 +389,33 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "more-itertools" +version = "9.0.0" +description = "More routines for operating on iterables, beyond itertools" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "mslex" +version = "0.3.0" +description = "shlex for windows" +category = "dev" +optional = false +python-versions = ">=3.5" + [[package]] name = "packaging" -version = "20.9" +version = "23.0" description = "Core utilities for Python packages" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -pyparsing = ">=2.0.2" +python-versions = ">=3.7" [[package]] name = "parso" -version = "0.8.1" +version = "0.8.3" description = "A Python Parser" category = "dev" optional = false @@ -240,31 +444,54 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "pkginfo" +version = "1.9.6" +description = "Query metadata from sdists / bdists / installed packages." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +testing = ["pytest", "pytest-cov"] + [[package]] name = "pluggy" -version = "0.13.1" +version = "1.0.0" description = "plugin and hook calling mechanisms for python" category = "dev" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" [package.dependencies] importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} [package.extras] dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] [[package]] name = "prompt-toolkit" -version = "3.0.3" +version = "3.0.36" description = "Library for building powerful interactive command lines in Python" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.6.2" [package.dependencies] wcwidth = "*" +[[package]] +name = "psutil" +version = "5.9.4" +description = "Cross-platform lib for process and system monitoring in Python." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +test = ["ipaddress", "mock", "enum34", "pywin32", "wmi"] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -274,24 +501,24 @@ optional = false python-versions = "*" [[package]] -name = "py" -version = "1.10.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "pycodestyle" +version = "2.7.0" +description = "Python style guide checker" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] -name = "pycodestyle" -version = "2.6.0" -description = "Python style guide checker" +name = "pycparser" +version = "2.21" +description = "C parser in Python" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pyflakes" -version = "2.2.0" +version = "2.3.1" description = "passive checker of Python programs" category = "dev" optional = false @@ -299,45 +526,39 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "pygments" -version = "2.7.4" +version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" -[[package]] -name = "pyparsing" -version = "2.4.7" -description = "Python parsing module" -category = "dev" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[package.extras] +plugins = ["importlib-metadata"] [[package]] name = "pytest" -version = "6.2.2" +version = "7.2.1" description = "pytest: simple powerful testing with Python" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0.0a1" -py = ">=1.8.2" -toml = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] [[package]] name = "pytest-cov" -version = "2.11.1" +version = "2.12.1" description = "Pytest plugin for measuring coverage." category = "dev" optional = false @@ -346,39 +567,157 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] coverage = ">=5.2.1" pytest = ">=4.6" +toml = "*" [package.extras] -testing = ["fields", "hunter", "process-tests (==2.0.2)", "six", "pytest-xdist", "virtualenv"] +testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"] + +[[package]] +name = "python-gitlab" +version = "3.12.0" +description = "Interact with GitLab API" +category = "dev" +optional = false +python-versions = ">=3.7.0" + +[package.dependencies] +requests = ">=2.25.0" +requests-toolbelt = ">=0.9.1" + +[package.extras] +autocompletion = ["argcomplete (>=1.10.0,<3)"] +yaml = ["PyYaml (>=5.2)"] + +[[package]] +name = "python-semantic-release" +version = "7.33.0" +description = "Automatic Semantic Versioning for Python projects" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +click = ">=7,<9" +click-log = ">=0.3,<1" +dotty-dict = ">=1.3.0,<2" +gitpython = ">=3.0.8,<4" +invoke = ">=1.4.1,<2" +packaging = "*" +python-gitlab = ">=2,<4" +requests = ">=2.25,<3" +semver = ">=2.10,<3" +tomlkit = ">=0.10,<1.0" +twine = ">=3,<4" + +[package.extras] +dev = ["tox", "isort", "black"] +docs = ["Sphinx (==1.3.6)", "Jinja2 (==3.0.3)"] +mypy = ["mypy", "types-requests"] +test = ["coverage (>=5,<6)", "pytest (>=7,<8)", "pytest-xdist (>=1,<2)", "pytest-mock (>=2,<3)", "responses (==0.13.3)", "mock (==1.3.0)"] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.0" +description = "" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "readme-renderer" +version = "37.3" +description = "readme_renderer is a library for rendering \"readme\" descriptions for Warehouse" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +bleach = ">=2.1.0" +docutils = ">=0.13.1" +Pygments = ">=2.5.1" + +[package.extras] +md = ["cmarkgfm (>=0.8.0)"] [[package]] name = "requests" -version = "2.25.1" +version = "2.28.2" description = "Python HTTP for Humans." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7, <4" [package.dependencies] certifi = ">=2017.4.17" -chardet = ">=3.0.2,<5" -idna = ">=2.5,<3" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" urllib3 = ">=1.21.1,<1.27" [package.extras] -security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] -socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-toolbelt" +version = "0.10.1" +description = "A utility belt for advanced users of python-requests" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +requests = ">=2.0.1,<3.0.0" + +[[package]] +name = "rfc3986" +version = "2.0.0" +description = "Validating URI References per RFC 3986" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +idna2008 = ["idna"] + +[[package]] +name = "secretstorage" +version = "3.3.3" +description = "Python bindings to FreeDesktop.org Secret Service API" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cryptography = ">=2.0" +jeepney = ">=0.6" + +[[package]] +name = "semver" +version = "2.13.0" +description = "Python helper for Semantic Versioning (http://semver.org/)" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "six" -version = "1.15.0" +version = "1.16.0" description = "Python 2 and 3 compatibility utilities" category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "smmap" +version = "5.0.0" +description = "A pure Python implementation of a sliding window memory map manager" +category = "dev" +optional = false +python-versions = ">=3.6" + [[package]] name = "tap.py" -version = "3.0" +version = "3.1" description = "Test Anything Protocol (TAP) tools" category = "dev" optional = false @@ -387,6 +726,20 @@ python-versions = "*" [package.extras] yaml = ["more-itertools", "PyYAML (>=5.1)"] +[[package]] +name = "taskipy" +version = "1.10.3" +description = "tasks runner for python projects" +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +colorama = ">=0.4.4,<0.5.0" +mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} +psutil = ">=5.7.2,<6.0.0" +tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version >= \"3.7\" and python_version < \"4.0\""} + [[package]] name = "toml" version = "0.10.2" @@ -396,50 +749,107 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] -name = "traitlets" -version = "4.3.3" -description = "Traitlets Python config system" +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" + +[[package]] +name = "tomlkit" +version = "0.11.6" +description = "Style preserving TOML library" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "tqdm" +version = "4.64.1" +description = "Fast, Extensible Progress Meter" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" [package.dependencies] -decorator = "*" -ipython-genutils = "*" -six = "*" +colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -test = ["pytest", "mock"] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "traitlets" +version = "5.8.1" +description = "Traitlets Python configuration system" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["myst-parser", "pydata-sphinx-theme", "sphinx"] +test = ["argcomplete (>=2.0)", "pre-commit", "pytest", "pytest-mock"] + +[[package]] +name = "twine" +version = "3.8.0" +description = "Collection of utilities for publishing packages on PyPI" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = ">=0.4.3" +importlib-metadata = ">=3.6" +keyring = ">=15.1" +pkginfo = ">=1.8.1" +readme-renderer = ">=21.0" +requests = ">=2.20" +requests-toolbelt = ">=0.8.0,<0.9.0 || >0.9.0" +rfc3986 = ">=1.4.0" +tqdm = ">=4.14" +urllib3 = ">=1.26.0" [[package]] name = "typing-extensions" -version = "3.7.4.3" -description = "Backported and Experimental Type Hints for Python 3.5+" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" category = "dev" optional = false -python-versions = "*" +python-versions = ">=3.7" [[package]] name = "urllib3" -version = "1.26.3" +version = "1.26.14" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" [package.extras] -brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "wcwidth" -version = "0.2.5" +version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" category = "dev" optional = false python-versions = "*" +[[package]] +name = "webencodings" +version = "0.5.1" +description = "Character encoding aliases for legacy web content" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "yapf" version = "0.30.0" @@ -450,234 +860,93 @@ python-versions = "*" [[package]] name = "zipp" -version = "3.4.0" +version = "3.11.0" description = "Backport of pathlib-compatible object wrapper for zip files" category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] [metadata] lock-version = "1.1" -python-versions = "^3.6" -content-hash = "b357baa9d439bc7f7cf618918ae08e8da1f71844547172b4e448193c0f6bbb10" +python-versions = "^3.7" +content-hash = "2ce3c41b1d2a1222b4e53674ab5901777d35e52bcee9eaeb93ff625c74e14d31" [metadata.files] -appnope = [ - {file = "appnope-0.1.2-py2.py3-none-any.whl", hash = "sha256:93aa393e9d6c54c5cd570ccadd8edad61ea0c4b9ea7a01409020c9aa019eb442"}, - {file = "appnope-0.1.2.tar.gz", hash = "sha256:dd83cd4b5b460958838f6eb3000c660b1f9caf2a5b1de4264e941512f603258a"}, -] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] -attrs = [ - {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, - {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, -] -backcall = [ - {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, - {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, -] -certifi = [ - {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, - {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, -] -chardet = [ - {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, - {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, -] -colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -coverage = [ - {file = "coverage-5.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:6d9c88b787638a451f41f97446a1c9fd416e669b4d9717ae4615bd29de1ac135"}, - {file = "coverage-5.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:66a5aae8233d766a877c5ef293ec5ab9520929c2578fd2069308a98b7374ea8c"}, - {file = "coverage-5.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9754a5c265f991317de2bac0c70a746efc2b695cf4d49f5d2cddeac36544fb44"}, - {file = "coverage-5.4-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:fbb17c0d0822684b7d6c09915677a32319f16ff1115df5ec05bdcaaee40b35f3"}, - {file = "coverage-5.4-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:b7f7421841f8db443855d2854e25914a79a1ff48ae92f70d0a5c2f8907ab98c9"}, - {file = "coverage-5.4-cp27-cp27m-win32.whl", hash = "sha256:4a780807e80479f281d47ee4af2eb2df3e4ccf4723484f77da0bb49d027e40a1"}, - {file = "coverage-5.4-cp27-cp27m-win_amd64.whl", hash = "sha256:87c4b38288f71acd2106f5d94f575bc2136ea2887fdb5dfe18003c881fa6b370"}, - {file = "coverage-5.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6809ebcbf6c1049002b9ac09c127ae43929042ec1f1dbd8bb1615f7cd9f70a0"}, - {file = "coverage-5.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ba7ca81b6d60a9f7a0b4b4e175dcc38e8fef4992673d9d6e6879fd6de00dd9b8"}, - {file = "coverage-5.4-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:89fc12c6371bf963809abc46cced4a01ca4f99cba17be5e7d416ed7ef1245d19"}, - {file = "coverage-5.4-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4a8eb7785bd23565b542b01fb39115a975fefb4a82f23d407503eee2c0106247"}, - {file = "coverage-5.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:7e40d3f8eb472c1509b12ac2a7e24158ec352fc8567b77ab02c0db053927e339"}, - {file = "coverage-5.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1ccae21a076d3d5f471700f6d30eb486da1626c380b23c70ae32ab823e453337"}, - {file = "coverage-5.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:755c56beeacac6a24c8e1074f89f34f4373abce8b662470d3aa719ae304931f3"}, - {file = "coverage-5.4-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:322549b880b2d746a7672bf6ff9ed3f895e9c9f108b714e7360292aa5c5d7cf4"}, - {file = "coverage-5.4-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:60a3307a84ec60578accd35d7f0c71a3a971430ed7eca6567399d2b50ef37b8c"}, - {file = "coverage-5.4-cp35-cp35m-win32.whl", hash = "sha256:1375bb8b88cb050a2d4e0da901001347a44302aeadb8ceb4b6e5aa373b8ea68f"}, - {file = "coverage-5.4-cp35-cp35m-win_amd64.whl", hash = "sha256:16baa799ec09cc0dcb43a10680573269d407c159325972dd7114ee7649e56c66"}, - {file = "coverage-5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2f2cf7a42d4b7654c9a67b9d091ec24374f7c58794858bff632a2039cb15984d"}, - {file = "coverage-5.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:b62046592b44263fa7570f1117d372ae3f310222af1fc1407416f037fb3af21b"}, - {file = "coverage-5.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:812eaf4939ef2284d29653bcfee9665f11f013724f07258928f849a2306ea9f9"}, - {file = "coverage-5.4-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:859f0add98707b182b4867359e12bde806b82483fb12a9ae868a77880fc3b7af"}, - {file = "coverage-5.4-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:04b14e45d6a8e159c9767ae57ecb34563ad93440fc1b26516a89ceb5b33c1ad5"}, - {file = "coverage-5.4-cp36-cp36m-win32.whl", hash = "sha256:ebfa374067af240d079ef97b8064478f3bf71038b78b017eb6ec93ede1b6bcec"}, - {file = "coverage-5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:84df004223fd0550d0ea7a37882e5c889f3c6d45535c639ce9802293b39cd5c9"}, - {file = "coverage-5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1b811662ecf72eb2d08872731636aee6559cae21862c36f74703be727b45df90"}, - {file = "coverage-5.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6b588b5cf51dc0fd1c9e19f622457cc74b7d26fe295432e434525f1c0fae02bc"}, - {file = "coverage-5.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:3fe50f1cac369b02d34ad904dfe0771acc483f82a1b54c5e93632916ba847b37"}, - {file = "coverage-5.4-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:32ab83016c24c5cf3db2943286b85b0a172dae08c58d0f53875235219b676409"}, - {file = "coverage-5.4-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:68fb816a5dd901c6aff352ce49e2a0ffadacdf9b6fae282a69e7a16a02dad5fb"}, - {file = "coverage-5.4-cp37-cp37m-win32.whl", hash = "sha256:a636160680c6e526b84f85d304e2f0bb4e94f8284dd765a1911de9a40450b10a"}, - {file = "coverage-5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:bb32ca14b4d04e172c541c69eec5f385f9a075b38fb22d765d8b0ce3af3a0c22"}, - {file = "coverage-5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4d7165a4e8f41eca6b990c12ee7f44fef3932fac48ca32cecb3a1b2223c21f"}, - {file = "coverage-5.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:a565f48c4aae72d1d3d3f8e8fb7218f5609c964e9c6f68604608e5958b9c60c3"}, - {file = "coverage-5.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:fff1f3a586246110f34dc762098b5afd2de88de507559e63553d7da643053786"}, - {file = "coverage-5.4-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:a839e25f07e428a87d17d857d9935dd743130e77ff46524abb992b962eb2076c"}, - {file = "coverage-5.4-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:6625e52b6f346a283c3d563d1fd8bae8956daafc64bb5bbd2b8f8a07608e3994"}, - {file = "coverage-5.4-cp38-cp38-win32.whl", hash = "sha256:5bee3970617b3d74759b2d2df2f6a327d372f9732f9ccbf03fa591b5f7581e39"}, - {file = "coverage-5.4-cp38-cp38-win_amd64.whl", hash = "sha256:03ed2a641e412e42cc35c244508cf186015c217f0e4d496bf6d7078ebe837ae7"}, - {file = "coverage-5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:14a9f1887591684fb59fdba8feef7123a0da2424b0652e1b58dd5b9a7bb1188c"}, - {file = "coverage-5.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:9564ac7eb1652c3701ac691ca72934dd3009997c81266807aef924012df2f4b3"}, - {file = "coverage-5.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:0f48fc7dc82ee14aeaedb986e175a429d24129b7eada1b7e94a864e4f0644dde"}, - {file = "coverage-5.4-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:107d327071061fd4f4a2587d14c389a27e4e5c93c7cba5f1f59987181903902f"}, - {file = "coverage-5.4-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:0cdde51bfcf6b6bd862ee9be324521ec619b20590787d1655d005c3fb175005f"}, - {file = "coverage-5.4-cp39-cp39-win32.whl", hash = "sha256:c67734cff78383a1f23ceba3b3239c7deefc62ac2b05fa6a47bcd565771e5880"}, - {file = "coverage-5.4-cp39-cp39-win_amd64.whl", hash = "sha256:c669b440ce46ae3abe9b2d44a913b5fd86bb19eb14a8701e88e3918902ecd345"}, - {file = "coverage-5.4-pp36-none-any.whl", hash = "sha256:c0ff1c1b4d13e2240821ef23c1efb1f009207cb3f56e16986f713c2b0e7cd37f"}, - {file = "coverage-5.4-pp37-none-any.whl", hash = "sha256:cd601187476c6bed26a0398353212684c427e10a903aeafa6da40c63309d438b"}, - {file = "coverage-5.4.tar.gz", hash = "sha256:6d2e262e5e8da6fa56e774fb8e2643417351427604c2b177f8e8c5f75fc928ca"}, -] -decorator = [ - {file = "decorator-4.4.2-py2.py3-none-any.whl", hash = "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760"}, - {file = "decorator-4.4.2.tar.gz", hash = "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"}, -] -flake8 = [ - {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, - {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"}, -] -idna = [ - {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, - {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, -] -importlib-metadata = [ - {file = "importlib_metadata-3.4.0-py3-none-any.whl", hash = "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771"}, - {file = "importlib_metadata-3.4.0.tar.gz", hash = "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"}, -] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] -ipdb = [ - {file = "ipdb-0.13.4.tar.gz", hash = "sha256:c85398b5fb82f82399fc38c44fe3532c0dde1754abee727d8f5cfcc74547b334"}, -] -ipython = [ - {file = "ipython-7.16.1-py3-none-any.whl", hash = "sha256:2dbcc8c27ca7d3cfe4fcdff7f45b27f9a8d3edfa70ff8024a71c7a8eb5f09d64"}, - {file = "ipython-7.16.1.tar.gz", hash = "sha256:9f4fcb31d3b2c533333893b9172264e4821c1ac91839500f31bd43f2c59b3ccf"}, -] -ipython-genutils = [ - {file = "ipython_genutils-0.2.0-py2.py3-none-any.whl", hash = "sha256:72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8"}, - {file = "ipython_genutils-0.2.0.tar.gz", hash = "sha256:eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"}, -] -jedi = [ - {file = "jedi-0.18.0-py2.py3-none-any.whl", hash = "sha256:18456d83f65f400ab0c2d3319e48520420ef43b23a086fdc05dff34132f0fb93"}, - {file = "jedi-0.18.0.tar.gz", hash = "sha256:92550a404bad8afed881a137ec9a461fed49eca661414be45059329614ed0707"}, -] -mccabe = [ - {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, - {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, -] -packaging = [ - {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"}, - {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"}, -] -parso = [ - {file = "parso-0.8.1-py2.py3-none-any.whl", hash = "sha256:15b00182f472319383252c18d5913b69269590616c947747bc50bf4ac768f410"}, - {file = "parso-0.8.1.tar.gz", hash = "sha256:8519430ad07087d4c997fda3a7918f7cfa27cb58972a8c89c2a0295a1c940e9e"}, -] -pexpect = [ - {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, - {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, -] -pickleshare = [ - {file = "pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"}, - {file = "pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca"}, -] -pluggy = [ - {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, - {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, -] -prompt-toolkit = [ - {file = "prompt_toolkit-3.0.3-py3-none-any.whl", hash = "sha256:c93e53af97f630f12f5f62a3274e79527936ed466f038953dfa379d4941f651a"}, - {file = "prompt_toolkit-3.0.3.tar.gz", hash = "sha256:a402e9bf468b63314e37460b68ba68243d55b2f8c4d0192f85a019af3945050e"}, -] -ptyprocess = [ - {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, - {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, -] -py = [ - {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, - {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, -] -pycodestyle = [ - {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, - {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, -] -pyflakes = [ - {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, - {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, -] -pygments = [ - {file = "Pygments-2.7.4-py3-none-any.whl", hash = "sha256:bc9591213a8f0e0ca1a5e68a479b4887fdc3e75d0774e5c71c31920c427de435"}, - {file = "Pygments-2.7.4.tar.gz", hash = "sha256:df49d09b498e83c1a73128295860250b0b7edd4c723a32e9bc0d295c7c2ec337"}, -] -pyparsing = [ - {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, - {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, -] -pytest = [ - {file = "pytest-6.2.2-py3-none-any.whl", hash = "sha256:b574b57423e818210672e07ca1fa90aaf194a4f63f3ab909a2c67ebb22913839"}, - {file = "pytest-6.2.2.tar.gz", hash = "sha256:9d1edf9e7d0b84d72ea3dbcdfd22b35fb543a5e8f2a60092dd578936bf63d7f9"}, -] -pytest-cov = [ - {file = "pytest-cov-2.11.1.tar.gz", hash = "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7"}, - {file = "pytest_cov-2.11.1-py2.py3-none-any.whl", hash = "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da"}, -] -requests = [ - {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, - {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, -] -six = [ - {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, - {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, -] -"tap.py" = [ - {file = "tap.py-3.0-py2.py3-none-any.whl", hash = "sha256:a598bfaa2e224d71f2e86147c2ef822c18ff2e1b8ef006397e5056b08f92f699"}, - {file = "tap.py-3.0.tar.gz", hash = "sha256:f5eeeeebfd64e53d32661752bb4c288589a3babbb96db3f391a4ec29f1359c70"}, -] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, -] -traitlets = [ - {file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"}, - {file = "traitlets-4.3.3.tar.gz", hash = "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"}, -] -typing-extensions = [ - {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, - {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, - {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, -] -urllib3 = [ - {file = "urllib3-1.26.3-py2.py3-none-any.whl", hash = "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80"}, - {file = "urllib3-1.26.3.tar.gz", hash = "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73"}, -] -wcwidth = [ - {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, - {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, -] -yapf = [ - {file = "yapf-0.30.0-py2.py3-none-any.whl", hash = "sha256:3abf61ba67cf603069710d30acbc88cfe565d907e16ad81429ae90ce9651e0c9"}, - {file = "yapf-0.30.0.tar.gz", hash = "sha256:3000abee4c28daebad55da6c85f3cd07b8062ce48e2e9943c8da1b9667d48427"}, -] -zipp = [ - {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, - {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, -] +appnope = [] +attrs = [] +backcall = [] +bleach = [] +certifi = [] +cffi = [] +charset-normalizer = [] +click = [] +click-log = [] +colorama = [] +coverage = [] +cryptography = [] +decorator = [] +docutils = [] +dotty-dict = [] +exceptiongroup = [] +flake8 = [] +gitdb = [] +gitpython = [] +idna = [] +importlib-metadata = [] +importlib-resources = [] +iniconfig = [] +invoke = [] +ipdb = [] +ipython = [] +"jaraco.classes" = [] +jedi = [] +jeepney = [] +keyring = [] +matplotlib-inline = [] +mccabe = [] +more-itertools = [] +mslex = [] +packaging = [] +parso = [] +pexpect = [] +pickleshare = [] +pkginfo = [] +pluggy = [] +prompt-toolkit = [] +psutil = [] +ptyprocess = [] +pycodestyle = [] +pycparser = [] +pyflakes = [] +pygments = [] +pytest = [] +pytest-cov = [] +python-gitlab = [] +python-semantic-release = [] +pywin32-ctypes = [] +readme-renderer = [] +requests = [] +requests-toolbelt = [] +rfc3986 = [] +secretstorage = [] +semver = [] +six = [] +smmap = [] +"tap.py" = [] +taskipy = [] +toml = [] +tomli = [] +tomlkit = [] +tqdm = [] +traitlets = [] +twine = [] +typing-extensions = [] +urllib3 = [] +wcwidth = [] +webencodings = [] +yapf = [] +zipp = [] diff --git a/pyproject.toml b/pyproject.toml index 1711ecb..241a341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,20 @@ [tool.poetry] name = "logdna" -version = "1.5.4" -description = "A python package for sending logs" +version = "1.18.12" +description = 'A Python Package for Sending Logs to LogDNA' authors = ["logdna "] license = "MIT" +[tool.semantic_release] +version_toml = "pyproject.toml:tool.poetry.version" +version_pattern = "logdna/VERSION:(\\d+\\.\\d+\\.\\d+)" +branch = "master" +commit_subject = "release: Version {version} [skip ci]" +commit_author = "LogDNA Bot " + [tool.poetry.dependencies] -python = "^3.6" -requests = "^2.25.1" +python = "^3.7" +requests = "^2.28.1" [tool.poetry.dev-dependencies] coverage = "^5.4" @@ -15,14 +22,39 @@ coverage = "^5.4" ipdb = "^0.13.4" flake8 = "^3.8.4" yapf = "^0.30.0" -pytest = "^6.2.2" +pytest = "^7.2.0" pytest-cov = "^2.11.1" +taskipy = "^1.6.0" +python-semantic-release = "^7.28.1" -[tool.poetry.scripts] -test = "scripts.test:run" -lint = "scripts.lint:run" -"lint:fix" = "scripts.lint:fix" +[tool.taskipy.tasks] +pre_test = "mkdir -p coverage" +test = "pytest --junitxml=coverage/test.xml --cov=logdna --cov-report=html --verbose tests/" +post_test = "python scripts/json_coverage.py" +lint = "flake8 --doctests" +"lint:fix" = "yapf -r -i logdna scripts tests" +"post_lint:fix" = "task lint" +release = "semantic-release publish" [build-system] requires = ["poetry>=0.12"] build-backend = "poetry.masonry.api" + +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = "tests" + +[tool.coverage.run] +branch = true +source = ["logdna"] + +[tool.coverage.report] +fail_under = 76 +show_missing = true + +[tool.coverage.json] +output = "coverage/coverage.json" + +[tool.coverage.html] +directory = "coverage" +show_contexts = true diff --git a/scripts/json_coverage.py b/scripts/json_coverage.py new file mode 100644 index 0000000..563d123 --- /dev/null +++ b/scripts/json_coverage.py @@ -0,0 +1,51 @@ +import json +from os import path +from coverage import Coverage + +ROOT = path.realpath(path.abspath(path.join(path.dirname(__file__), '..'))) +COVERAGE_DIR = path.join(ROOT, 'coverage') +JUNIT_PATH = path.join(COVERAGE_DIR, 'test.xml') + + +def json_coverage(): + COVERAGE_FILE = path.join(COVERAGE_DIR, 'coverage-final.json') + COVERAGE_SUMMARY = path.join(COVERAGE_DIR, 'coverage-summary.json') + + coverage = Coverage(config_file=path.join(ROOT, 'pyproject.toml')) + coverage.load() + coverage.json_report(outfile=COVERAGE_FILE) + + report = json.load(open(COVERAGE_FILE)) + totals = report.get('totals') + summary = { + 'lines': { + 'total': + totals['covered_lines'] + totals['missing_lines'], + 'covered': + totals['covered_lines'], + 'pct': + totals['covered_lines'] / + (totals['covered_lines'] + totals['missing_lines']) * 100 + }, + 'statements': { + 'total': None, + 'covered': None, + 'pct': None, + }, + 'functions': { + 'total': None, + 'covered': None, + 'pct': None, + }, + 'branches': { + 'total': totals['num_branches'], + 'covered': totals['covered_branches'], + 'pct': totals['covered_branches'] / totals['num_branches'] * 100 + } + } + + json.dump({'total': summary}, open(COVERAGE_SUMMARY, 'w')) + + +if __name__ == "__main__": + json_coverage() diff --git a/scripts/lint.py b/scripts/lint.py deleted file mode 100644 index 87bc65f..0000000 --- a/scripts/lint.py +++ /dev/null @@ -1,20 +0,0 @@ -from os import path -from subprocess import check_call - -ROOT = path.realpath(path.abspath(path.join(path.dirname(__file__), '..'))) - - -def run(): - check_call(['flake8', '--doctests']) - - -def fix(): - check_call(cwd=ROOT, - args=[ - 'yapf', '-r', '-i', - path.join(ROOT, 'logdna'), - path.join(ROOT, 'scripts'), - path.join(ROOT, 'tests') - ]) - - run() diff --git a/scripts/test.py b/scripts/test.py deleted file mode 100644 index 7c95bf4..0000000 --- a/scripts/test.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -from os import path -from subprocess import check_call - -ROOT = path.realpath(path.abspath(path.join(path.dirname(__file__), '..'))) -COVERAGE_DIR = path.join(ROOT, 'coverage') -JUNIT_PATH = path.join(COVERAGE_DIR, 'test.xml') -print(JUNIT_PATH) - - -def run(): - try: - os.mkdir(COVERAGE_DIR) - except FileExistsError: - pass - - check_call([ - 'pytest', - '--junitxml={0}'.format(JUNIT_PATH), - '--cov=logdna', - '--cov-report=html', - '--cov-report=xml', - '--cov-config=setup.cfg', - '--cov-branch' - ]) diff --git a/setup.cfg b/setup.cfg index 92b7bc5..c24cd82 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ description-file = README.md [flake8] -exclude = .git,__pycache__,docs/source/conf.py,old,build,dist +exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,.cache,pip,pypoetry max-complexity = 10 [yapf] @@ -10,21 +10,3 @@ based_on_style = pep8 indent_width = 4 use_tabs = False -[tool:pytest] -minversion = 6.0 -testpaths = tests - -[coverage:run] -branch = True -source = logdna/* - -[coverage:html] -directory = coverage -show_contexts = True - -[coverage:xml] -output = coverage/coverage.xml - -[coverage:report] -fail_under = 67 - diff --git a/setup.py b/setup.py index 58289e2..fc16e48 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ from setuptools import setup -from os import path +from os import path, sep # read the contents of your README file this_directory = path.abspath(path.dirname(__file__)) @@ -7,7 +7,8 @@ with open(path.join(this_directory, 'README.md'), 'rb') as f: long_description = f.read().decode('utf-8') -with open("%s/logdna/VERSION" % this_directory) as f: +kwargs = {"dir": this_directory, "sep": sep} +with open("{dir}{sep}logdna{sep}VERSION".format(**kwargs)) as f: version = f.read().strip('\n') setup( @@ -25,7 +26,14 @@ install_requires=[ 'requests', ], - classifiers=[], + classifiers=[ + 'Topic :: System :: Logging', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9' + ], long_description=long_description, long_description_content_type='text/markdown', ) diff --git a/tests/mock/__init.__.py b/tests/mock/__init.__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/mock/log.py b/tests/mock/log.py deleted file mode 100644 index d9999e0..0000000 --- a/tests/mock/log.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging -from threading import Thread - -__all__ = ['logger', 'info', 'LOGDNA_API_KEY'] -LOGDNA_API_KEY = '< YOUR INGESTION KEY HERE >' -logger = logging.getLogger('logdna') -logger.setLevel(logging.INFO) - - -def info(*args): - def fn(): - for line in args: - logger.info(line) - - thread = Thread(target=fn) - thread.setDaemon(True) - thread.start() - return thread diff --git a/tests/mock/server.py b/tests/mock/server.py deleted file mode 100644 index c0a1791..0000000 --- a/tests/mock/server.py +++ /dev/null @@ -1,19 +0,0 @@ -import socket -from http.server import HTTPServer -from threading import Thread - - -def get_port(): - s = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM) - s.bind(('localhost', 0)) - _, port = s.getsockname() - s.close() - return port - - -def start_server(port, cls): - server = HTTPServer(('localhost', port), cls) - thread = Thread(target=server.handle_request) - thread.setDaemon(True) - thread.start() - return thread diff --git a/tests/test_logger.py b/tests/test_logger.py index 3c5fa8d..0a610c2 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,112 +1,362 @@ -import json +import logging import unittest +import requests +import time +import os -from http.server import BaseHTTPRequestHandler from logdna import LogDNAHandler -from .mock.server import get_port, start_server -from .mock.log import logger, info, LOGDNA_API_KEY +from concurrent.futures import ThreadPoolExecutor +from logdna.configs import defaults +from unittest import mock +from unittest.mock import patch +now = int(time.time()) expectedLines = [] +LOGDNA_API_KEY = os.environ.get('LOGDNA_INGESTION_KEY') +logger = logging.getLogger('logdna') +logger.setLevel(logging.INFO) +sample_args = { + 'app': 'differentTest', + 'level': 'debug', + 'hostname': 'differentHost', + 'env': 'differentEnv' +} +sample_record = logging.LogRecord('test', logging.INFO, 'test', 5, + 'Something to test', [sample_args], '', '', + '') +sample_message = { + 'line': 'Something to test', + 'hostname': 'differentHost', + 'level': 'debug', + 'app': 'differentTest', + 'env': 'differentEnv', + 'meta': { + 'args': sample_args, + 'name': 'test', + 'pathname': 'test', + 'lineno': 5 + } +} -class SuccessfulRequestHandler(BaseHTTPRequestHandler): - def do_POST(self): - content_length = int(self.headers['Content-Length']) - body = self.rfile.read(content_length) - self.send_response(200) +sample_options = { + 'hostname': 'localhost', + 'ip': '10.0.1.1', + 'mac': 'C0:FF:EE:C0:FF:EE', + 'tags': 'sample,test', + 'index_meta': True, + 'now': int(time.time() * 1000), + 'retry_interval_secs': 0.5 +} - self.end_headers() - body = json.loads(body)['ls'] - for keys in body: - expectedLines.append(keys['line']) +class MockThreadPoolExecutor(): + def __init__(self, **kwargs): + pass -class FailedRequestHandler(BaseHTTPRequestHandler): - def do_POST(self): - content_length = int(self.headers['Content-Length']) - self.rfile.read(content_length) - self.send_response(400) - self.end_headers() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + pass + + def submit(self, fn, *args, **kwargs): + # execute functions in series without creating threads + # for easier unit testing + result = fn(*args, **kwargs) + return result + + def shutdown(self, wait=True): + pass class LogDNAHandlerTest(unittest.TestCase): - def server_recieves_messages(self): - port = get_port() - options = { - 'hostname': 'localhost', - 'url': 'http://localhost:{0}'.format(port), - 'ip': '10.0.1.1', - 'mac': 'C0:FF:EE:C0:FF:EE' - } - - handler = LogDNAHandler(LOGDNA_API_KEY, options) - logger.addHandler(handler) - line = "python python python" - - server_thread = start_server(port, SuccessfulRequestHandler) - logdna_thread = info(line) - - server_thread.join() - logdna_thread.join() - - self.assertEqual(len(expectedLines), 1) - self.assertIn(line, expectedLines) - logger.removeHandler(handler) - - def messages_preserved_if_excp(self): - port = get_port() - options = { - 'hostname': 'localhost', - 'url': 'http://localhost:{0}'.format(port), - 'ip': '10.0.1.1', - 'mac': 'C0:FF:EE:C0:FF:EE' - } - - handler = LogDNAHandler(LOGDNA_API_KEY, options) - logger.addHandler(handler) - line = "second test. server fails" - - server_thread = start_server(port, FailedRequestHandler) - logdna_thread = info(line) - - server_thread.join() - logdna_thread.join() - - self.assertEqual(len(handler.buf), 1) - logger.removeHandler(handler) - - def stops_retention_when_buf_is_full(self): - port = get_port() - options = { - 'hostname': 'localhost', - 'url': 'http://localhost:{0}'.format(port), - 'ip': '10.0.1.1', - 'mac': 'C0:FF:EE:C0:FF:EE', - 'buf_retention_limit': 50, - 'equest_timeout': 10, - 'flush_interval': 1, - 'retry_interval_secs': 1 - } - - handler = LogDNAHandler(LOGDNA_API_KEY, options) - logger.addHandler(handler) - line = "when buffer grows bigger than we want" - lineTwo = "when buffer grows bigger than we want. And more and more" - - server_thread = start_server(port, FailedRequestHandler) - logdna_thread = info(line, lineTwo) - - server_thread.join() - logdna_thread.join() - - self.assertEqual(len(handler.buf), 1) - self.assertNotEqual(handler.buf[0]['line'], lineTwo) - logger.removeHandler(handler) - - def test_run_tests(self): - self.server_recieves_messages() - self.messages_preserved_if_excp() - self.stops_retention_when_buf_is_full() + def test_handler(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + self.assertIsInstance(handler, logging.Handler) + self.assertIsInstance( + handler.internal_handler, logging.StreamHandler) + self.assertIsNotNone(handler.internalLogger) + self.assertEqual(handler.key, LOGDNA_API_KEY) + self.assertEqual(handler.hostname, sample_options['hostname']) + self.assertEqual(handler.ip, sample_options['ip']) + self.assertEqual(handler.mac, sample_options['mac']) + self.assertEqual(handler.loglevel, 'info') + self.assertEqual(handler.app, '') + self.assertEqual(handler.env, '') + self.assertEqual(handler.tags, sample_options['tags'].split(',')) + self.assertEqual(handler.custom_fields, defaults['META_FIELDS']) + + # Set the Connection Variables + self.assertEqual(handler.url, defaults['LOGDNA_URL']) + self.assertEqual(handler.request_timeout, + defaults['DEFAULT_REQUEST_TIMEOUT']) + self.assertEqual(handler.user_agent, defaults['USER_AGENT']) + self.assertEqual(handler.max_retry_attempts, + defaults['MAX_RETRY_ATTEMPTS']) + self.assertEqual(handler.max_retry_jitter, + defaults['MAX_RETRY_JITTER']) + self.assertEqual(handler.max_concurrent_requests, + defaults['MAX_CONCURRENT_REQUESTS']) + self.assertEqual(handler.retry_interval_secs, + sample_options['retry_interval_secs']) + + # Set the Flush-related Variables + self.assertEqual(handler.buf, []) + self.assertEqual(handler.buf_size, 0) + self.assertIsNone(handler.flusher) + self.assertTrue(handler.index_meta) + self.assertEqual(handler.flush_limit, defaults['FLUSH_LIMIT']) + self.assertEqual(handler.flush_interval_secs, + defaults['FLUSH_INTERVAL_SECS']) + self.assertEqual(handler.buf_retention_limit, + defaults['BUF_RETENTION_LIMIT']) + + # Set up the Thread Pools + self.assertIsInstance( + handler.worker_thread_pool, ThreadPoolExecutor) + self.assertIsInstance( + handler.request_thread_pool, ThreadPoolExecutor) + self.assertEqual(handler.level, logging.DEBUG) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_flusher(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + post_mock.return_value = r + handler.emit(sample_record) + self.assertIsNotNone(handler.flusher) + handler.close_flusher() + self.assertIsNone(handler.flusher) + + def test_emit(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + handler.buffer_log = unittest.mock.Mock() + handler.emit(sample_record) + sample_message['timestamp'] = unittest.mock.ANY + handler.buffer_log.assert_called_once_with(sample_message) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_lock_and_do_flush_request(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + test_buf = handler.buf.copy() + handler.try_lock_and_do_flush_request() + post_mock.assert_called_with( + url=handler.url, + json={ + 'e': 'ls', + 'ls': test_buf + }, + params={ + 'hostname': handler.hostname, + 'ip': handler.ip, + 'mac': handler.mac, + 'tags': handler.tags, + 'now': int(now * 1000) + }, + stream=True, + allow_redirects=True, + timeout=handler.request_timeout, + headers={ + 'user-agent': handler.user_agent, + 'apikey': LOGDNA_API_KEY}) + self.assertTrue(post_mock.call_count, 1) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_500(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 500 + r.reason = 'Internal Server Error' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_502(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 502 + r.reason = 'Bad Gateway' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_504(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 504 + r.reason = 'Gateway Timeout' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_429(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 429 + r.reason = 'Too Many Requests' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 3) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_403(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 403 + r.reason = 'Forbidden' + post_mock.return_value = r + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 1) + + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_try_request_403_log_response(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 403 + r.reason = 'Forbidden' + post_mock.return_value = r + sample_options['log_error_response'] = True + sample_message['timestamp'] = unittest.mock.ANY + handler.buf = [sample_message] + handler.try_request([]) + self.assertTrue(post_mock.call_count, 1) + + def test_close(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + close_flusher_mock = unittest.mock.Mock() + close_flusher_mock.side_effect = handler.close_flusher + handler.schedule_flush_sync = unittest.mock.Mock() + handler.close_flusher = close_flusher_mock + handler.close() + handler.close_flusher.assert_called_once_with() + handler.schedule_flush_sync.assert_called_once_with( + should_block=True) + self.assertIsNone(handler.worker_thread_pool) + self.assertIsNone(handler.request_thread_pool) + + # These should be separate objects, since there is already + # a variable in the base class named self.lock. We want + # to make sure that a separate lock is created for the + # locking semantics of the LogDNA Handler + def test_lock_var_separate_from_local_lock_var(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + self.assertIsNotNone(handler) + + # Test that we did not replace the base class' instance var. + self.assertIsNotNone(handler._lock) + self.assertIsNotNone(handler.lock) + self.assertNotEquals(handler.lock, handler._lock) + + def test_flush(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + handler.worker_thread_pool = MockThreadPoolExecutor() + handler.request_thread_pool = MockThreadPoolExecutor() + handler.buf = [sample_message] + handler.buf_size += len(handler.buf) + handler.try_request = unittest.mock.Mock() + handler.flush() + handler.try_request.assert_called_once_with([sample_message]) + + def test_buffer_log(self): + with patch('requests.post') as post_mock: + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + post_mock.return_value = r + handler.worker_thread_pool = MockThreadPoolExecutor() + handler.request_thread_pool = MockThreadPoolExecutor() + handler.flush = unittest.mock.Mock() + sample_message['timestamp'] = now + handler.flush_limit = 0 + handler.buffer_log(sample_message) + handler.flush.assert_called_once_with() + self.assertEqual(handler.buf, [sample_message]) + self.assertEqual(handler.buf_size, len(sample_message['line'])) + + # Attempts to reproduce the specific scenario that resulted in + # https://mezmo.atlassian.net/browse/LOG-15414 where log messages + # would be dropped due to race conditions. The test essentially + # does the following: + # 1. Create a LogDNAHandler + # 2. Call handler.emit() with a large number of log records at a rate + # sufficiently high to trigger the race + # 3. Verify that no log records are dropped. + # + # This test is not deterministic, but it should be sufficient to + # catch regressions. It reliably reproduces the issue in question + # and fails with the previous version of this code. + @mock.patch('time.time', unittest.mock.MagicMock(return_value=now)) + def test_when_emitManyLogs_then_noLogsDropped(self): + num_logs = 10**5 + received = list() + + def append_received(json=None, **kwargs): + ids = [int(log['line']) for log in json['ls']] + for id in ids: + received.append(id) + r = requests.Response() + r.status_code = 200 + r.reason = 'OK' + # Simulate some reasonable request latency + time.sleep(0.1) + return r + + def get_sample_record(id): + return logging.LogRecord( + name='test', + level=logging.INFO, + pathname='test', + lineno=5, + msg=str(id), + args=[sample_args], + exc_info='', + func='', + sinfo='') + + with patch('requests.post', side_effect=append_received): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + for i in range(num_logs): + handler.emit(get_sample_record(i)) + handler.close() + + self.assertEqual(len(received), num_logs) + self.assertEqual(set(received), set(range(num_logs))) + + def test_when_handlerShutDown_then_handlerDoesNotHang(self): + handler = LogDNAHandler(LOGDNA_API_KEY, sample_options) + self.assertIsNotNone(handler) + # Do nothing. This test should pass by virtue of not hanging. if __name__ == '__main__': diff --git a/tests/test_utils.py b/tests/test_utils.py index 59fee87..446840f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,9 @@ import unittest from unittest.mock import patch -from logdna.utils import is_jsonable, sanitize_meta, get_ip +from logdna.utils import is_jsonable +from logdna.utils import sanitize_meta +from logdna.utils import get_ip +from logdna.utils import normalize_list_option IP = '10.0.50.10' VIP = '10.1.60.20' @@ -25,11 +28,11 @@ def setUp(self): self.invalid = {'bar': 'foo', 'baz': set()} def test_sanitize_simple(self): - clean = sanitize_meta(self.valid) + clean = sanitize_meta(self.valid, True) self.assertDictEqual(clean, self.valid) def test_sanitize_complex(self): - clean = sanitize_meta(self.invalid) + clean = sanitize_meta(self.invalid, True) self.assertDictEqual(clean, { 'bar': 'foo', '__errors': 'These keys have been sanitized: baz' @@ -42,9 +45,17 @@ def test_get_ip_socket_error(self, _): self.assertEqual(get_ip(), '127.0.0.1', 'default to localhost on error') - @patch( - 'socket.socket', - **{'return_value.getsockname.return_value': [IP, VIP]} - ) + @patch('socket.socket', + **{'return_value.getsockname.return_value': [IP, VIP]}) def test_get_ip_default(self, _): self.assertEqual(get_ip(), IP, 'default to localhost on error') + + +class NormalizeListOptionTest(unittest.TestCase): + def test_normalize_simple(self): + value1 = normalize_list_option({'tags': ' a, b'}, 'tags') + value2 = normalize_list_option({'tags': ['a', 'b']}, 'tags') + value3 = normalize_list_option({'tags': ('a', 'b')}, 'tags') + self.assertEqual(value1, ['a', 'b']) + self.assertEqual(value1, value2) + self.assertEqual(value3, [])