diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f572791..0ddbe65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,11 @@ # https://docs.github.com/en/actions/reference # https://github.com/actions # +# uses: https://github.com/actions/checkout @v3 +# uses: https://github.com/actions/setup-python @v4 +# uses: https://github.com/actions/download-artifact @v3 +# uses: https://github.com/actions/upload-artifact @v3 +# name: CI @@ -11,8 +16,8 @@ on: jobs: - pylint: - name: "PyLint" + check: + name: "Check" runs-on: ubuntu-latest strategy: matrix: @@ -23,17 +28,16 @@ jobs: uses: actions/checkout@v3 - name: "Setup Python ${{matrix.test.PY}}" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{matrix.test.PY}} - - name: "Install tox" - run: python -m pip -q install tox + - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check - - name: "Run checker" + - name: "Test" env: TOXENV: ${{matrix.test.TOXENV}} - run: python -m tox -r + run: python3 -m tox -r database: name: "Python ${{matrix.test.PY}} + PostgreSQL ${{matrix.test.PG}}" @@ -41,6 +45,9 @@ jobs: strategy: matrix: test: + - {PY: "3.7", PG: "11", TOXENV: "py37"} + - {PY: "3.8", PG: "12", TOXENV: "py38"} + - {PY: "3.9", PG: "13", TOXENV: "py39"} - {PY: "3.10", PG: "14", TOXENV: "py310"} - {PY: "3.11", PG: "15", TOXENV: "py311"} steps: @@ -48,19 +55,17 @@ jobs: uses: actions/checkout@v3 - name: "Setup Python ${{matrix.test.PY}}" - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{matrix.test.PY}} - - name: "Install tox" - run: | - python -m pip -q install tox + - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check - name: "InstallDB" run: | echo "::group::apt-get-update" sudo -nH apt-get -q update - sudo -nH apt install curl ca-certificates gnupg + sudo -nH apt-get -q install curl ca-certificates gnupg curl https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | gpg --dearmor \ | sudo -nH tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg @@ -75,13 +80,30 @@ jobs: sudo -nH mkdir -p /etc/postgresql-common/createcluster.d echo "create_main_cluster = false" | sudo -nH tee /etc/postgresql-common/createcluster.d/no-main.conf - sudo -nH apt-get -qyu install postgresql-${{matrix.test.PG}} pgqd postgresql-${{matrix.test.PG}}-pgq-node + sudo -nH apt-get -qyu install postgresql-${{matrix.test.PG}} postgresql-server-dev-${{matrix.test.PG}} pgqd echo "::endgroup::" # tune environment echo "/usr/lib/postgresql/${{matrix.test.PG}}/bin" >> $GITHUB_PATH echo "PGHOST=/tmp" >> $GITHUB_ENV + - name: "Install extensions" + run: | + echo "::group::install-pgq" + git clone -q https://github.com/pgq/pgq pgq-sql; make -C pgq-sql + sudo -nH bash -c "PATH='${PATH}' make install -C pgq-sql" + echo "::endgroup::" + + echo "::group::install-pgq-node" + git clone -q https://github.com/pgq/pgq-node; make -C pgq-node + sudo -nH bash -c "PATH='${PATH}' make install -C pgq-node" + echo "::endgroup::" + + echo "::group::install-londiste" + git clone -q https://github.com/pgq/londiste-sql; make -C londiste-sql + sudo -nH bash -c "PATH='${PATH}' make install -C londiste-sql" + echo "::endgroup::" + - name: "StartDB" run: | rm -rf data log @@ -103,7 +125,7 @@ jobs: TEST_Q_NAME: testq PGDATABASE: testdb run: | - python -m tox -r -- --color=yes + python3 -m tox -r -- --color=yes - name: "StopDB" run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e07e7e6..c05f7fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,5 @@ # -# https://docs.github.com/en/actions/reference -# https://github.com/actions +# This runs when version tag is pushed # name: REL @@ -10,72 +9,75 @@ on: tags: ["v[0-9]*"] jobs: - release: - name: Release + sdist: + name: "Build source package" runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: {python-version: "3.11"} + - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check + - run: python3 setup.py sdist + - run: python3 setup.py bdist_wheel + - uses: actions/upload-artifact@v3 + with: {name: "dist", path: "dist"} - - name: Checkout code - id: checkout - uses: actions/checkout@v3 + publish: + name: "Publish" + runs-on: ubuntu-latest + needs: [sdist] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: {python-version: "3.11"} + + - run: python3 -m pip install -r etc/requirements.build.txt --disable-pip-version-check + + - name: "Get files" + uses: actions/download-artifact@v3 + with: {name: "dist", path: "dist"} - - name: "Setup Python" - uses: actions/setup-python@v2 - with: - python-version: "3.10" + - name: "Install pandoc" + run: | + sudo -nH apt-get -u -y install pandoc + pandoc --version - - name: Build tarball - id: build + - name: "Prepare" run: | - python -m pip install --disable-pip-version-check -U setuptools wheel - PACKAGE=$(python setup.py --name) - VERSION=$(python setup.py --version) + PACKAGE=$(python3 setup.py --name) + VERSION=$(python3 setup.py --version) TGZ="${PACKAGE}-${VERSION}.tar.gz" + # default - gh:release, pypi # PRERELEASE - gh:prerelease, pypi # DRAFT - gh:draft,prerelease, testpypi - PRERELEASE="false" - DRAFT="false" - if echo "${VERSION}" | grep -qE '(a|b|rc)'; then PRERELEASE="true"; fi - if echo "${VERSION}" | grep -qE '(dev)'; then DRAFT="true"; PRERELEASE="true"; fi + PRERELEASE="false"; DRAFT="false" + case "${VERSION}" in + *[ab]*|*rc*) PRERELEASE="true";; + *dev*) PRERELEASE="true"; DRAFT="true";; + esac + test "${{github.ref}}" = "refs/tags/v${VERSION}" || { echo "ERR: tag mismatch"; exit 1; } - python setup.py sdist test -f "dist/${TGZ}" || { echo "ERR: sdist failed"; exit 1; } - python -m pip wheel --disable-pip-version-check -w dist dist/${TGZ} - rm -f dist/skytools* echo "PACKAGE=${PACKAGE}" >> $GITHUB_ENV echo "VERSION=${VERSION}" >> $GITHUB_ENV echo "TGZ=${TGZ}" >> $GITHUB_ENV echo "PRERELEASE=${PRERELEASE}" >> $GITHUB_ENV echo "DRAFT=${DRAFT}" >> $GITHUB_ENV - sudo -nH apt-get -u -y install pandoc - pandoc --version mkdir -p tmp make -s shownote > tmp/note.md cat tmp/note.md + ls -l dist - - name: "Create release" - id: github_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - with: - tag_name: ${{github.ref}} - release_name: ${{env.PACKAGE}} v${{env.VERSION}} - body_path: tmp/note.md - prerelease: ${{env.PRERELEASE}} - draft: ${{env.DRAFT}} - - - name: "Upload to Github" - id: github_upload - uses: actions/upload-release-asset@v1 + - name: "Create Github release" env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - with: - upload_url: ${{steps.github_release.outputs.upload_url}} - asset_path: dist/${{env.TGZ}} - asset_name: ${{env.TGZ}} - asset_content_type: application/x-gzip + GH_TOKEN: ${{secrets.GITHUB_TOKEN}} + run: | + title="${PACKAGE} v${VERSION}" + ghf="--notes-file=./tmp/note.md" + if test "${DRAFT}" = "true"; then ghf="${ghf} --draft"; fi + if test "${PRERELEASE}" = "true"; then ghf="${ghf} --prerelease"; fi + gh release create "v${VERSION}" "dist/${TGZ}" --title="${title}" ${ghf} - name: "Upload to PYPI" id: pypi_upload @@ -83,7 +85,6 @@ jobs: PYPI_TOKEN: ${{secrets.PYPI_TOKEN}} PYPI_TEST_TOKEN: ${{secrets.PYPI_TEST_TOKEN}} run: | - pip install --disable-pip-version-check -U twine ls -l dist if test "${DRAFT}" = "false"; then python -m twine upload -u __token__ -p ${PYPI_TOKEN} \ diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 4c69adb..0000000 --- a/.pylintrc +++ /dev/null @@ -1,517 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS,tmp,dist - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=bare-except, - broad-except, - consider-using-in, - consider-using-ternary, - fixme, - global-statement, - invalid-name, - missing-docstring, - no-else-raise, - no-else-return, - trailing-newlines, - unused-argument, - unused-variable, - using-constant-test, - useless-object-inheritance, - duplicate-code, - consider-using-f-string, - unused-private-member, - - arguments-differ, - multiple-statements, - len-as-condition, - chained-comparison, - unnecessary-pass, - cyclic-import, - invalid-name, - too-many-ancestors, - import-outside-toplevel, - protected-access, - try-except-raise, - deprecated-module, - no-else-break, - no-else-continue - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=colorized - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=no - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=10 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, while `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package.. -#spelling-dict=en_US - -# List of comma separated words that should not be checked. -spelling-ignore-words=usr,bin,env - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file=.local.dict - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[STRING] - -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. -check-str-concat-over-line-jumps=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format=LF - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=180 - -# Maximum number of lines in a module. -max-module-lines=10000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=15 - -# Maximum number of attributes for a class (see R0902). -max-attributes=37 - -# Maximum number of boolean expressions in an if statement. -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=50 - -# Maximum number of locals for function / method body. -max-locals=45 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=420 - -# Maximum number of return / yield for function / method body. -max-returns=16 - -# Maximum number of statements in function / method body. -max-statements=150 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=0 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception diff --git a/Makefile b/Makefile index f88bf15..ec9654a 100644 --- a/Makefile +++ b/Makefile @@ -10,12 +10,18 @@ clean: rm -rf build *.egg-info */__pycache__ tests/*.pyc rm -rf .pybuild MANIFEST +lint: + tox -q -e lint + xlint: - tox -e xlint + tox -q -e xlint xclean: clean rm -rf .tox dist +sdist: + python3 setup.py -q sdist + test: PGDATABASE=testdb TEST_Q_NAME=testq PGHOST=/tmp PGPORT=5120 tox -e py38 diff --git a/NEWS.rst b/NEWS.rst index 4096875..895beab 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1,6 +1,23 @@ NEWS ==== +pgq 3.8 +------- + +* build: convert to pyproject.toml +* typing: add full typing +* ci: drop unmaintained actions + +pgq 3.7.3 +--------- + +Fixes: + +* worker: another refactor for wait-behind - use dedicated code path for it, + following main path makes things too complicated. Fixes the problem of + missing node location events. +* nodeinfo: set more fields on dead node + pgq 3.7.2 --------- diff --git a/etc/requirements.build.txt b/etc/requirements.build.txt new file mode 100644 index 0000000..ff9e40b --- /dev/null +++ b/etc/requirements.build.txt @@ -0,0 +1,4 @@ +setuptools>=67 +wheel>=0.41 +twine==4.0.2 +tox==4.8.0 diff --git a/pgq/__init__.py b/pgq/__init__.py index c178d3c..d2a9e80 100644 --- a/pgq/__init__.py +++ b/pgq/__init__.py @@ -4,6 +4,7 @@ from pgq.cascade.consumer import CascadedConsumer from pgq.cascade.nodeinfo import MemberInfo, NodeInfo, QueueInfo from pgq.cascade.worker import CascadedWorker +from pgq.baseconsumer import EventList, BatchInfo from pgq.consumer import Consumer from pgq.coopconsumer import CoopConsumer from pgq.event import Event @@ -17,8 +18,9 @@ 'bulk_insert_events', 'insert_event', 'RemoteConsumer', 'SerialConsumer', 'PGQStatus', 'CascadeAdmin', 'CascadedConsumer', 'CascadedWorker', - 'MemberInfo', 'NodeInfo', 'QueueInfo' + 'MemberInfo', 'NodeInfo', 'QueueInfo', 'EventList', + 'BatchInfo', ] -__version__ = '3.7.2' +__version__ = '3.8' diff --git a/pgq/baseconsumer.py b/pgq/baseconsumer.py index 2bb34d9..a468eae 100644 --- a/pgq/baseconsumer.py +++ b/pgq/baseconsumer.py @@ -212,7 +212,7 @@ def reload(self) -> None: # filter out specific tables only tfilt = [] - for t in self.cf.getlist('table_filter', ''): + for t in self.cf.getlist('table_filter', []): tfilt.append(skytools.quote_literal(skytools.fq_name(t))) if len(tfilt) > 0: expr = "ev_extra1 in (%s)" % ','.join(tfilt) @@ -254,7 +254,7 @@ def process_batch(self, db: Connection, batch_id: int, event_list: EventList) -> for ev in event_list: self.process_event(db, ev) - def work(self) -> int: + def work(self) -> Optional[int]: """Do the work loop, once (internal). Returns: true if wants to be called again, false if script can sleep. diff --git a/pgq/cascade/admin.py b/pgq/cascade/admin.py index 55078b4..086d2ef 100644 --- a/pgq/cascade/admin.py +++ b/pgq/cascade/admin.py @@ -584,7 +584,7 @@ def change_provider(self, node: str = '', consumer: str = '', new_provider: str cinfo = cmap[consumer] # is it node worker or plain consumer? - is_worker = (ninfo.worker_name == consumer) + is_worker = ninfo.worker_name == consumer # fixme: expect the node to be described already q = "select * from pgq_node.register_location(%s, %s, %s, false)" @@ -967,7 +967,7 @@ def find_subscribers_for(self, parent_node_name: str) -> List[str]: res[n.name] = 1 return list(sorted(res.keys())) - def cmd_tag_dead(self, dead_node_name) -> None: + def cmd_tag_dead(self, dead_node_name: str) -> None: queue_info = self.load_local_info() # tag node dead in memory @@ -1408,7 +1408,7 @@ def resurrect_process_lost_events(self, db: Connection, failover_tick: int) -> D _json_dump_file = None - def resurrect_dump_event(self, ev: DictRow, stats: Dict[str, int], batch_info: DictRow) -> None: + def resurrect_dump_event(self, ev: DictRow, stats: Dict[str, Any], batch_info: DictRow) -> None: if self._json_dump_file is None: self._json_dump_file = open(RESURRECT_DUMP_FILE, "w", encoding="utf8") # pylint: disable=consider-using-with sep = '[' @@ -1555,7 +1555,7 @@ def unsubscribe_node(self, target_node: str, subscriber_node: str) -> None: _node_cache: Dict[str, Optional[NodeInfo]] = {} - def get_node_info_opt(self, node_name) -> Optional[NodeInfo]: + def get_node_info_opt(self, node_name: str) -> Optional[NodeInfo]: """Cached node info lookup.""" if node_name in self._node_cache: return self._node_cache[node_name] @@ -1563,7 +1563,7 @@ def get_node_info_opt(self, node_name) -> Optional[NodeInfo]: self._node_cache[node_name] = inf return inf - def get_node_info(self, node_name) -> NodeInfo: + def get_node_info(self, node_name: str) -> NodeInfo: """Cached node info lookup.""" inf = self.get_node_info_opt(node_name) if not inf: diff --git a/pgq/cascade/consumer.py b/pgq/cascade/consumer.py index ecb75f0..b482deb 100644 --- a/pgq/cascade/consumer.py +++ b/pgq/cascade/consumer.py @@ -194,7 +194,7 @@ def process_root_node(self, dst_db: Connection) -> None: self.log.info('{standby: 1}') - def work(self) -> int: + def work(self) -> Optional[int]: """Refresh state before calling Consumer.work().""" dst_db = self.get_database(self.target_db) diff --git a/pgq/cascade/nodeinfo.py b/pgq/cascade/nodeinfo.py index 86ae99e..11ac5ef 100644 --- a/pgq/cascade/nodeinfo.py +++ b/pgq/cascade/nodeinfo.py @@ -103,6 +103,9 @@ def __init__(self, queue_name: str, row: Optional[DictRow], main_worker: bool = assert node_name self.name = node_name self.type = 'dead' + self.last_tick = None + self.paused = True + self.uptodate = False return self.name = row['node_name'] @@ -136,19 +139,6 @@ def __init__(self, queue_name: str, row: Optional[DictRow], main_worker: bool = if a: self.node_attrs = skytools.db_urldecode(a) - def __get_target_queue(self) -> Optional[str]: - qname = None - if self.type == LEAF: - if self.combined_queue: - qname = self.combined_queue - else: - return None - else: - qname = self.queue_name - if qname is None: - raise Exception("no target queue") - return qname - def get_title(self) -> str: if self.service: return "%s (%s, %s)" % (self.name, self.type, self.service) diff --git a/pgq/cascade/worker.py b/pgq/cascade/worker.py index 851daa3..8db15d7 100644 --- a/pgq/cascade/worker.py +++ b/pgq/cascade/worker.py @@ -10,8 +10,8 @@ import skytools from skytools.basetypes import Cursor, Connection, DictRow -from pgq.cascade.consumer import CascadedConsumer, EventList -from pgq.baseconsumer import BatchInfo +from pgq.cascade.consumer import CascadedConsumer +from pgq.baseconsumer import BatchInfo, EventList from pgq.event import Event from pgq.producer import bulk_insert_events diff --git a/pgq/coopconsumer.py b/pgq/coopconsumer.py index cd3ca5a..9d7df87 100644 --- a/pgq/coopconsumer.py +++ b/pgq/coopconsumer.py @@ -5,6 +5,7 @@ from skytools.basetypes import Cursor +from pgq.baseconsumer import EventList from pgq.consumer import Consumer __all__ = ['CoopConsumer'] @@ -76,7 +77,7 @@ def _load_next_batch(self, curs: Cursor) -> Optional[int]: curs.execute(q, [self.queue_name, self.consumer_name, self.subconsumer_name]) return curs.fetchone()[0] - def _finish_batch(self, curs: Cursor, batch_id: int, ev_list) -> None: + def _finish_batch(self, curs: Cursor, batch_id: int, ev_list: EventList) -> None: """Finish batch. (internal)""" self._flush_retry(curs, batch_id, ev_list) diff --git a/pgq/event.py b/pgq/event.py index 9f1bcfd..2b20222 100644 --- a/pgq/event.py +++ b/pgq/event.py @@ -1,7 +1,7 @@ """PgQ event container. """ -from typing import Any, Optional, Iterable, Tuple +from typing import Any, Optional, Mapping, KeysView, Iterator, ValuesView, ItemsView from skytools.basetypes import DictRow @@ -32,7 +32,7 @@ } -class Event(object): +class Event(Mapping[str, Any]): """Event data for consumers. Will be removed from the queue by default. @@ -51,31 +51,37 @@ def __init__(self, queue_name: str, row: DictRow) -> None: def __getattr__(self, key: str) -> Any: return self._event_row[_fldmap[key]] + def __iter__(self) -> Iterator[str]: + return iter(self._event_row) + + def __len__(self) -> int: + return len(self._event_row) + # would be better in RetriableEvent only since we don't care but # unfortunately it needs to be defined here due to compatibility concerns def tag_done(self) -> None: pass # be also dict-like - def __getitem__(self, k: str) -> Any: - return self._event_row.__getitem__(k) + def __getitem__(self, key: str) -> Any: + return self._event_row.__getitem__(key) - def __contains__(self, k: str) -> bool: - return self._event_row.__contains__(k) + def __contains__(self, key: object) -> bool: + return self._event_row.__contains__(key) - def get(self, k: str, d: Optional[Any] = None) -> Any: - return self._event_row.get(k, d) + def get(self, key: str, default: Optional[Any] = None) -> Any: + return self._event_row.get(key, default) - def has_key(self, k: str) -> bool: - return k in self._event_row + def has_key(self, key: str) -> bool: + return key in self._event_row - def keys(self) -> Iterable[str]: + def keys(self) -> KeysView[str]: return self._event_row.keys() - def values(self) -> Iterable[Any]: + def values(self) -> ValuesView[Any]: return self._event_row.values() - def items(self) -> Iterable[Tuple[str, Any]]: + def items(self) -> ItemsView[str, Any]: return self._event_row.items() def __str__(self) -> str: diff --git a/pgq/localconsumer.py b/pgq/localconsumer.py index 4c74050..6466416 100644 --- a/pgq/localconsumer.py +++ b/pgq/localconsumer.py @@ -106,7 +106,7 @@ def check_queue(self) -> None: else: self.log.info("Ticks match: Queue=%d Local=%d", queue_tick, local_tick) - def work(self) -> int: + def work(self) -> Optional[int]: if self.work_state < 0: self.check_queue() return super().work() diff --git a/pgq/producer.py b/pgq/producer.py index 8880a81..49b6dc0 100644 --- a/pgq/producer.py +++ b/pgq/producer.py @@ -1,8 +1,12 @@ """PgQ producer helpers for Python. """ +from typing import Sequence, Optional, Any, Mapping, Union + import skytools +from skytools.basetypes import Cursor + __all__ = ['bulk_insert_events', 'insert_event'] _fldmap = { @@ -26,17 +30,25 @@ } -def bulk_insert_events(curs, rows, fields, queue_name): +def bulk_insert_events(curs: Cursor, + rows: Union[Sequence[Sequence[Any]], Sequence[Mapping[str, Any]]], + fields: Sequence[str], + queue_name: str) -> None: q = "select pgq.current_event_table(%s)" curs.execute(q, [queue_name]) tbl = curs.fetchone()[0] - db_fields = map(_fldmap.get, fields) + db_fields = [_fldmap[name] for name in fields] skytools.magic_insert(curs, tbl, rows, db_fields) -def insert_event(curs, queue, ev_type, ev_data, - extra1=None, extra2=None, - extra3=None, extra4=None): +def insert_event(curs: Cursor, + queue: str, + ev_type: Optional[str], + ev_data: Optional[str], + extra1: Optional[str] = None, + extra2: Optional[str] = None, + extra3: Optional[str] = None, + extra4: Optional[str] = None) -> int: q = "select pgq.insert_event(%s, %s, %s, %s, %s, %s, %s)" curs.execute(q, [queue, ev_type, ev_data, extra1, extra2, extra3, extra4]) diff --git a/pgq/remoteconsumer.py b/pgq/remoteconsumer.py index c497f0a..4163b45 100644 --- a/pgq/remoteconsumer.py +++ b/pgq/remoteconsumer.py @@ -7,7 +7,8 @@ import optparse from skytools.basetypes import Cursor, Connection -from pgq.consumer import Consumer, EventList +from pgq.baseconsumer import EventList +from pgq.consumer import Consumer __all__ = ['RemoteConsumer', 'SerialConsumer'] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b291097 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,702 @@ +[project] +name = "pgq" +description = "PgQ client library for Python" +readme = "README.rst" +keywords = ["database", "queue"] +dynamic = ["version"] +requires-python = ">= 3.7" +maintainers = [{name = "Marko Kreen", email = "markokr@gmail.com"}] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: ISC License (ISCL)", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Programming Language :: Python :: 3", + "Topic :: Database", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = ["skytools"] + +[project.optional-dependencies] +test = ["pytest", "pytest-cov", "coverage[toml]", "psycopg2-binary"] +doc = ["sphinx"] + +[project.urls] +homepage = "https://github.com/pgq/python-pgq" +#documentation = "https://readthedocs.org" +repository = "https://github.com/pgq/python-pgq" +changelog = "https://github.com/pgq/python-pgq/blob/master/NEWS.rst" + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["pgq", "pgq.cascade"] +package-data = {"pgq" = ["py.typed"]} +zip-safe = false + +[tool.setuptools.dynamic.version] +attr = "pgq.__version__" + +# +# testing +# + +[tool.pytest] +testpaths = ["tests"] + +[tool.coverage.paths] +source = ["pgq", "**/site-packages/pgq"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", +] + +# +# formatting +# + +[tool.isort] +atomic = true +line_length = 100 +multi_line_output = 5 +known_first_party = ["pgq"] +known_third_party = ["pytest", "yaml", "skytools"] +include_trailing_comma = true +balanced_wrapping = true + +[tool.autopep8] +exclude = ".tox, git, tmp, build, cover, dist" +ignore = ["E301", "E265", "W391"] +max-line-length = 110 +in-place = true +recursive = true +aggressive = 2 + +[tool.doc8] +extensions = "rst" + +# +# linters +# + +[tool.mypy] +python_version = "3.10" +strict = true + +disallow_any_unimported = true +disallow_any_expr = false +disallow_any_decorated = false +disallow_any_explicit = false +disallow_any_generics = false +warn_return_any = false +warn_unreachable = false + +[[tool.mypy.overrides]] +module = [] +strict = false +disallow_untyped_defs = false +disallow_untyped_calls = false +disallow_incomplete_defs = false + +[tool.ruff] +line-length = 120 +select = ["E", "F", "Q", "W", "UP", "YTT", "ANN"] +ignore = [ + "ANN101", # Missing type annotation for `self` in method + "ANN102", # Missing type annotation for `cls` in classmethod + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed + "UP006", # Use `dict` instead of `Dict` + "UP007", # Use `X | Y` for type annotations + "UP031", # Use format specifiers instead of percent format + "UP032", # Use f-string instead of `format` call + "UP035", # typing.List` is deprecated + "UP037", # Remove quotes from type annotation + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` +] + +[tool.ruff.flake8-quotes] +docstring-quotes = "double" + +# +# reference links +# +# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ +# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html +# + +[tool.pylint.main] +# Analyse import fallback blocks. This can be used to support both Python 2 and 3 +# compatible code, which means that the block might have code that exists only in +# one or another interpreter, leading to false positives when analysed. +# analyse-fallback-blocks = + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint in +# a server-like mode. +# clear-cache-post-run = + +# Always return a 0 (non-error) status code, even if lint errors are found. This +# is primarily useful in continuous integration scripts. +# exit-zero = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +# extension-pkg-allow-list = + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +# extension-pkg-whitelist = + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +# fail-on = + +# Specify a score threshold under which the program will exit with error. +fail-under = 10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +# from-stdin = + +# Files or directories to be skipped. They should be base names, not paths. +ignore = ["CVS", "tmp", "dist"] + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, it +# can't be used as an escape character. +# ignore-paths = + +# Files or directories matching the regular expression patterns are skipped. The +# regex matches against base names, not paths. The default value ignores Emacs +# file locks +# ignore-patterns = + +# List of module names for which member attributes should not be checked (useful +# for modules/projects where namespaces are manipulated during runtime and thus +# existing member attributes cannot be deduced by static analysis). It supports +# qualified module names, as well as Unix pattern matching. +# ignored-modules = + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +# init-hook = + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs = 1 + +# Control the amount of potential inferred values when inferring a single object. +# This can help the performance when dealing with large functions or complex, +# nested conditions. +limit-inference-results = 100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +# load-plugins = + +# Pickle collected data for later comparisons. +persistent = true + +# Minimum Python version to use for version dependent checks. Will default to the +# version used to run pylint. +py-version = "3.10" + +# Discover python modules and packages in the file system subtree. +# recursive = + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +# source-roots = + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode = true + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +# unsafe-load-any-extension = + +[tool.pylint.basic] +# Naming style matching correct argument names. +argument-naming-style = "snake_case" + +# Regular expression matching correct argument names. Overrides argument-naming- +# style. If left empty, argument names will be checked with the set naming style. +# argument-rgx = + +# Naming style matching correct attribute names. +attr-naming-style = "snake_case" + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +# attr-rgx = + +# Bad variable names which should always be refused, separated by a comma. +bad-names = ["foo", "bar", "baz", "toto", "tutu", "tata"] + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +# bad-names-rgxs = + +# Naming style matching correct class attribute names. +class-attribute-naming-style = "any" + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +# class-attribute-rgx = + +# Naming style matching correct class constant names. +class-const-naming-style = "UPPER_CASE" + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +# class-const-rgx = + +# Naming style matching correct class names. +class-naming-style = "PascalCase" + +# Regular expression matching correct class names. Overrides class-naming-style. +# If left empty, class names will be checked with the set naming style. +# class-rgx = + +# Naming style matching correct constant names. +const-naming-style = "UPPER_CASE" + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming style. +# const-rgx = + +# Minimum line length for functions/classes that require docstrings, shorter ones +# are exempt. +docstring-min-length = -1 + +# Naming style matching correct function names. +function-naming-style = "snake_case" + +# Regular expression matching correct function names. Overrides function-naming- +# style. If left empty, function names will be checked with the set naming style. +# function-rgx = + +# Good variable names which should always be accepted, separated by a comma. +good-names = ["i", "j", "k", "ex", "Run", "_"] + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +# good-names-rgxs = + +# Include a hint for the correct naming format with invalid-name. +# include-naming-hint = + +# Naming style matching correct inline iteration names. +inlinevar-naming-style = "any" + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +# inlinevar-rgx = + +# Naming style matching correct method names. +method-naming-style = "snake_case" + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +# method-rgx = + +# Naming style matching correct module names. +module-naming-style = "snake_case" + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +# module-rgx = + +# Colon-delimited sets of names that determine each other's naming style when the +# name regexes allow several styles. +# name-group = + +# Regular expression which should only match function or class names that do not +# require a docstring. +no-docstring-rgx = "^_" + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. These +# decorators are taken in consideration only for invalid-name. +property-classes = ["abc.abstractproperty"] + +# Regular expression matching correct type alias names. If left empty, type alias +# names will be checked with the set naming style. +# typealias-rgx = + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +# typevar-rgx = + +# Naming style matching correct variable names. +variable-naming-style = "snake_case" + +# Regular expression matching correct variable names. Overrides variable-naming- +# style. If left empty, variable names will be checked with the set naming style. +# variable-rgx = + +[tool.pylint.classes] +# Warn about protected attribute access inside special methods +# check-protected-access-in-special-methods = + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods = ["__init__", "__new__", "setUp"] + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected = ["_asdict", "_fields", "_replace", "_source", "_make"] + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg = ["cls"] + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg = ["cls"] + +[tool.pylint.design] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +# exclude-too-few-public-methods = + +# List of qualified class names to ignore when counting class parents (see R0901) +# ignored-parents = + +# Maximum number of arguments for function / method. +max-args = 15 + +# Maximum number of attributes for a class (see R0902). +max-attributes = 37 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr = 5 + +# Maximum number of branch for function / method body. +max-branches = 50 + +# Maximum number of locals for function / method body. +max-locals = 45 + +# Maximum number of parents for a class (see R0901). +max-parents = 7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods = 420 + +# Maximum number of return / yield for function / method body. +max-returns = 16 + +# Maximum number of statements in function / method body. +max-statements = 150 + +# Minimum number of public methods for a class (see R0903). +min-public-methods = 0 + +[tool.pylint.exceptions] +# Exceptions that will emit a warning when caught. +overgeneral-exceptions = ["builtins.BaseException", "builtins.Exception"] + +[tool.pylint.format] +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format = "LF" + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines = "^\\s*(# )??$" + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren = 4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string = " " + +# Maximum number of characters on a single line. +max-line-length = 190 + +# Maximum number of lines in a module. +max-module-lines = 10000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +# single-line-class-stmt = + +# Allow the body of an if to be on the same line as the test if there is no else. +# single-line-if-stmt = + +[tool.pylint.imports] +# List of modules that can be imported at any level, not just the top level one. +# allow-any-import-level = + +# Allow explicit reexports by alias from a package __init__. +# allow-reexport-from-package = + +# Allow wildcard imports from modules that define __all__. +# allow-wildcard-with-all = + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules = ["optparse", "tkinter.tix"] + +# Output a graph (.gv or any supported image format) of external dependencies to +# the given file (report RP0402 must not be disabled). +# ext-import-graph = + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be disabled). +# import-graph = + +# Output a graph (.gv or any supported image format) of internal dependencies to +# the given file (report RP0402 must not be disabled). +# int-import-graph = + +# Force import order to recognize a module as part of the standard compatibility +# libraries. +# known-standard-library = + +# Force import order to recognize a module as part of a third party library. +known-third-party = ["enchant"] + +# Couples of modules and preferred modules, separated by a comma. +# preferred-modules = + +[tool.pylint.logging] +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style = "old" + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules = ["logging"] + +[tool.pylint."messages control"] +# Only show warnings with the listed confidence levels. Leave empty to show all. +# Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence = ["HIGH", "CONTROL_FLOW", "INFERENCE", "INFERENCE_FAILURE", "UNDEFINED"] + +# Disable the message, report, category or checker with the given id(s). You can +# either give multiple identifiers separated by comma (,) or put this option +# multiple times (only on the command line, not in the configuration file where +# it should appear only once). You can also use "--disable=all" to disable +# everything first and then re-enable specific checks. For example, if you want +# to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable = [ + "raw-checker-failed", "bad-inline-option", "locally-disabled", + "file-ignored", "suppressed-message", "useless-suppression", + "deprecated-pragma", "use-symbolic-message-instead", "bare-except", + "broad-exception-caught", "useless-return", "consider-using-in", + "consider-using-ternary", "fixme", "global-statement", "invalid-name", + "missing-module-docstring", "missing-class-docstring", + "missing-function-docstring", "no-else-raise", "no-else-return", + "trailing-newlines", "unused-argument", "unused-variable", + "using-constant-test", "useless-object-inheritance", "duplicate-code", + "singleton-comparison", "consider-using-f-string", "broad-exception-raised", + "arguments-differ", "multiple-statements", "use-implicit-booleaness-not-len", + "chained-comparison", "unnecessary-pass", "cyclic-import", + "too-many-ancestors", "import-outside-toplevel", "protected-access", + "try-except-raise", "deprecated-module", "no-else-break", "no-else-continue", + # junk + "trailing-newlines", "consider-using-f-string", + # expected + "cyclic-import", + # issues + "broad-exception-caught", + "no-else-return", +] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where it +# should appear only once). See also the "--disable" option for examples. +enable = ["c-extension-no-member"] + +[tool.pylint.method_args] +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods = [ + "requests.api.delete", "requests.api.get", "requests.api.head", + "requests.api.options", "requests.api.patch", "requests.api.post", + "requests.api.put", "requests.api.request" +] + +[tool.pylint.miscellaneous] +# List of note tags to take in consideration, separated by a comma. +notes = ["FIXME", "XXX", "TODO"] + +# Regular expression of note tags to take in consideration. +# notes-rgx = + +[tool.pylint.refactoring] +# Maximum number of nested blocks for function / method body +max-nested-blocks = 10 + +# Complete name of functions that never returns. When checking for inconsistent- +# return-statements if a never returning function is called then it will be +# considered as an explicit return statement and no message will be printed. +never-returning-functions = ["sys.exit"] + +[tool.pylint.reports] +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each category, +# as well as 'statement' which is the total number of statements analyzed. This +# score is used by the global evaluation report (RP0004). +evaluation = "10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)" + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +# msg-template = + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +# output-format = + +# Tells whether to display a full report or only the messages. +# reports = + +# Activate the evaluation score. +# score = + +[tool.pylint.similarities] +# Comments are removed from the similarity computation +ignore-comments = true + +# Docstrings are removed from the similarity computation +ignore-docstrings = true + +# Imports are removed from the similarity computation +# ignore-imports = + +# Signatures are removed from the similarity computation +ignore-signatures = true + +# Minimum lines number of a similarity. +min-similarity-lines = 4 + +[tool.pylint.spelling] +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions = 4 + +# Spelling dictionary name. No available dictionaries : You need to install both +# the python package and the system dependency for enchant to work.. +# spelling-dict = + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives = "fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy:" + +# List of comma separated words that should not be checked. +spelling-ignore-words = "usr,bin,env" + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file = ".local.dict" + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +# spelling-store-unknown-words = + +[tool.pylint.typecheck] +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators = ["contextlib.contextmanager"] + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +# generated-members = + +# Tells whether missing members accessed in mixin class should be ignored. A +# class is considered mixin if its name matches the mixin-class-rgx option. +# Tells whether to warn about missing members when the owner of the attribute is +# inferred to be None. +ignore-none = true + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference can +# return multiple potential results while evaluating a Python object, but some +# branches might not be evaluated, which results in partial inference. In that +# case, it might be useful to still emit no-member and other checks for the rest +# of the inferred objects. +ignore-on-opaque-inference = true + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins = ["no-member", "not-async-context-manager", "not-context-manager", "attribute-defined-outside-init"] + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes = ["optparse.Values", "thread._local", "_thread._local"] + +# Show a hint with possible names when a member name was not found. The aspect of +# finding the hint is based on edit distance. +missing-member-hint = true + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance = 1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices = 1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx = ".*[Mm]ixin" + +# List of decorators that change the signature of a decorated function. +# signature-mutators = + +[tool.pylint.variables] +# List of additional names supposed to be defined in builtins. Remember that you +# should avoid defining new builtins when possible. +# additional-builtins = + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables = true + +# List of names allowed to shadow builtins +# allowed-redefined-builtins = + +# List of strings which can identify a callback function by name. A callback name +# must start or end with one of those strings. +callbacks = ["cb_", "_cb"] + +# A regular expression matching the name of dummy variables (i.e. expected to not +# be used). +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" + +# Argument names that match this expression will be ignored. +ignored-argument-names = "_.*|^ignored_|^unused_" + +# Tells whether we should check for unused import in __init__ files. +# init-import = + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"] + + diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 9e6f0c7..0000000 --- a/setup.cfg +++ /dev/null @@ -1,15 +0,0 @@ - -[bdist_wheel] -universal=0 - -[flake8] -max-line-length = 120 -ignore = W391, W503, W504, H306, E1, E2, E3, E4, E5, E7 -exclude = .git,__pycache__,.tox - -[mypy] -python_version = 3.8 - -[mypy-setuptools] -ignore_missing_imports = True - diff --git a/setup.py b/setup.py index 51932db..bb8c386 100644 --- a/setup.py +++ b/setup.py @@ -3,44 +3,5 @@ from setuptools import setup - -# load version from pgq/__init__.py -_version = None -with open("pgq/__init__.py") as f: - for ln in f: - if ln.startswith("__version__"): - _version = ln.split()[2].strip("\"'") -assert _version - -# load info -with open("README.rst") as f: - ldesc = f.read().strip() - sdesc = ldesc.split('\n')[0] - -setup( - name="pgq", - description=sdesc, - long_description=ldesc, - version=_version, - license="ISC", - url="https://github.com/pgq/python-pgq", - maintainer="Marko Kreen", - maintainer_email="markokr@gmail.com", - packages=["pgq", "pgq.cascade"], - package_data={"pgq": ["py.typed"]}, - install_requires=["skytools"], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "License :: OSI Approved :: ISC License (ISCL)", - "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX", - "Programming Language :: Python :: 3", - "Topic :: Database", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Utilities", - ] -) +setup() diff --git a/tox.ini b/tox.ini index a2d2647..030e204 100644 --- a/tox.ini +++ b/tox.ini @@ -5,17 +5,21 @@ envlist = lint,py3 [package] name = pgq deps = - psycopg2-binary==2.9.5 - skytools==3.8.1 + psycopg2-binary==2.9.7 + skytools==3.9.2 test_deps = - coverage==6.5.0 - pytest==7.2.0 - pytest-cov==4.0.0 + coverage==7.2.7 + pytest==7.4.0 + pytest-cov==4.1.0 lint_deps = - pylint==2.15.6 - flake8==5.0.4 - mypy==0.991 - pytype==2022.11.18 + mypy==1.5.1 + pyflakes==3.1.0 + typing-extensions==4.7.1 + types-setuptools==68.1.0.0 + types-psycopg2==2.9.21.11 +xlint_deps = + pylint==2.17.5 + pytype==2023.8.22 [testenv] changedir = {envsitepackagesdir} @@ -48,13 +52,12 @@ commands = mypy {[package]name} [testenv:xlint] -basepython = python3 +basepython = python3.10 changedir = {toxinidir} deps = {[package]deps} {[package]lint_deps} -setenv = - PYLINTRC={toxinidir}/.pylintrc + {[package]xlint_deps} commands = pylint {[package]name} pytype {[package]name}