From 9f9636717e2797b47f5fd995fecd695cb4adf624 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 20 Jun 2021 19:52:59 -0400 Subject: [PATCH 1/9] Initial attempts adopting pytest-perf --- .github/workflows/main.yml | 16 --------------- exercises.ini | 15 ++++++++++++++ setup.cfg | 1 + tox.ini | 41 ++------------------------------------ 4 files changed, 18 insertions(+), 55 deletions(-) create mode 100644 exercises.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 35890dd4..e19dccf0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,22 +24,6 @@ jobs: - name: Run tests run: tox - benchmark: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install tox - run: | - python -m pip install tox - - name: Run benchmarks - run: tox - env: - TOXENV: perf{,-ref} - diffcov: runs-on: ubuntu-latest steps: diff --git a/exercises.ini b/exercises.ini new file mode 100644 index 00000000..8f4e6578 --- /dev/null +++ b/exercises.ini @@ -0,0 +1,15 @@ +[discovery] +warmup = import importlib_metadata +exercise = importlib_metadata.distribution("ipython") + +[entry_points()] +warmup = import importlib_metadata +exercise = importlib_metadata.entry_points() + +[cached distribution] +warmup = import importlib_metadata; importlib_metadata.distribution("ipython") +exercise = importlib_metadata.distribution("ipython") + +[uncached distribution] +warmup = import importlib, importlib_metadata +exercise = importlib.invalidate_caches(); importlib_metadata.distribution("ipython") diff --git a/setup.cfg b/setup.cfg index eb9b6541..51306d44 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,6 +51,7 @@ testing = pep517 pyfakefs flufl.flake8 + pytest-perf docs = # upstream diff --git a/tox.ini b/tox.ini index 2bff971d..cf96d1ad 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,8 @@ toxworkdir={env:TOX_WORK_DIR:.tox} deps = commands = pytest {posargs} +passenv = + HOME usedevelop = True extras = testing setenv = @@ -33,45 +35,6 @@ commands = diff-cover coverage.xml --compare-branch=origin/main --html-report diffcov.html diff-cover coverage.xml --compare-branch=origin/main --fail-under=100 -[perf] -deps = - ipython - pyperf - path -commands = - python -c "import path; path.Path('{env:SCOPE}.json').remove_p()" - - python -m pyperf timeit --name discovery --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.distribution("ipython")' - - python -m pyperf timeit --name 'entry_points()' --append {env:SCOPE}.json -s 'import importlib_metadata' 'importlib_metadata.entry_points()' - - python -m pyperf timeit --name 'cached distribution' --append {env:SCOPE}.json -s 'import importlib_metadata; importlib_metadata.distribution("ipython")' 'importlib_metadata.distribution("ipython")' - - python -m pyperf timeit --name 'uncached distribution' --append {env:SCOPE}.json -s 'import importlib, importlib_metadata' 'importlib.invalidate_caches(); importlib_metadata.distribution("ipython")' - -[testenv:perf] -use_develop = False -# change dir to avoid picking up local package. -changedir = {toxworkdir} -setenv = - SCOPE = local -deps = {[perf]deps} -commands = {[perf]commands} - -[testenv:perf-ref] -# compare perf results to the main branch -skip_install = True -# change dir to avoid picking up local package. -changedir = {toxworkdir} -setenv = - SCOPE = main -deps = - {[perf]deps} - git+https://github.com/python/importlib_metadata -commands = - {[perf]commands} - python -m pyperf compare_to --verbose main.json local.json --table - [testenv:release] skip_install = True deps = From 9f22c33326b029c3e71bbd845924375b05ce4b9e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 20 Jun 2021 21:19:59 -0400 Subject: [PATCH 2/9] Ensure that ipython gets installed via testing extra. --- exercises.ini | 3 +++ setup.cfg | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/exercises.ini b/exercises.ini index 8f4e6578..e9daf5d2 100644 --- a/exercises.ini +++ b/exercises.ini @@ -1,6 +1,7 @@ [discovery] warmup = import importlib_metadata exercise = importlib_metadata.distribution("ipython") +extras = testing [entry_points()] warmup = import importlib_metadata @@ -9,7 +10,9 @@ exercise = importlib_metadata.entry_points() [cached distribution] warmup = import importlib_metadata; importlib_metadata.distribution("ipython") exercise = importlib_metadata.distribution("ipython") +extras = testing [uncached distribution] warmup = import importlib, importlib_metadata exercise = importlib.invalidate_caches(); importlib_metadata.distribution("ipython") +extras = testing diff --git a/setup.cfg b/setup.cfg index 51306d44..52b8c9c1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,10 @@ testing = pep517 pyfakefs flufl.flake8 - pytest-perf + pytest-perf>=0.4 + + # for performance tests + ipython docs = # upstream From d4f6f7b82fc16a42ed4294da3fd8edb055abd017 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 24 Jun 2021 22:48:33 -0400 Subject: [PATCH 3/9] Bump pytest-perf. --- exercises.ini | 6 +++--- setup.cfg | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exercises.ini b/exercises.ini index e9daf5d2..b86049b3 100644 --- a/exercises.ini +++ b/exercises.ini @@ -1,7 +1,7 @@ [discovery] warmup = import importlib_metadata exercise = importlib_metadata.distribution("ipython") -extras = testing +extras = perf [entry_points()] warmup = import importlib_metadata @@ -10,9 +10,9 @@ exercise = importlib_metadata.entry_points() [cached distribution] warmup = import importlib_metadata; importlib_metadata.distribution("ipython") exercise = importlib_metadata.distribution("ipython") -extras = testing +extras = perf [uncached distribution] warmup = import importlib, importlib_metadata exercise = importlib.invalidate_caches(); importlib_metadata.distribution("ipython") -extras = testing +extras = perf diff --git a/setup.cfg b/setup.cfg index 52b8c9c1..5e56d47c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ testing = pep517 pyfakefs flufl.flake8 - pytest-perf>=0.4 + pytest-perf >= 0.5.1 # for performance tests ipython From 6ed9128015fbc54de18c545e97aca5617471b922 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 25 Jun 2021 23:19:19 -0400 Subject: [PATCH 4/9] Bump to pytest-perf 0.6.2 and remove ipython from testing reqs. --- setup.cfg | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 134ca0e4..b0518189 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,10 +51,7 @@ testing = pep517 pyfakefs flufl.flake8 - pytest-perf >= 0.5.1 - - # for performance tests - ipython + pytest-perf >= 0.6.2 docs = # upstream From 5f08b8d02a3f1cf02a54363b13c2e55c697d3d01 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Jun 2021 18:58:06 -0400 Subject: [PATCH 5/9] Rely on pytest_perf 0.9 and the new Python syntax. --- exercises.ini | 18 ------------------ exercises.py | 32 ++++++++++++++++++++++++++++++++ setup.cfg | 2 +- 3 files changed, 33 insertions(+), 19 deletions(-) delete mode 100644 exercises.ini create mode 100644 exercises.py diff --git a/exercises.ini b/exercises.ini deleted file mode 100644 index b86049b3..00000000 --- a/exercises.ini +++ /dev/null @@ -1,18 +0,0 @@ -[discovery] -warmup = import importlib_metadata -exercise = importlib_metadata.distribution("ipython") -extras = perf - -[entry_points()] -warmup = import importlib_metadata -exercise = importlib_metadata.entry_points() - -[cached distribution] -warmup = import importlib_metadata; importlib_metadata.distribution("ipython") -exercise = importlib_metadata.distribution("ipython") -extras = perf - -[uncached distribution] -warmup = import importlib, importlib_metadata -exercise = importlib.invalidate_caches(); importlib_metadata.distribution("ipython") -extras = perf diff --git a/exercises.py b/exercises.py new file mode 100644 index 00000000..6358b05a --- /dev/null +++ b/exercises.py @@ -0,0 +1,32 @@ +from pytest_perf.deco import extras + + +@extras('perf') +def discovery_perf(): + "discovery" + import importlib_metadata # end warmup + importlib_metadata.distribution('ipython') + + +def entry_points_perf(): + "entry_points()" + import importlib_metadata # end warmup + importlib_metadata.entry_points() + + +@extras('perf') +def cached_distribution_perf(): + "cached distribution" + import importlib_metadata + importlib_metadata.distribution('ipython') # end warmup + importlib_metadata.distribution('ipython') + + +@extras('perf') +def uncached_distribution_perf(): + "uncached distribution" + import importlib + import importlib_metadata + # end warmup + importlib.invalidate_caches() + importlib_metadata.distribution('ipython') diff --git a/setup.cfg b/setup.cfg index b0518189..86b83819 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ testing = pep517 pyfakefs flufl.flake8 - pytest-perf >= 0.6.2 + pytest-perf >= 0.9 docs = # upstream From f0d021ab817bbca135924a58d756ffa73a58b48f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Jun 2021 19:08:50 -0400 Subject: [PATCH 6/9] =?UTF-8?q?=E2=9A=AB=20Fade=20to=20black.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- exercises.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/exercises.py b/exercises.py index 6358b05a..bc8a44e9 100644 --- a/exercises.py +++ b/exercises.py @@ -5,12 +5,14 @@ def discovery_perf(): "discovery" import importlib_metadata # end warmup + importlib_metadata.distribution('ipython') def entry_points_perf(): "entry_points()" import importlib_metadata # end warmup + importlib_metadata.entry_points() @@ -18,6 +20,7 @@ def entry_points_perf(): def cached_distribution_perf(): "cached distribution" import importlib_metadata + importlib_metadata.distribution('ipython') # end warmup importlib_metadata.distribution('ipython') @@ -27,6 +30,7 @@ def uncached_distribution_perf(): "uncached distribution" import importlib import importlib_metadata + # end warmup importlib.invalidate_caches() importlib_metadata.distribution('ipython') From 1937287ce1f7d40639f2d1fa17a150dfbe0ffdc0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Jun 2021 19:11:54 -0400 Subject: [PATCH 7/9] Bump to pytest-perf 0.9.1 with bugfix for module collection. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 86b83819..def82258 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ testing = pep517 pyfakefs flufl.flake8 - pytest-perf >= 0.9 + pytest-perf >= 0.9.1 docs = # upstream From dadbb4f852ea2b6ff6d7ef8c49eb452339da3ae3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Jun 2021 19:49:58 -0400 Subject: [PATCH 8/9] Bump to pytest-perf 0.9.2 with bugfix for module collection. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index def82258..98124e5c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ testing = pep517 pyfakefs flufl.flake8 - pytest-perf >= 0.9.1 + pytest-perf >= 0.9.2 docs = # upstream From 6dfb53082be7b8bce2cd21f042a11bae11b7119e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Jun 2021 21:43:14 -0400 Subject: [PATCH 9/9] Ignore exercises.py in coverage. --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 98113f51..49fe2faf 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,7 @@ omit = tests/* prepare/* */_itertools.py + exercises.py [report] show_missing = True