diff --git a/.circleci/config.yml b/.circleci/config.yml index 9324de943607..8cf18d809d86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,9 +40,9 @@ jobs: name: create release notes command: | . venv/bin/activate - pip install git+https://github.com/hawkowl/towncrier.git@master + pip install towncrier VERSION=$(python -c "import setup; print(setup.VERSION)") - towncrier --version $VERSION --yes + towncrier build --version $VERSION --yes ./tools/ci/test_all_newsfragments_used.py - run: @@ -62,6 +62,7 @@ jobs: - run: name: build devdocs + no_output_timeout: 30m command: | . venv/bin/activate cd doc @@ -90,7 +91,7 @@ jobs: - run: name: deploy devdocs command: | - if [ "${CIRCLE_BRANCH}" == "master" ]; then + if [ "${CIRCLE_BRANCH}" == "main" ]; then touch doc/build/html/.nojekyll ./tools/ci/push_docs_to_repo.py doc/build/html \ @@ -100,7 +101,7 @@ jobs: --message "Docs build of $CIRCLE_SHA1" \ --force else - echo "Not on the master branch; skipping deployment" + echo "Not on the main branch; skipping deployment" fi - add_ssh_keys: @@ -119,7 +120,7 @@ jobs: - run: name: deploy neps command: | - if [ "${CIRCLE_BRANCH}" == "master" ]; then + if [ "${CIRCLE_BRANCH}" == "main" ]; then touch doc/neps/_build/html/.nojekyll ./tools/ci/push_docs_to_repo.py doc/neps/_build/html \ @@ -129,5 +130,5 @@ jobs: --message "Docs build of $CIRCLE_SHA1" \ --force else - echo "Not on the master branch; skipping deployment" + echo "Not on the main branch; skipping deployment" fi diff --git a/.codecov.yml b/.codecov.yml index 8c19f9e8ee58..165b3099df18 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -6,8 +6,9 @@ coverage: status: project: default: - # Require 1% coverage, i.e., always succeed - target: 1 - patch: true + informational: true + patch: + default: + informational: true changes: false comment: off diff --git a/.dependabot/config.yml b/.dependabot/config.yml deleted file mode 100644 index 160ec85cfa7f..000000000000 --- a/.dependabot/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: 1 -update_configs: - - package_manager: "python" - directory: "/" - update_schedule: "weekly" - commit_message: - prefix: "MAINT" - default_labels: - - "03 - Maintenance" diff --git a/.gitattributes b/.gitattributes index bce3dbe6daad..8723dd9dc95a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,21 +1,109 @@ -# Numerical data files -numpy/lib/tests/data/*.npy binary - -# Release notes, reduce number of conflicts. -doc/release/*.rst merge=union - # Highlight our custom templating language as C, since it's hopefully better # than nothing. This also affects repo statistics. -*.c.src linguist-language=C -*.inc.src linguist-language=C -*.h.src linguist-language=C +*.c.src text linguist-language=C +*.inc.src text linguist-language=C +*.h.src text linguist-language=C +*.pyx.in text linguist-language=Python +*.pxd.in text linguist-language=Python # Mark some files as vendored numpy/linalg/lapack_lite/f2c.c linguist-vendored numpy/linalg/lapack_lite/f2c.h linguist-vendored tools/npy_tempita/* linguist-vendored +numpy/core/include/numpy/libdivide/* linguist-vendored # Mark some files as generated numpy/linalg/lapack_lite/f2c_*.c linguist-generated numpy/linalg/lapack_lite/lapack_lite_names.h linguist-generated +numpy/_version.py export-subst + +# Configuration files +*.ini text +*.cfg text +./MANIFEST.in text +./numpy/core/npymath.ini.in text +./numpy/core/mlib.ini.in text +./site.cfg.example text + +# Python sources +*.py text diff=python +*.pxd text diff=python +*.pyx text diff=python +*.pyi text diff=python + +# C/C++ sources +*.c text diff=c +*.h text diff=c +*.cc text diff=cpp +*.cxx text diff=cpp +*.cpp text diff=cpp +*.hpp text diff=cpp +*.hh text diff=cpp + +# Fortran sources +*.f text diff=fortran +*.for text diff=fortran +*.f90 text diff=fortran +*.f95 text diff=fortran +*.f03 text diff=fortran + +# JavaScript +*.js text + +# F2py +./doc/source/f2py/*.pyf text +./doc/source/f2py/*.dat text +./numpy/f2py/tests/src/module_data/mod.mod binary + +# Documents +*.md text diff=markdown +*.txt text +*.rst text +*.pdf binary +*.css text diff=css +*.html text diff=html + +# Graphics +*.png binary +*.ico binary +*.dia binary +*.gif binary +*.odg binary +*.fig text +*.svg text +# SVG is treated as an asset (binary) by default. If you want +# to treat it as binary, use the following line instead. +# *.svg binary + +# Scripts +*.sh text eol=lf +*.sed text +# These are explicitly windows files and should use crlf +*.bat text eol=crlf +*.cmd text eol=crlf + +# Serialisation +*.json text +*.toml text +*.xml text +*.yaml text +*.yml text + +# Data files +*.csv text +*.pkl binary +*.fits binary +*.npy binary +*.npz binary + +# Misc. +*.swg text +*.patch text +./doc/neps/index.rst.tmpl text +./benchmarks/asv_compare.conf.json.tpl text +./tools/swig/test/*.i text +./tools/gitpod/gitpod.Dockerfile text +./doc/source/dev/gitwash/git_links.inc text +./doc/source/reference/simd/*.inc text +./numpy/core/src/_simd/*.inc text diff=c diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index da599095647a..8f16950f765e 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -8,7 +8,7 @@ also include a brief, self-contained code example that demonstrates the problem. If you are reporting a segfault please include a GDB traceback, which you can generate by following -[these instructions.](https://github.com/numpy/numpy/blob/master/doc/source/dev/development_environment.rst#debugging) +[these instructions.](https://github.com/numpy/numpy/blob/main/doc/source/dev/development_environment.rst#debugging) ## Contributing code diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 78ffc1063eaf..6da1f7370d00 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -20,7 +20,7 @@ import numpy as np +https://github.com/numpy/numpy/blob/main/doc/source/dev/development_environment.rst#debugging --> diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md index 00c6f59c5faf..68872ec06caf 100644 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -12,5 +12,5 @@ post your idea on the [numpy-discussion mailing list] (https://mail.python.org/mailman/listinfo/numpy-discussion) to explain your reasoning in addition to opening an issue or pull request. You can also check out our [Contributor Guide] -(https://github.com/numpy/numpy/blob/master/doc/source/dev/index.rst) if you +(https://github.com/numpy/numpy/blob/main/doc/source/dev/index.rst) if you need more information. --> diff --git a/.github/ISSUE_TEMPLATE/post-install.md b/.github/ISSUE_TEMPLATE/post-install.md index c0ec7896a40d..11b91384c1fd 100644 --- a/.github/ISSUE_TEMPLATE/post-install.md +++ b/.github/ISSUE_TEMPLATE/post-install.md @@ -15,7 +15,7 @@ labels: 32 - Installation +https://github.com/numpy/numpy/blob/main/doc/source/dev/development_environment.rst#debugging --> diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 508c8c034869..704d2d16fd9c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,16 +1,16 @@ - + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000000..16ce0846cb59 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,22 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 10 + labels: + - 03 - Maintenance + ignore: + - dependency-name: gitpython + versions: + - "> 3.1.13, < 3.2" + - dependency-name: pydata-sphinx-theme + versions: + - 0.6.0 + - 0.6.1 + - dependency-name: hypothesis + versions: + - 6.3.0 + commit-message: + prefix: MAINT diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index d38ae0934b33..c7d463348f6d 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -1,6 +1,14 @@ name: Build_Test -on: [push, pull_request] +on: + push: + branches: + - main + - maintenance/** + pull_request: + branches: + - main + - maintenance/** defaults: run: @@ -11,7 +19,27 @@ env: PYTHON_VERSION: 3.7 jobs: + lint: + if: "github.repository == 'numpy/numpy' && github.ref != 'refs/heads/main' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install linter requirements + run: + python -m pip install -r linter_requirements.txt + - name: Run linter on PR diff + run: + python tools/linter.py --branch origin/${{ github.base_ref }} + smoke_test: + if: "github.repository == 'numpy/numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -24,7 +52,7 @@ jobs: - uses: ./.github/actions basic: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest strategy: matrix: @@ -36,11 +64,11 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v2 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: ${{ matrix.python-version }} - uses: ./.github/actions debug: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-20.04 env: USE_DEBUG: 1 @@ -55,7 +83,7 @@ jobs: - uses: ./.github/actions blas64: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest env: NPY_USE_BLAS_ILP64: 1 @@ -70,8 +98,8 @@ jobs: - uses: ./.github/actions full: - needs: smoke_test - runs-on: ubuntu-latest + needs: [smoke_test, lint] + runs-on: ubuntu-18.04 env: USE_WHEEL: 1 RUN_FULL_TESTS: 1 @@ -88,7 +116,7 @@ jobs: - uses: ./.github/actions benchmark: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest env: PYTHONOPTIMIZE: 2 @@ -109,7 +137,7 @@ jobs: - uses: ./.github/actions no_relaxed_strides: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest env: NPY_RELAXED_STRIDES_CHECKING: 0 @@ -126,7 +154,7 @@ jobs: - uses: ./.github/actions use_wheel: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest env: USE_WHEEL: 1 @@ -142,7 +170,7 @@ jobs: - uses: ./.github/actions no_array_func: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest env: NUMPY_EXPERIMENTAL_ARRAY_FUNCTION: 0 @@ -157,7 +185,7 @@ jobs: - uses: ./.github/actions no_openblas: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest env: BLAS: None @@ -175,27 +203,30 @@ jobs: - uses: ./.github/actions pypy37: - needs: smoke_test + needs: [smoke_test, lint] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: submodules: recursive fetch-depth: 0 - - name: get_pypy - run: | - wget -q https://downloads.python.org/pypy/pypy3.6-v7.3.2-linux64.tar.bz2 -O pypy.tar.bz2 - mkdir -p pypy3 - (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2) - pypy3/bin/pypy3 -mensurepip - pypy3/bin/pypy3 -m pip install --upgrade pip wheel - if [ ! -e pypy3/bin/python ] - then - pushd pypy3/bin - ln -s pypy3 python - popd - fi - echo $PWD/pypy3/bin >> $GITHUB_PATH + - uses: actions/setup-python@v2 + with: + python-version: pypy-3.7-v7.3.4 + - uses: ./.github/actions + sdist: + needs: [smoke_test, lint] + runs-on: ubuntu-latest + env: + USE_SDIST: 1 + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} - uses: ./.github/actions diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000000..cc4950590af0 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,57 @@ +name: Build Base Docker Image + +on: + push: + branches: + - main + paths: + - 'environment.yml' + +jobs: + build: + name: Build base Docker image + runs-on: ubuntu-latest + environment: numpy-dev + if: "github.repository_owner == 'numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + steps: + - name: Clone repository + uses: actions/checkout@v2 + - name: Lint Docker + uses: brpaz/hadolint-action@v1.2.1 + with: + dockerfile: ./tools/gitpod/Dockerfile + - name: Get refs + shell: bash + run: | + export raw_branch=${GITHUB_REF#refs/heads/} + echo "::set-output name=branch::${raw_branch//\//-}" + echo "::set-output name=date::$(date +'%Y%m%d')" + echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)" + id: getrefs + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: ${{ runner.os }}-buildx- + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: "." + file: "./tools/gitpod/Dockerfile" + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + numpy/numpy-dev:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, numpy/numpy-dev:latest + - name: Image digest + # Return details of the image build: sha and shell + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/gitpod.yml b/.github/workflows/gitpod.yml new file mode 100644 index 000000000000..55683bcae78d --- /dev/null +++ b/.github/workflows/gitpod.yml @@ -0,0 +1,55 @@ +name: Build Gitpod Docker image + +on: + push: + branches: + - main + +jobs: + build: + name: Build Gitpod Docker image + runs-on: ubuntu-latest + environment: numpy-dev + if: "github.repository_owner == 'numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + steps: + - name: Clone repository + uses: actions/checkout@v2 + - name: Lint Docker + uses: brpaz/hadolint-action@v1.2.1 + with: + dockerfile: ./tools/gitpod/gitpod.Dockerfile + - name: Get refs + shell: bash + run: | + export raw_branch=${GITHUB_REF#refs/heads/} + echo "::set-output name=branch::${raw_branch//\//-}" + echo "::set-output name=date::$(date +'%Y%m%d')" + echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)" + id: getrefs + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: ${{ runner.os }}-buildx- + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: "." + file: "./tools/gitpod/gitpod.Dockerfile" + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + numpy/numpy-gitpod:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, numpy/numpy-gitpod:latest + - name: Image digest + # Return details of the image build: sha and shell + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index d24605871e4b..a2a1f2b68725 100644 --- a/.gitignore +++ b/.gitignore @@ -120,9 +120,7 @@ numpy/__config__.py numpy/core/include/numpy/__multiarray_api.h numpy/core/include/numpy/__ufunc_api.h numpy/core/include/numpy/_numpyconfig.h -numpy/version.py site.cfg -setup.cfg .tox numpy/core/include/numpy/__multiarray_api.c numpy/core/include/numpy/__ufunc_api.c @@ -131,20 +129,11 @@ numpy/core/include/numpy/config.h numpy/core/include/numpy/multiarray_api.txt numpy/core/include/numpy/ufunc_api.txt numpy/core/lib/ -numpy/core/src/_simd/_simd.dispatch.avx512_skx.c -numpy/core/src/_simd/_simd.dispatch.avx512f.c -numpy/core/src/_simd/_simd.dispatch.c -numpy/core/src/_simd/_simd.dispatch.fma3.avx2.c -numpy/core/src/_simd/_simd.dispatch.h -numpy/core/src/_simd/_simd.dispatch.sse42.c -numpy/core/src/_simd/_simd_data.inc -numpy/core/src/_simd/_simd_inc.h numpy/core/src/common/npy_binsearch.h numpy/core/src/common/npy_cpu_features.c numpy/core/src/common/npy_partition.h numpy/core/src/common/npy_sort.h numpy/core/src/common/templ_common.h -numpy/core/src/common/_cpu_dispatch.h numpy/core/src/multiarray/_multiarray_tests.c numpy/core/src/multiarray/arraytypes.c numpy/core/src/multiarray/einsum.c @@ -181,9 +170,7 @@ numpy/core/src/umath/simd.inc numpy/core/src/umath/struct_ufunc_test.c numpy/core/src/umath/test_rational.c numpy/core/src/umath/umath_tests.c -numpy/core/src/umath/_umath_tests.dispatch.avx2.c -numpy/core/src/umath/_umath_tests.dispatch.h -numpy/core/src/umath/_umath_tests.dispatch.sse41.c +numpy/core/src/umath/loops_utils.h numpy/distutils/__config__.py numpy/linalg/umath_linalg.c doc/source/**/generated/ @@ -200,6 +187,7 @@ numpy/random/legacy/*.c numpy/random/_mtrand/randint_helpers.pxi numpy/random/bounded_integers.pyx numpy/random/bounded_integers.pxd +numpy/random/lib/npyrandom.lib tools/swig/test/Array_wrap.cxx tools/swig/test/Farray_wrap.cxx tools/swig/test/Farray.py @@ -214,3 +202,20 @@ tools/swig/test/Tensor.py tools/swig/test/Vector.py tools/swig/test/Vector_wrap.cxx tools/swig/test/Array.py + +# SIMD generated files # +################################### +# config headers of dispatchable sources +*.dispatch.h +# wrapped sources of dispatched targets, e.g. *.dispatch.avx2.c +*.dispatch.*.c +# _simd module +numpy/core/src/_simd/_simd.dispatch.c +numpy/core/src/_simd/_simd_data.inc +numpy/core/src/_simd/_simd_inc.h +# umath module +numpy/core/src/umath/loops_unary_fp.dispatch.c +numpy/core/src/umath/loops_arithm_fp.dispatch.c +numpy/core/src/umath/loops_arithmetic.dispatch.c +numpy/core/src/umath/loops_trigonometric.dispatch.c +numpy/core/src/umath/loops_exponent_log.dispatch.c diff --git a/.gitmodules b/.gitmodules index e69de29bb2d1..0d6857868837 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "doc/source/_static/scipy-mathjax"] + path = doc/source/_static/scipy-mathjax + url = https://github.com/scipy/scipy-mathjax.git diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 000000000000..dfbee831a33b --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,63 @@ +# Rebuilding NumPy on init - rather than on prebuild: this ensures +# that even forks do have a usable freshly built NumPy +# Might delegate this later to prebuild with Q2 improvements on gitpod +# https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal +# ------------------------------------------------------------------------- + +image: numpy/numpy-gitpod:latest +tasks: + - name: Prepare development + init: | + mkdir -p .vscode + cp tools/gitpod/settings.json .vscode/settings.json + conda activate numpy-dev + python setup.py build_ext --inplace + echo "🛠 Completed rebuilding NumPy!! 🛠 " + echo "📖 Building docs 📖 " + git submodule update --init + cd doc + make html + echo "✨ Pre-build complete! You can close this terminal ✨ " + + +# -------------------------------------------------------- +# exposing ports for liveserve +ports: + - port: 5500 + onOpen: notify + +# -------------------------------------------------------- +# some useful extensions to have +vscode: + extensions: + - eamodio.gitlens + - njpwerner.autodocstring + - lextudio.restructuredtext + - ritwickdey.liveserver + - ms-python.python + - yzhang.markdown-all-in-one + - bungcip.better-toml + - mhutchie.git-graph + +# -------------------------------------------------------- +# using prebuilds for the container - note: atm this only +# works for the NumPy repo +# With this configuration the prebuild will happen on push to master +github: + prebuilds: + # enable for main/default branch + master: true + # enable for other branches (defaults to false) + branches: false + # enable for pull requests coming from this repo (defaults to true) + pullRequests: false + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: false + # add a check to pull requests (defaults to true) + addCheck: false + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: false + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: false + # add a label once the prebuild is ready to pull requests (defaults to false) + addLabel: false \ No newline at end of file diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000000..0188ba2cf627 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,7 @@ +--- +ignored: + - DL3006 + - DL3008 + - SC2016 + - DL3004 + - DL3007 \ No newline at end of file diff --git a/.mailmap b/.mailmap index f3fd63519e3b..3f45904fcd8b 100644 --- a/.mailmap +++ b/.mailmap @@ -8,294 +8,429 @@ # This file is up-to-date if the command git log --format="%aN <%aE>" | sort -u # gives no duplicates. -Aaron Baecker abaecker -Aarthi Agurusa agurusa -Alan Fontenot logeaux -Alan Fontenot logeaux <36168460+logeaux@users.noreply.github.com> -Abdul Muneer abdulmuneer -Abhilash Barigidad abhilash42 <64172584+abhilash42@users.noreply.github.com> -Abhinav Reddy tabhi0797 -Adam Ginsburg Adam Ginsburg -Albert Jornet Puig jurnix -Alex Rockhill Alex -Alex Griffing alex -Alex Griffing argriffing -Alex Griffing argriffing -Alex Griffing argriffing -Alex Thomas alexthomas93 -Alexander Belopolsky Alexander Belopolsky -Alexander Belopolsky Alexander Belopolsky +@8bitmp3 <19637339+8bitmp3@users.noreply.github.com> +@DWesl <22566757+DWesl@users.noreply.github.com> +@Endolith +@Illviljan <14371165+Illviljan@users.noreply.github.com> +@LSchroefl <65246829+LSchroefl@users.noreply.github.com> +@Lbogula +@Lisa <34400837+lyzlisa@users.noreply.github.com> +@Patrick <39380924+xamm@users.noreply.github.com> +@Scian <65375075+hoony6134@users.noreply.github.com> +@h-vetinari +@h6197627 <44726212+h6197627@users.noreply.github.com> +@jbCodeHub +@legoffant <58195095+legoffant@users.noreply.github.com> +@luzpaz +@luzpaz +@sfolje0 +@spacescientist +@tautaus +@xoviat <49173759+xoviat@users.noreply.github.com> +@xoviat <49173759+xoviat@users.noreply.github.com> +@yetanothercheer +Aaron Baecker +Aarthi Agurusa +Alan Fontenot +Alan Fontenot <36168460+logeaux@users.noreply.github.com> +Abdul Muneer +Abhilash Barigidad +Abhilash Barigidad <64172584+abhilash42@users.noreply.github.com> +Abhinav Reddy +Adam Ginsburg +Aerik Pawson <45904740+aerikpawson@users.noreply.github.com> +Albert Jornet Puig +Alex Rockhill +Alex Griffing +Alex Griffing +Alex Griffing +Alex Henrie +Alex Rogozhnikov +Alex Thomas +Alexander Belopolsky +Alexander Belopolsky +Alexander Belopolsky Alexander Belopolsky sasha -Alexander Jung aleju -Alexander Shadchin Alexandr Shadchin -Alexander Shadchin shadchin -Allan Haldane ahaldane +Alexander Hunt +Alexander Jung +Alexander Shadchin +Alexander Shadchin +Alizé Papp <68250865+alize-papp@users.noreply.github.com> +Allan Haldane +Al-Baraa El-Hag <48454648+a-elhag@users.noreply.github.com> Alok Singhal Alok Singhal -Alyssa Quek alyssaq -Amir Sarabadani amir -Anatoly Techtonik anatoly techtonik -Andras Deak adeak -Andrea Olivo andryandrew@gmail.com andryandrew -Andrea Pattori patto90 -Andrea Sangalli and-sang <53617841+and-sang@users.noreply.github.com> -Andrei Kucharavy chiffa -Anne Archibald aarchiba -Anne Archibald Anne Archibald -Anže Starič astaric -Aron Ahmadia ahmadia -Arun Persaud Arun Persaud -Ashutosh Singh ashutosh619-sudo -Ashutosh Singh Ashutosh singh <55102089+Ashutosh619-sudo@users.noreply.github.com> -Åsmund Hjulstad Åsmund Hjulstad -Auke Wiggers auke -Badhri Narayanan Krishnakumar badhrink -Bangcheng Yang DumbMice -Behzad Nouri behzad nouri -Ben Nathanson bjnath -Benjamin Root Ben Root +Alyssa Quek +Amir Sarabadani +Anas Khan +Anatoly Techtonik +Andras Deak +Andrea Olivo +Andrea Pattori +Andrea Sangalli <53617841+and-sang@users.noreply.github.com> +Andreas Klöckner +Andreas Schwab +Andrei Kucharavy +Andrew Lawson +Anirudh Subramanian +Anne Archibald +Anne Archibald +Anne Bonner <35413198+bonn0062@users.noreply.github.com> +Anthony Vo <43098273+anthonyhvo12@users.noreply.github.com> +Antoine Pitrou +Anže Starič +Aron Ahmadia +Arun Persaud +Ashutosh Singh +Ashutosh Singh <55102089+Ashutosh619-sudo@users.noreply.github.com> +Åsmund Hjulstad +Auke Wiggers +Badhri Narayanan Krishnakumar +Bangcheng Yang +Bhargav V <12525622+brpy@users.noreply.github.com> +Bas van Beek <43369155+BvB93@users.noreply.github.com> +Behzad Nouri +Ben Nathanson +Benjamin Root Benjamin Root weathergod -Bernardt Duvenhage bduvenhage -Bertrand Lefebvre bertrand -Bertrand Lefebvre Bertrand -Bharat Raghunathan Bharat123Rox -Bill Spotz William Spotz -Bill Spotz wfspotz@sandia.gov -Bob Eldering bobeldering -Brett R Murphy brettrmurphy +Bernardt Duvenhage +Bernie Gray +Bertrand Lefebvre +Bharat Raghunathan +Bharat Raghunathan +Bob Eldering +Brett R Murphy +Brigitta Sipocz +Brian Soto +Brian Soto +Brian Soto Bryan Van de Ven Bryan Van de Ven Bryan Van de Ven Bryan Van de Ven -Bui Duc Minh Mibu287 <41239569+Mibu287@users.noreply.github.com> -Carl Kleffner carlkl -Carl Leake leakec -Chris Burns chris.burns -Chris Kerr Chris Kerr -Christian Clauss cclauss -Christopher Hanley chanley +Bui Duc Minh <41239569+Mibu287@users.noreply.github.com> +Carl Kleffner +Carl Leake +Charles Stern <62192187+cisaacstern@users.noreply.github.com> +Chris Barker +Chris Burns +Chris Holland <41524756+ChrisAHolland@users.noreply.github.com> +Chris Kerr +Chris Vavaliaris +Christian Clauss +Christopher Dahlin +Christopher Hanley +Christoph Gohlke +Christoph Gohlke Christoph Gohlke cgholke -Christoph Gohlke cgohlke -Christoph Gohlke Christolph Gohlke -Chunlin Fang Qiyu8 -Chunlin Fang qiyu8 -Chunlin Fang Chunlin -Colin Snyder <47012605+colinsnyder@users.noreply.github.com> colinsnyder <47012605+colinsnyder@users.noreply.github.com> -Daniel B Allan danielballan -Daniel da Silva Daniel da Silva -Daniel da Silva Daniel da Silva -Daniel Hrisca danielhrisca -Daniel J Farrell danieljfarrell +Chunlin Fang +Chunlin Fang +Chunlin Fang <834352945@qq.com> +Colin Snyder <8csnyder@gmail.com> <47012605+colinsnyder@users.noreply.github.com> +Constanza Fierro +Daniel B Allan +Daniel da Silva +Daniel da Silva +Daniel Hrisca +Daniel J Farrell +Daniel Montes <53720019+Aerysv@users.noreply.github.com> +Daniel Müllner Daniel Müllner Daniel -Daniel Müllner dmuellner -Daniel Rasmussen drasmuss +Daniel Rasmussen +Daniel G. A. Smith +Daniel G. A. Smith +Dario Mory David Huard dhuard -David M Cooke cookedm -David Nicholson davidjn -David Ochoa ochoadavid -Davide Dal Bosco davidedalbosco <62077652+davidedalbosco@users.noreply.github.com> -Dawid Zych silenc3r -Dennis Zollo denniszollo -Derek Homeier Derek Homeier -Derek Homeier Derek Homeir -Derek Homeier Derek Homier -Derrick Williams derrick -Dmitriy Shalyga zuko3d -Dustan Levenstein dustanlevenstein <43019642+dustanlevenstein@users.noreply.github.com> -Ed Schofield edschofield -Egor Zindy zindy -Endolith -Erik M. Bray E. M. Bray -Erik M. Bray Erik Bray +David M Cooke +David Nicholson +David Ochoa +David Pitchford +Davide Dal Bosco <62077652+davidedalbosco@users.noreply.github.com> +Dawid Zych +Dennis Zollo +Derek Homeier +Derek Homeier +Derek Homeier +Derrick Williams +Dmitriy Shalyga +Dustan Levenstein <43019642+dustanlevenstein@users.noreply.github.com> +Dylan Cutler +Ed Schofield +Egor Zindy +Elliott M. Forney +Erik M. Bray +Erik M. Bray +Erik M. Bray Eric Fode Eric Fode -Eric Quintero e-q -Ernest N. Mamikonyan mamikony -Etienne Guesnet EGuesnet <51407514+EGuesnet@users.noreply.github.com> +Eric Quintero +Ernest N. Mamikonyan +Etienne Guesnet <51407514+EGuesnet@users.noreply.github.com> +Eva Jau Evgeni Burovski Evgeni Burovski -Evgeny Toder eltjpm -Fernando Perez Fernando Perez +Evgeny Toder +Fernando Perez +Filip Trojan +François Le Lay +Frank Breitling Friedrich Dunne dunneff Frederic Bastien Frederic -Gael Varoquaux GaelVaroquaux -Gerrit Holl Gerrit Holl -Giuseppe Venturini ggventurini -Golnaz Irannejad golnazir -Gopal Singh Meena gopalmeena -Greg Knoll gkBCCN -Greg Yang eulerreich -Greg Young gfyoung -Greg Young gfyoung +FX Coudert +Gael Varoquaux +Gerrit Holl +Gerrit Holl +Giuseppe Venturini +Golnaz Irannejad +Gopal Singh Meena +Greg Knoll +Greg Yang +Greg Young +Greg Young +Gregory R. Lee +Gregory R. Lee Guo Ci guoci -Han Genuit 87 -Han Genuit hangenuit@gmail.com -Han Genuit Han +Hameer Abbasi +Han Genuit Hanno Klemm hklemm -Hemil Desai hemildesai -Hiroyuki V. Yamazaki hvy -Hugo van Kemenade Hugo -Irvin Probst I--P -Isabela Presedo-Floyd isabela-pf -Gerhard Hobler hobler -Guillaume Peillex hippo91 -Jaime Fernandez Jaime Fernandez -Jaime Fernandez jaimefrio -Jaime Fernandez Jaime -Jakob Jakobson Jakob -Jakob Jacobson jakobjakobson13 <43045863+jakobjakobson13@users.noreply.github.com> -James Webber jamestwebber +Helder Oliveira +Hemil Desai +Hiroyuki V. Yamazaki +Hugo van Kemenade +I-Shen Leong +Inessa Pawson +Irvin Probst +Isabela Presedo-Floyd +Gerhard Hobler +Giannis Zapantis +Guillaume Peillex +Jack J. Woehr +Jaime Fernandez +Jaime Fernandez +Jaime Fernandez +Jamie Macey +Jakob Jakobson +Jakob Jakobson <43045863+jakobjakobson13@users.noreply.github.com> +James Bourbeau +James Webber +Jan Schlüter Jarrod Millman Jarrod Millman -Jason Grout Jason Grout -Jason King jason king -Jay Bourque jayvius -Jean Utke jutke -Jeffrey Yancey Jeff <3820914+jeffyancey@users.noreply.github.com> -Jeremy Lay jeremycl01 +Jason Grout +Jason King +Jay Bourque +Jean Utke +Jeff VanOss +Jeffrey Yancey <3820914+jeffyancey@users.noreply.github.com> +Jeremy Lay Jérémie du Boisberranger jeremiedbb <34657725+jeremiedbb@users.noreply.github.com> -Jerome Kelleher jeromekelleher -Johannes Hampp euronion <42553970+euronion@users.noreply.github.com> -Johannes Schönberger Johannes Schönberger -Johann Faouzi johann.faouzi -John Darbyshire <24256554+attack68@users.noreply.github.com> attack68 <24256554+attack68@users.noreply.github.com> -John Hagen johnthagen -John Kirkham jakirkham -Joseph Fox-Rabinovitz Joseph Fox-Rabinovitz -Joseph Fox-Rabinovitz Joseph Fox-Rabinovitz -Joseph Fox-Rabinovitz Mad Physicist -Joseph Martinot-Lagarde Joseph Martinot-Lagarde -Julian Taylor Julian Taylor -Julian Taylor Julian Taylor +Jérome Eertmans +Jerome Kelleher +Johannes Hampp <42553970+euronion@users.noreply.github.com> +Johannes Schönberger +Johann Faouzi +John Darbyshire <24256554+attack68@users.noreply.github.com> <24256554+attack68@users.noreply.github.com> +John Hagen +John Kirkham +John Kirkham +Joseph Fox-Rabinovitz +Joseph Fox-Rabinovitz +Joseph Fox-Rabinovitz +Joseph Martinot-Lagarde +Julian Taylor +Julian Taylor +Julian Taylor Julien Lhermitte Julien Lhermitte -Julien Schueller jschueller -Justus Magin keewis -Justus Magin Keewis -Kai Striega kai -Kai Striega kai-striega -Kai Striega kai-striega -Karan Dhir karan-dhir -Khaled Ben Abdallah Okuda KhaledTo +Julien Schueller +Justus Magin +Justus Magin +Kai Striega +Kai Striega +Kasia Leszek +Kasia Leszek <39829548+katleszek@users.noreply.github.com> +Karan Dhir +Keller Meier +Kevin Sheppard +Kevin Sheppard +Kerem Hallaç +Khaled Ben Abdallah Okuda Kiko Correoso kikocorreoso Kiko Correoso kikocorreoso -Konrad Kapp k_kapp@yahoo.com -Kriti Singh kritisingh1 -Kmol Yuan Yuan +Konrad Kapp +Kriti Singh +Kmol Yuan +Kumud Lakara <55556183+kumudlakara@users.noreply.github.com> Lars Buitinck Lars Buitinck Lars Buitinck Lars Buitinck -Lars Grüter Lars G -Luis Pedro Coelho Luis Pedro Coelho -Luke Zoltan Kelley lzkelley -Madhulika Jain Chambers madhulikajc <53166646+madhulikajc@users.noreply.github.com> -Magdalena Proszewska mpro -Magdalena Proszewska mproszewska <38814059+mproszewska@users.noreply.github.com> -Manoj Kumar MechCoder -Marcin Podhajski m-podhajski <36967358+m-podhajski@users.noreply.github.com> -Mark DePristo markdepristo -Mark Weissman m-d-w -Mark Wiebe Mark -Mark Wiebe Mark Wiebe -Mark Wiebe Mark Wiebe -Mark Wiebe Mark Wiebe -Martin Goodson martingoodson -Martin Reinecke mreineck -Martin Teichmann Martin Teichmann -Matt Hancock matt -Martino Sorbaro martinosorb -Mattheus Ueckermann empeeu -Matthew Harrigan MattHarrigan -Matti Picus mattip -Maximilian Konrad MLK97 -Melissa Weber Mendonça Melissa Weber Mendonca -Melissa Weber Mendonça melissawm +Lars Grüter +Lars Grüter +Leonardus Chen +Licht Takeuchi +Luis Pedro Coelho +Luke Zoltan Kelley +Madhulika Jain Chambers <53166646+madhulikajc@users.noreply.github.com> +Magdalena Proszewska +Magdalena Proszewska <38814059+mproszewska@users.noreply.github.com> +Manoj Kumar +Marcin Podhajski <36967358+m-podhajski@users.noreply.github.com> +Mark DePristo +Mark Weissman +Mark Wiebe +Mark Wiebe +Mark Wiebe +Mark Wiebe +Martin Goodson +Martin Reinecke +Martin Teichmann +Mary Conley +Matheus Vieira Portela +Mathieu Lamarre +Matías Ríos +Matt Ord +Matt Ord <55235095+Matt-Ord@users.noreply.github.com> +Matt Hancock +Martino Sorbaro +Mattheus Ueckermann +Matthew Harrigan +Matthias Bussonnier +Matti Picus +Maximilian Konrad +Melissa Weber Mendonça +Meltem Eren Copur Michael Behrisch behrisch Michael Droettboom mdroe -Michael K. Tran mtran -Michael Martin mmartin -Michael Schnaitter schnaitterm -Muhammad Kasim mfkasim91 -Masashi Kishimoto kishimoto-banana -Nathaniel J. Smith njsmith +Michael Dubravski +Michael Dubravski <41096057+mdubravski@users.noreply.github.com> +Michael Felt +Michael Hirsch +Michael K. Tran +Michael Martin +Michael Schnaitter +Michael Seifert +Michel Fruchart +Mike Toews +Mircea Akos Bruma +Mircea Akos Bruma +Mitchell Faas <35742861+Mitchell-Faas@users.noreply.github.com> +Muhammad Kasim +Masashi Kishimoto +Mukulikaa Parhari <60316606+Mukulikaa@users.noreply.github.com> +Nathaniel J. Smith Naveen Arunachalam naveenarun +Neil Girdhar +Nick Papior +Nicola Soranzo Nicolas Scheffer Nicolas Scheffer Nicholas A. Del Grosso nickdg +Nicholas McKibben Nick Minkyu Lee fivemok <9394929+fivemok@users.noreply.github.com> -Ondřej Čertík Ondrej Certik -Óscar Villellas Guillén ovillellas +Oliver Eberle +Ondřej Čertík +Óscar Villellas Guillén +Panos Mavrogiorgos Pat Miller patmiller -Paul Ivanov Paul Ivanov +Paul Ivanov +Paul Ivanov Paul YS Lee Paul -Paul Jacobson hpaulj -Pearu Peterson Pearu Peterson -Pete Peeradej Tanruangporn petetanru -Peter Bell peterbell10 -Peter J Cock peterjc +Paul Jacobson +Pearu Peterson +Pete Peeradej Tanruangporn +Peter Bell +Peter J Cock Phil Elson -Pierre GM pierregm +Pierre GM Pierre GM pierregm -Piotr Gaiński panpiort8 +Piotr Gaiński Piotr Gaiński Pan Jan Prabhu Ramachandran prabhu -Przemyslaw Bartosik przemb +Przemyslaw Bartosik +Raghuveer Devulapalli +Raghuveer Devulapalli <44766858+r-devulap@users.noreply.github.com> Rajas Rade lkdmttg7 -Ralf Gommers Ralf Gommers -Ralf Gommers rgommers -Rehas Sachdeva rehassachdeva -Ritta Narita RittaNarita -Riya Sharma ayir -Robert Kern Robert Kern -Robert LU RobberPhex +Rakesh Vasudevan +Ralf Gommers +Rehas Sachdeva +Ritta Narita +Riya Sharma +Robert Kern +Robert LU +Robert T. McGibbon +Roland Kaufmann +Roman Yurchak Ronan Lamy Ronan Lamy -Russell Hewett rhewett -Ryan Blakemore ryanblak -Sam Preston jspreston -Sam Radhakrishnan = <=> -Sam Radhakrishnan sam09 -Sanchez Gonzalez Alvaro alvarosg -Saullo Giovani saullogiovani +Russell Hewett +Ryan Blakemore +Ryan Polley +Ryan Soklaski +Ryan Soklaski +Sabrina Simao +Sabrina Simao SabrinaSimao +Sam Preston +Sam Radhakrishnan = <=> # committed without an email address +Samesh Lakhotia +Samesh Lakhotia <43701530+sameshl@users.noreply.github.com> +Sami Salonen +Sanchez Gonzalez Alvaro +Saullo Giovani Saurabh Mehta -Sebastian Berg seberg -Sergei Vorfolomeev vorfol <39548292+vorfol@users.noreply.github.com> -Shekhar Prasad Rajak shekharrajak -Shota Kawabuchi skwbc -Siavash Eliasi siavashserver -Simon Gasse sgasse -Søren Rasmussen sorenrasmussenai <47032123+sorenrasmussenai@users.noreply.github.com> -Stefan Behnel scoder -Stefan van der Walt Stefan van der Walt -Stefan van der Walt Stefan van der Walt -Stephan Hoyer Stephan Hoyer +Sebastian Berg +Sergei Vorfolomeev <39548292+vorfol@users.noreply.github.com> +Shekhar Prasad Rajak +Shen Zhou +Shota Kawabuchi +Siavash Eliasi +Simon Conseil +Simon Gasse +Simon Gasse +Søren Rasmussen <47032123+sorenrasmussenai@users.noreply.github.com> +Spencer Hill +Stefan Behnel +Stefan van der Walt +Stefan van der Walt +Stephan Hoyer +Stephan Hoyer +Steve Stagg Steven J Kern -Stuart Archibald stuartarchibald -SuryaChand P Surya P -SuryaChand P psschand -Takanori Hirano takanori-pskq -Takanori H takanori-pskq -Thomas A Caswell Thomas A Caswell -Tim Cera tim cera -Tim Teichmann tteichmann -Tim Teichmann tteichmann <44259103+tteichmann@users.noreply.github.com> -Tirth Patel tirthasheshpatel -Tom Boyd pezcore -Tom Poole tpoole -Tony LaTorre tlatorre -Travis Oliphant Travis E. Oliphant -Travis Oliphant Travis Oliphant -Valentin Haenel Valentin Haenel -Rakesh Vasudevan vrakesh -Vrinda Narayan vrindaaa <48102157+vrindaaa@users.noreply.github.com> -Wansoo Kim marload -Warren Weckesser Warren Weckesser -Weitang Li wtli@Dirac -Weitang Li wtli -Wendell Smith Wendell Smith -Wim Glenn wim glenn -Wojtek Ruszczewski wrwrwr -Yogesh Raisinghani raisinghanii <46864533+raisinghanii@users.noreply.github.com> -Yuji Kanagawa kngwyu -Yury Kirienko kirienko -Zac Hatfield-Dodds Zac-HD -Zixu Zhao ZZhaoTireless -Ziyan Zhou Ziyan -Zieji Pohz jpoh -Zieji Pohz zjpoh -Zieji Pohz Zijie (ZJ) Poh <8103276+zjpoh@users.noreply.github.com> -Zolisa Bleki zoj613 <44142765+zoj613@users.noreply.github.com> -Zolisa Bleki RedRuM <44142765+zoj613@users.noreply.github.com> -luzpaz luz.paz -luzpaz luzpaz -spacescientist spacescientist +Stuart Archibald +Stuart Archibald +SuryaChand P +Takanori Hirano +Thomas A Caswell +Thomas Kluyver +Thomas Orgis +Tim Cera +Tim Teichmann +Tim Teichmann <44259103+tteichmann@users.noreply.github.com> +Tirth Patel +Tobias Pitters +Tobias Pitters <31857876+CloseChoice@users.noreply.github.com> +Tobias Uelwer +Tom Boyd +Tom Poole +Tong Zou +Tony LaTorre +Toshiki Kataoka +Travis Oliphant +Travis Oliphant +Travis Oliphant +Valentin Haenel +Valentin Haenel +Varun Nayyar +Vrinda Narayan +Vrinda Narayan <48102157+vrindaaa@users.noreply.github.com> +Wansoo Kim +Warren Weckesser +Weitang Li +Wendell Smith +William Spotz +Wim Glenn +Wojtek Ruszczewski +Wojciech Rzadkowski <33913808+wrzadkow@users.noreply.github.com> +Yang Hau +Yang Hau +Yogesh Raisinghani <46864533+raisinghanii@users.noreply.github.com> +Yu Feng +Yuji Kanagawa +Yury Kirienko +Zac Hatfield-Dodds +Zé Vinícius +Zixu Zhao +Ziyan Zhou +Zieji Pohz +Zieji Pohz <8103276+zjpoh@users.noreply.github.com> +Zolboo Erdenebaatar +Zolisa Bleki <44142765+zoj613@users.noreply.github.com> diff --git a/.travis.yml b/.travis.yml index 6b7585a56be7..1486bbb885fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,10 @@ addons: # Speedup builds, particularly when USE_CHROOT=1 - eatmydata +# Disable clone depth +git: + depth: false + cache: directories: - $HOME/.cache/pip diff --git a/INSTALL.rst.txt b/INSTALL.rst.txt index 5ee97d7901a0..1bc97c4b5f86 100644 --- a/INSTALL.rst.txt +++ b/INSTALL.rst.txt @@ -14,7 +14,7 @@ Prerequisites Building NumPy requires the following installed software: -1) Python__ 3.6.x or newer. +1) Python__ 3.7.x or newer. Please note that the Python development headers also need to be installed, e.g., on Debian/Ubuntu one needs to install both `python3` and diff --git a/LICENSE.txt b/LICENSE.txt index 8ce64521862e..4723d4ea009e 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2005-2020, NumPy Developers. +Copyright (c) 2005-2021, NumPy Developers. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt index 00b7473777ca..26c7a7829361 100644 --- a/LICENSES_bundled.txt +++ b/LICENSES_bundled.txt @@ -15,3 +15,8 @@ Name: dragon4 Files: numpy/core/src/multiarray/dragon4.c License: MIT For license text, see numpy/core/src/multiarray/dragon4.c + +Name: libdivide +Files: numpy/core/include/numpy/libdivide/* +License: Zlib + For license text, see numpy/core/include/numpy/libdivide/LICENSE.txt diff --git a/MANIFEST.in b/MANIFEST.in index 35eb05a61619..8ec62123b998 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,16 +4,16 @@ # data, etc files to distribution (*for installation*). # Avoid using MANIFEST.in for that. # -include MANIFEST.in -include pyproject.toml -include pytest.ini -include *.txt -include README.md -include site.cfg.example -include runtests.py -include tox.ini -include .coveragerc -include test_requirements.txt +# Files in top-level directory: +include *.* +# Exclude license file that we append to the main license when running +# `python setup.py sdist`. And exclude generated files in repo root. +exclude LICENSES_bundled.txt +exclude .* +exclude azure-*.yml + +# Sub-directories. Included are: numpy/, doc/, benchmarks/, tools/ +include numpy/_version.py recursive-include numpy/random *.pyx *.pxd *.pyx.in *.pxd.in include numpy/py.typed include numpy/random/include/* @@ -45,6 +45,3 @@ prune benchmarks/numpy # Exclude generated files prune */__pycache__ global-exclude *.pyc *.pyo *.pyd *.swp *.bak *~ -# Exclude license file that we append to the main license when running -# `python setup.py sdist` -exclude LICENSES_bundled.txt diff --git a/README.md b/README.md index 9271a5d28872..88c1151a0f89 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,4 @@ -# NumPy - -[![Travis CI](https://img.shields.io/travis/com/numpy/numpy/master?label=Travis%20CI)]( - https://travis-ci.com/github/numpy/numpy) -[![Azure](https://dev.azure.com/numpy/numpy/_apis/build/status/azure-pipeline%20numpy.numpy)]( - https://dev.azure.com/numpy/numpy/_build/latest?definitionId=5) -[![codecov](https://codecov.io/gh/numpy/numpy/branch/master/graph/badge.svg)]( - https://codecov.io/gh/numpy/numpy) +# NumPy NumPy is the fundamental package needed for scientific computing with Python. @@ -26,10 +19,7 @@ It provides: Testing: -- NumPy versions ≥ 1.15 require `pytest` -- NumPy versions < 1.15 require `nose` - -Tests can then be run after installation with: +NumPy requires `pytest`. Tests can then be run after installation with: python -c 'import numpy; numpy.test()' @@ -37,13 +27,39 @@ Tests can then be run after installation with: Call for Contributions ---------------------- -NumPy appreciates help from a wide range of different backgrounds. -Work such as high level documentation or website improvements are valuable -and we would like to grow our team with people filling these roles. -Small improvements or fixes are always appreciated and issues labeled as easy -may be a good starting point. -If you are considering larger contributions outside the traditional coding work, -please contact us through the mailing list. +The NumPy project welcomes your expertise and enthusiasm! + +Small improvements or fixes are always appreciated; issues labeled as "good +first issue" may be a good starting point. If you are considering larger +contributions to the source code, please contact us through the [mailing +list](https://mail.python.org/mailman/listinfo/numpy-discussion) first. + +Writing code isn’t the only way to contribute to NumPy. You can also: +- review pull requests +- triage issues +- develop tutorials, presentations, and other educational materials +- maintain and improve [our website](https://github.com/numpy/numpy.org) +- develop graphic design for our brand assets and promotional materials +- translate website content +- help with outreach and onboard new contributors +- write grant proposals and help with other fundraising efforts + +If you’re unsure where to start or how your skills fit in, reach out! You can +ask on the mailing list or here, on GitHub, by opening a new issue or leaving a +comment on a relevant issue that is already open. + +Our preferred channels of communication are all public, but if you’d like to +speak to us in private first, contact our community coordinators at +numpy-team@googlegroups.com or on Slack (write numpy-team@googlegroups.com for +an invitation). + +We also have a biweekly community call, details of which are announced on the +mailing list. You are very welcome to join. + +If you are new to contributing to open source, [this +guide](https://opensource.guide/how-to-contribute/) helps explain why, what, +and how to successfully get involved. + [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f8773dc36abc..14a59e8800af 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,7 +3,7 @@ trigger: batch: False branches: include: - - master + - main - maintenance/* @@ -18,6 +18,54 @@ stages: - stage: InitialTests jobs: + - job: Lint + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + pool: + vmImage: 'ubuntu-18.04' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + addToPath: true + architecture: 'x64' + - script: >- + python -m pip install -r linter_requirements.txt + displayName: 'Install tools' + # pip 21.1 emits a pile of garbage messages to annoy users :) + # failOnStderr: true + - script: | + python tools/linter.py --branch origin/$(System.PullRequest.TargetBranch) + displayName: 'Run Lint Checks' + failOnStderr: true + # Native build is based on gcc flag `-march=native` + - job: Linux_baseline_native + pool: + vmImage: 'ubuntu-20.04' + steps: + - script: | + if ! `gcc 2>/dev/null`; then + sudo apt install gcc + fi + sudo apt install python3 + sudo apt install python3-dev + # python3 has no setuptools, so install one to get us going + python3 -m pip install --user --upgrade pip 'setuptools<49.2.0' + python3 -m pip install --user -r test_requirements.txt + displayName: 'install python/requirements' + - script: | + python3 runtests.py --show-build-log --cpu-baseline=native --cpu-dispatch=none \ + --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml + displayName: 'Run native baseline Build / Tests' + - task: PublishTestResults@2 + condition: succeededOrFailed() + inputs: + testResultsFiles: '**/test-*.xml' + failTaskOnFailedTests: true + testRunTitle: 'Publish test results for baseline/native' + +- stage: ComprehensiveTests + jobs: + - job: WindowsFast pool: @@ -32,19 +80,13 @@ stages: steps: - template: azure-steps-windows.yml - -- stage: ComprehensiveTests - jobs: - - - job: Linux_Python_38_32bit_full_with_asserts pool: - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-20.04' steps: - script: | - docker pull quay.io/pypa/manylinux2010_i686 docker run -v $(pwd):/numpy -e CFLAGS="-msse2 -std=c99 -UNDEBUG" \ - -e F77=gfortran-5 -e F90=gfortran-5 quay.io/pypa/manylinux2010_i686 \ + -e F77=gfortran-5 -e F90=gfortran-5 quay.io/pypa/manylinux2010_i686:2021-02-28-1f32361 \ /bin/bash -xc "cd numpy && \ /opt/python/cp38-cp38/bin/python -mvenv venv &&\ source venv/bin/activate && \ @@ -155,7 +197,6 @@ stages: - script: python setup.py build -j 4 build_ext --inplace install displayName: 'Build NumPy without OpenBLAS and new casting' env: - NPY_USE_NEW_CASTINGIMPL: 1 BLAS: None LAPACK: None ATLAS: None @@ -166,7 +207,7 @@ stages: - script: python -m pip install matplotlib displayName: 'Install matplotlib before refguide run' - script: python runtests.py -g --refguide-check - displayName: 'Run Refuide Check' + displayName: 'Run Refguide Check' condition: eq(variables['USE_OPENBLAS'], '1') - script: python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml displayName: 'Run Full NumPy Test Suite' @@ -206,6 +247,11 @@ stages: PYTHON_ARCH: 'x64' TEST_MODE: full BITS: 64 + #PyPy37-64bit-full: + # PYTHON_VERSION: 'PyPy3.7' + # PYTHON_ARCH: 'x64' + # TEST_MODE: fast + # BITS: 64 Python38-32bit-fast: PYTHON_VERSION: '3.8' PYTHON_ARCH: 'x86' @@ -238,6 +284,7 @@ stages: vmImage: 'ubuntu-18.04' steps: - script: | + sudo apt update sudo apt install python3.7 sudo apt install python3.7-dev if ! `gcc-4.8 2>/dev/null`; then @@ -258,3 +305,4 @@ stages: failTaskOnFailedTests: true testRunTitle: 'Publish test results for gcc 4.8' + diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml index 28762f5d9528..6a69db7539b3 100644 --- a/azure-steps-windows.yml +++ b/azure-steps-windows.yml @@ -6,7 +6,7 @@ steps: architecture: $(PYTHON_ARCH) condition: not(contains(variables['PYTHON_VERSION'], 'PyPy')) - powershell: | - $url = "http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-win32.zip" + $url = "http://buildbot.pypy.org/nightly/py3.7/pypy-c-jit-latest-win64.zip" $output = "pypy.zip" $wc = New-Object System.Net.WebClient $wc.DownloadFile($url, $output) diff --git a/benchmarks/README.rst b/benchmarks/README.rst index f56c253399ad..2700e95e7ab2 100644 --- a/benchmarks/README.rst +++ b/benchmarks/README.rst @@ -50,7 +50,7 @@ Compare change in benchmark results to another version/commit/branch:: python runtests.py --bench-compare v1.6.2 bench_core python runtests.py --bench-compare 8bf4e9b bench_core - python runtests.py --bench-compare master bench_core + python runtests.py --bench-compare main bench_core All of the commands above display the results in plain text in the console, and the results are not saved for comparison with diff --git a/benchmarks/asv_compare.conf.json.tpl b/benchmarks/asv_compare.conf.json.tpl index 1f339077c66d..03d13d985c8d 100644 --- a/benchmarks/asv_compare.conf.json.tpl +++ b/benchmarks/asv_compare.conf.json.tpl @@ -78,7 +78,9 @@ "build_command" : [ "python setup.py build {numpy_build_options}", - "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}" + // pip ignores '--global-option' when pep517 is enabled, we also enabling pip verbose to + // be reached from asv `--verbose` so we can verify the build options. + "PIP_NO_BUILD_ISOLATION=false python {build_dir}/benchmarks/asv_pip_nopep517.py -v {numpy_global_options} --no-deps --no-index -w {build_cache_dir} {build_dir}" ], // The commits after which the regression search in `asv publish` // should start looking for regressions. Dictionary whose keys are diff --git a/benchmarks/asv_pip_nopep517.py b/benchmarks/asv_pip_nopep517.py new file mode 100644 index 000000000000..9ba165493085 --- /dev/null +++ b/benchmarks/asv_pip_nopep517.py @@ -0,0 +1,15 @@ +""" +This file is used by asv_compare.conf.json.tpl. +""" +import subprocess, sys +# pip ignores '--global-option' when pep517 is enabled therefore we disable it. +cmd = [sys.executable, '-mpip', 'wheel', '--no-use-pep517'] +try: + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, universal_newlines=True) +except Exception as e: + output = str(e.output) +if "no such option" in output: + print("old version of pip, escape '--no-use-pep517'") + cmd.pop() + +subprocess.run(cmd + sys.argv[1:]) diff --git a/benchmarks/benchmarks/__init__.py b/benchmarks/benchmarks/__init__.py index e4193cf0582f..7b9f1d3e688d 100644 --- a/benchmarks/benchmarks/__init__.py +++ b/benchmarks/benchmarks/__init__.py @@ -1 +1,53 @@ from . import common +import sys +import os + +def show_cpu_features(): + from numpy.lib.utils import _opt_info + info = _opt_info() + info = "NumPy CPU features: " + (info if info else 'nothing enabled') + # ASV wrapping stdout & stderr, so we assume having a tty here + if 'SHELL' in os.environ and sys.platform != 'win32': + # to avoid the red color that imposed by ASV + print(f"\033[33m{info}\033[0m") + else: + print(info) + +def dirty_lock(lock_name, lock_on_count=1): + # this lock occurred before each round to avoid duplicate printing + if not hasattr(os, "getppid"): + return False + ppid = os.getppid() + if not ppid or ppid == os.getpid(): + # not sure if this gonna happen, but ASV run each round in + # a separate process so the lock should be based on the parent + # process id only + return False + lock_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), "..", "env", lock_name) + ) + # ASV load the 'benchmark_dir' to discovering the available benchmarks + # the issue here is ASV doesn't capture any strings from stdout or stderr + # during this stage so we escape it and lock on the second increment + try: + with open(lock_path, 'a+') as f: + f.seek(0) + count, _ppid = (f.read().split() + [0, 0])[:2] + count, _ppid = int(count), int(_ppid) + if _ppid == ppid: + if count >= lock_on_count: + return True + count += 1 + else: + count = 0 + f.seek(0) + f.truncate() + f.write(f"{str(count)} {str(ppid)}") + except IOError: + pass + return False + + +# FIXME: there's no official way to provide extra information to the test log +if not dirty_lock("print_cpu_features.lock"): + show_cpu_features() diff --git a/benchmarks/benchmarks/bench_app.py b/benchmarks/benchmarks/bench_app.py index bee95c201172..d22aa2e09604 100644 --- a/benchmarks/benchmarks/bench_app.py +++ b/benchmarks/benchmarks/bench_app.py @@ -70,8 +70,8 @@ def maxes_of_dots(self, arrays): Arrays must agree only on the first dimension. - For numpy it a join benchmark of dot products and max() - on a set of arrays. + Numpy uses this as a simultaneous benchmark of 1) dot products + and 2) max(, axis=). """ feature_scores = ([0] * len(arrays)) for (i, sd) in enumerate(arrays): diff --git a/benchmarks/benchmarks/bench_core.py b/benchmarks/benchmarks/bench_core.py index 0c2a18c15333..30647f4b850f 100644 --- a/benchmarks/benchmarks/bench_core.py +++ b/benchmarks/benchmarks/bench_core.py @@ -93,6 +93,12 @@ def time_triu_l10x10(self): def time_tril_l10x10(self): np.tril(self.l10x10) + def time_triu_indices_500(self): + np.triu_indices(500) + + def time_tril_indices_500(self): + np.tril_indices(500) + class Temporaries(Benchmark): def setup(self): @@ -136,7 +142,7 @@ class CountNonzero(Benchmark): params = [ [1, 2, 3], [100, 10000, 1000000], - [bool, int, str, object] + [bool, np.int8, np.int16, np.int32, np.int64, str, object] ] def setup(self, numaxes, size, dtype): @@ -165,6 +171,9 @@ def setup(self, dtype): def time_packbits(self, dtype): np.packbits(self.d) + def time_packbits_little(self, dtype): + np.packbits(self.d, bitorder="little") + def time_packbits_axis0(self, dtype): np.packbits(self.d2, axis=0) diff --git a/benchmarks/benchmarks/bench_function_base.py b/benchmarks/benchmarks/bench_function_base.py index b1e59274986c..062843d10cc0 100644 --- a/benchmarks/benchmarks/bench_function_base.py +++ b/benchmarks/benchmarks/bench_function_base.py @@ -47,6 +47,8 @@ class Median(Benchmark): def setup(self): self.e = np.arange(10000, dtype=np.float32) self.o = np.arange(10001, dtype=np.float32) + self.tall = np.random.random((10000, 20)) + self.wide = np.random.random((20, 10000)) def time_even(self): np.median(self.e) @@ -66,6 +68,12 @@ def time_even_small(self): def time_odd_small(self): np.median(self.o[:500], overwrite_input=True) + def time_tall(self): + np.median(self.tall, axis=-1) + + def time_wide(self): + np.median(self.wide, axis=0) + class Percentile(Benchmark): def setup(self): diff --git a/benchmarks/benchmarks/bench_lib.py b/benchmarks/benchmarks/bench_lib.py index c22ceaa5e42f..f7884cd6c309 100644 --- a/benchmarks/benchmarks/bench_lib.py +++ b/benchmarks/benchmarks/bench_lib.py @@ -53,6 +53,7 @@ def setup(self, shape, pad_width, mode): def time_pad(self, shape, pad_width, mode): np.pad(self.array, pad_width, mode) + class Nan(Benchmark): """Benchmarks for nan functions""" @@ -113,3 +114,26 @@ def time_nanquantile(self, array_size, percent_nans): def time_nanpercentile(self, array_size, percent_nans): np.nanpercentile(self.arr, q=50) + + +class Unique(Benchmark): + """Benchmark for np.unique with np.nan values.""" + + param_names = ["array_size", "percent_nans"] + params = [ + # sizes of the 1D arrays + [200, int(2e5)], + # percent of np.nan in arrays + [0, 0.1, 2., 50., 90.], + ] + + def setup(self, array_size, percent_nans): + np.random.seed(123) + # produce a randomly shuffled array with the + # approximate desired percentage np.nan content + base_array = np.random.uniform(size=array_size) + base_array[base_array < percent_nans / 100.] = np.nan + self.arr = base_array + + def time_unique(self, array_size, percent_nans): + np.unique(self.arr) diff --git a/benchmarks/benchmarks/bench_linalg.py b/benchmarks/benchmarks/bench_linalg.py index a72cccb5f7ed..5ed5b6eecd6d 100644 --- a/benchmarks/benchmarks/bench_linalg.py +++ b/benchmarks/benchmarks/bench_linalg.py @@ -91,8 +91,8 @@ def setup(self, op, typename): # check that dtype is supported at all try: self.func(self.a[:2, :2]) - except TypeError: - raise NotImplementedError() + except TypeError as e: + raise NotImplementedError() from e def time_op(self, op, typename): self.func(self.a) diff --git a/benchmarks/benchmarks/bench_ufunc.py b/benchmarks/benchmarks/bench_ufunc.py index 9f45a72575ff..b036581e1aae 100644 --- a/benchmarks/benchmarks/bench_ufunc.py +++ b/benchmarks/benchmarks/bench_ufunc.py @@ -134,6 +134,23 @@ def time_less_than_scalar2(self, dtype): (self.d < 1) +class CustomScalarFloorDivideInt(Benchmark): + params = (np.sctypes['int'] + np.sctypes['uint'], [8, -8, 43, -43]) + param_names = ['dtype', 'divisors'] + + def setup(self, dtype, divisor): + if dtype in np.sctypes['uint'] and divisor < 0: + raise NotImplementedError( + "Skipping test for negative divisor with unsigned type") + + iinfo = np.iinfo(dtype) + self.x = np.random.randint( + iinfo.min, iinfo.max, size=10000, dtype=dtype) + + def time_floor_divide_int(self, dtype, divisor): + self.x // divisor + + class Scalar(Benchmark): def setup(self): self.x = np.asarray(1.0) diff --git a/benchmarks/benchmarks/bench_avx.py b/benchmarks/benchmarks/bench_ufunc_strides.py similarity index 87% rename from benchmarks/benchmarks/bench_avx.py rename to benchmarks/benchmarks/bench_ufunc_strides.py index 82866c17077f..213ff0020293 100644 --- a/benchmarks/benchmarks/bench_avx.py +++ b/benchmarks/benchmarks/bench_ufunc_strides.py @@ -2,7 +2,7 @@ import numpy as np -avx_ufuncs = ['sin', +unary_ufuncs = ['sin', 'cos', 'exp', 'log', @@ -20,24 +20,26 @@ 'isinf', 'signbit'] stride = [1, 2, 4] +stride_out = [1, 2, 4] dtype = ['f', 'd'] -class AVX_UFunc(Benchmark): - params = [avx_ufuncs, stride, dtype] - param_names = ['avx_based_ufunc', 'stride', 'dtype'] +class Unary(Benchmark): + params = [unary_ufuncs, stride, stride_out, dtype] + param_names = ['ufunc', 'stride_in', 'stride_out', 'dtype'] timeout = 10 - def setup(self, ufuncname, stride, dtype): + def setup(self, ufuncname, stride, stride_out, dtype): np.seterr(all='ignore') try: self.f = getattr(np, ufuncname) except AttributeError: - raise NotImplementedError() + raise NotImplementedError(f"No ufunc {ufuncname} found") from None N = 10000 self.arr = np.ones(stride*N, dtype) + self.arr_out = np.empty(stride_out*N, dtype) - def time_ufunc(self, ufuncname, stride, dtype): - self.f(self.arr[::stride]) + def time_ufunc(self, ufuncname, stride, stride_out, dtype): + self.f(self.arr[::stride], self.arr_out[::stride_out]) class AVX_UFunc_log(Benchmark): params = [stride, dtype] @@ -66,7 +68,7 @@ def setup(self, ufuncname, dtype, stride): try: self.f = getattr(np, ufuncname) except AttributeError: - raise NotImplementedError() + raise NotImplementedError(f"No ufunc {ufuncname} found") from None N = 10000 self.arr1 = np.array(np.random.rand(stride*N), dtype=dtype) self.arr2 = np.array(np.random.rand(stride*N), dtype=dtype) @@ -107,7 +109,7 @@ def setup(self, bfuncname, stride, dtype): try: self.f = getattr(np, bfuncname) except AttributeError: - raise NotImplementedError() + raise NotImplementedError(f"No bfunc {bfuncname} found") from None N = 10000 self.arr1 = np.ones(stride*N, dtype) self.arr2 = np.ones(stride*N, dtype) @@ -130,7 +132,7 @@ def setup(self, bfuncname, stride, dtype): try: self.f = getattr(np, bfuncname) except AttributeError: - raise NotImplementedError() + raise NotImplementedError(f"No bfunc {bfuncname} found") from None N = 10000 self.arr1 = np.ones(stride*N, dtype) diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py index b65cc5fd212a..0c40e85b0612 100644 --- a/benchmarks/benchmarks/common.py +++ b/benchmarks/benchmarks/common.py @@ -14,14 +14,14 @@ # time-consuming functions (ufunc, linalg, etc) nxs, nys = 100, 100 -# a set of interesting types to test +# a list of interesting types to test TYPES1 = [ 'int16', 'float16', 'int32', 'float32', 'int64', 'float64', 'complex64', 'longfloat', 'complex128', ] -if 'complex256' in numpy.typeDict: +if 'complex256' in numpy.sctypeDict: TYPES1.append('complex256') diff --git a/branding/logo/logoguidelines.md b/branding/logo/logoguidelines.md index c674a9b37763..0c37e3dd455e 100644 --- a/branding/logo/logoguidelines.md +++ b/branding/logo/logoguidelines.md @@ -4,7 +4,7 @@ These guidelines are meant to help keep the NumPy logo consistent and recognizab The primary logo is the horizontal option (logomark and text next to each other) and the secondary logo is the stacked version (logomark over text). I’ve also provided the logomark on its own (meaning it doesn’t have text). When in doubt, it’s preferable to use primary or secondary options over the logomark alone. ## Color -The full color options are a combo of Maximum Blue (#4DABCF) and Han Blue (#4D77CF), while light options are white (#FFFFFF) and dark options Warm Black (#013243). +The full color options are a combo of two shades of blue, rgb(77, 171, 207) and rgb(77, 119, 207), while light options are rgb(255, 255, 255) and dark options are rgb(1, 50, 67). Whenever possible, use the full color logos. One color logos (light or dark) are to be used when full color will not have enough contrast, usually when logos must be on colored backgrounds. @@ -15,4 +15,4 @@ Please do not make the primary logo smaller than 50px wide, secondary logo small A few other notes to keep in mind when using the logo: - Make sure to scale the logo proportionally. - Maintain a good amount of space around the logo. Don’t let it overlap with text, images, or other elements. -- Do not try and recreate or modify the logo. For example, do not use the logomark and then try to write NumPy in another font. \ No newline at end of file +- Do not try and recreate or modify the logo. For example, do not use the logomark and then try to write NumPy in another font. diff --git a/branding/logo/logomark/numpylogoicon.svg b/branding/logo/logomark/numpylogoicon.svg index 4fef2a9c8617..50810223b355 100644 --- a/branding/logo/logomark/numpylogoicon.svg +++ b/branding/logo/logomark/numpylogoicon.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/branding/logo/primary/numpylogo.svg b/branding/logo/primary/numpylogo.svg index e5791dc3e666..63d61c50f6a0 100644 --- a/branding/logo/primary/numpylogo.svg +++ b/branding/logo/primary/numpylogo.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/branding/logo/secondary/numpylogo2.svg b/branding/logo/secondary/numpylogo2.svg index 76b076beb74d..20385487c94e 100644 --- a/branding/logo/secondary/numpylogo2.svg +++ b/branding/logo/secondary/numpylogo2.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/doc/DISTUTILS.rst.txt b/doc/DISTUTILS.rst.txt index c58a423c0bfb..539a3b9c121a 100644 --- a/doc/DISTUTILS.rst.txt +++ b/doc/DISTUTILS.rst.txt @@ -472,7 +472,7 @@ by these modules during the build process are ready to be compiled. This form of generic typing is also supported for C header files (preprocessed to produce ``.h`` files). -.. _conv_template.py: https://github.com/numpy/numpy/blob/master/numpy/distutils/conv_template.py +.. _conv_template.py: https://github.com/numpy/numpy/blob/main/numpy/distutils/conv_template.py Useful functions in ``numpy.distutils.misc_util`` ------------------------------------------------- diff --git a/doc/HOWTO_RELEASE.rst.txt b/doc/HOWTO_RELEASE.rst.txt index 5a9c4d5052b4..9af58dd24e96 100644 --- a/doc/HOWTO_RELEASE.rst.txt +++ b/doc/HOWTO_RELEASE.rst.txt @@ -18,7 +18,7 @@ Source tree NumPy Docs ---------- -- https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt +- https://github.com/numpy/numpy/blob/main/doc/HOWTO_RELEASE.rst.txt SciPy.org wiki @@ -35,7 +35,7 @@ Supported platforms and versions ================================ :ref:`NEP 29 ` outlines which Python versions are supported; For the first half of 2020, this will be Python >= 3.6. We test -NumPy against all these versions every time we merge code to master. Binary +NumPy against all these versions every time we merge code to main. Binary installers may be available for a subset of these versions (see below). OS X @@ -185,7 +185,7 @@ A typical release schedule is one beta, two release candidates and a final release. It's best to discuss the timing on the mailing list first, in order for people to get their commits in on time, get doc wiki edits merged, etc. After a date is set, create a new maintenance/x.y.z branch, add new empty -release notes for the next version in the master branch and update the Trac +release notes for the next version in the main branch and update the Trac Milestones. @@ -258,10 +258,8 @@ Check the release notes Use `towncrier`_ to build the release note and commit the changes. This will remove all the fragments from ``doc/release/upcoming_changes`` and add ``doc/release/-note.rst``. -Note that currently towncrier must be installed from its master branch as the -last release (19.2.0) is outdated. - towncrier --version "" + towncrier build --version "" git commit -m"Create release note" Check that the release notes are up-to-date. @@ -275,7 +273,7 @@ following: - for SciPy, supported NumPy version(s) - outlook for the near future -.. _towncrier: https://github.com/hawkowl/towncrier +.. _towncrier: https://pypi.org/project/towncrier/ Update the release status and create a release "tag" @@ -336,8 +334,8 @@ to public keyservers, with a command such as:: gpg --send-keys -Update the version of the master branch ---------------------------------------- +Update the version of the main branch +------------------------------------- Increment the release number in setup.py. Release candidates should have "rc1" (or "rc2", "rcN") appended to the X.Y.Z format. @@ -460,6 +458,15 @@ The scipy.org should be a PR at https://github.com/scipy/scipy.org. The file that needs modification is ``www/index.rst``. Search for ``News``. +Update oldest-supported-numpy +----------------------------- +If this release is the first one to support a new Python version, or the first +to provide wheels for a new platform or PyPy version, the version pinnings +in https://github.com/scipy/oldest-supported-numpy should be updated. +Either submit a PR with changes to ``setup.cfg`` there, or open an issue with +info on needed changes. + + Announce to the lists --------------------- The release should be announced on the mailing lists of @@ -483,5 +490,5 @@ After the final release is announced, a few administrative tasks are left to be done: - Forward port changes in the release branch to release notes and release - scripts, if any, to master branch. + scripts, if any, to main branch. - Update the Milestones in Trac. diff --git a/doc/Makefile b/doc/Makefile index dd63702de1bd..68d496389e84 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -49,7 +49,7 @@ help: @echo " show to show the html output in a browser" clean: - -rm -rf build/* + -rm -rf build/* find . -name generated -type d -prune -exec rm -rf "{}" ";" gitwash-update: @@ -66,7 +66,7 @@ gitwash-update: # Build the current numpy version, and extract docs from it. # We have to be careful of some issues: -# +# # - Everything must be done using the same Python version # - We must use eggs (otherwise they might override PYTHONPATH on import). # - Different versions of easy_install install to different directories (!) @@ -80,8 +80,7 @@ UPLOAD_DIR=/srv/docs_scipy_org/doc/numpy-$(RELEASE) DIST_VARS=SPHINXBUILD="LANG=C PYTHONPATH=$(INSTALL_PPH) python$(PYVER) `which sphinx-build`" PYTHON="PYTHONPATH=$(INSTALL_PPH) python$(PYVER)" NUMPYVER:=$(shell $(PYTHON) -c "import numpy; print(numpy.version.git_revision[:10])" 2>/dev/null) -GITVER ?= $(shell cd ..; $(PYTHON) -c "from setup import git_version; \ - print(git_version()[:10])") +GITVER ?= $(shell cd ..; $(PYTHON) -c "import versioneer as v; print(v.get_versions()['full-revisionid'][:10])") version-check: ifeq "$(GITVER)" "Unknown" @@ -163,7 +162,7 @@ endif @echo " " @echo in build/merge/index.html, @echo then \"git commit\", \"git push\" - + #------------------------------------------------------------------------------ # Basic Sphinx generation rules for different formats diff --git a/doc/RELEASE_WALKTHROUGH.rst.txt b/doc/RELEASE_WALKTHROUGH.rst.txt index 733f681af704..4fbc7af1c6f4 100644 --- a/doc/RELEASE_WALKTHROUGH.rst.txt +++ b/doc/RELEASE_WALKTHROUGH.rst.txt @@ -29,7 +29,7 @@ appended to ``doc/release/1.19.0-notes.rst`` for patch release, though not for new releases like ``1.19.0``, as the changelogs for ``*.0`` releases tend to be excessively long. The ``doc/source/release.rst`` file should also be updated with a link to the new release notes. These changes should be committed to the -maintenance branch, and later will be forward ported to master. The changelog +maintenance branch, and later will be forward ported to main. The changelog should be reviewed for name duplicates or short names and the ``.mailmap`` file updated if needed. @@ -252,6 +252,10 @@ work:: $ firefox doc/build/merge/index.html +Update the stable link:: + + $ ln -sfn 1.19 stable + Once everything seems satisfactory, commit and upload the changes:: $ pushd doc/build/merge @@ -288,7 +292,7 @@ python-announce-list is BCC so that replies will not be sent to that list. Post-Release Tasks ------------------ -Checkout master and forward port the documentation changes:: +Checkout main and forward port the documentation changes:: $ git checkout -b post-1.19.0-release-update $ git checkout maintenance/1.19.x doc/source/release/1.19.0-notes.rst @@ -297,7 +301,7 @@ Checkout master and forward port the documentation changes:: $ gvim doc/source/release.rst # Add link to new notes $ git add doc/changelog/1.19.0-changelog.rst doc/source/release/1.19.0-notes.rst $ git status # check status before commit - $ git commit -a -m"REL: Update master after 1.19.0 release." + $ git commit -a -m"REL: Update main after 1.19.0 release." $ git push origin HEAD Go to github and make a PR. diff --git a/doc/TESTS.rst.txt b/doc/TESTS.rst.txt index d1af7017bc03..ba09aa80028a 100644 --- a/doc/TESTS.rst.txt +++ b/doc/TESTS.rst.txt @@ -59,7 +59,7 @@ that are run; but if it is greater than 1, then the tests will also provide warnings on missing tests. So if you want to run every test and get messages about which modules don't have tests:: - >>> numpy.test(label='full', verbose=2) # or numpy.test('full', 2) + >>> numpy.test(label='full', verbose=2) # or numpy.test('full', 2) Finally, if you are only interested in testing a subset of NumPy, for example, the ``core`` module, use the following:: @@ -101,27 +101,39 @@ module called ``test_yyy.py``. If you only need to test one aspect of ``zzz``, you can simply add a test function:: def test_zzz(): - assert_(zzz() == 'Hello from zzz') + assert zzz() == 'Hello from zzz' More often, we need to group a number of tests together, so we create a test class:: - from numpy.testing import assert_, assert_raises + import pytest # import xxx symbols from numpy.xxx.yyy import zzz + import pytest class TestZzz: def test_simple(self): - assert_(zzz() == 'Hello from zzz') + assert zzz() == 'Hello from zzz' def test_invalid_parameter(self): - assert_raises(...) + with pytest.raises(ValueError, match='.*some matching regex.*'): + ... -Within these test methods, ``assert_()`` and related functions are used to test +Within these test methods, ``assert`` and related functions are used to test whether a certain assumption is valid. If the assertion fails, the test fails. -Note that the Python builtin ``assert`` should not be used, because it is -stripped during compilation with ``-O``. +``pytest`` internally rewrites the ``assert`` statement to give informative +output when it fails, so should be preferred over the legacy variant +``numpy.testing.assert_``. Whereas plain ``assert`` statements are ignored +when running Python in optimized mode with ``-O``, this is not an issue when +running tests with pytest. + +Similarly, the pytest functions :func:`pytest.raises` and :func:`pytest.warns` +should be preferred over their legacy counterparts +:func:`numpy.testing.assert_raises` and :func:`numpy.testing.assert_warns`, +since the pytest variants are more broadly used and allow more explicit +targeting of warnings and errors when used with the ``match`` regex. + Note that ``test_`` functions or methods should not have a docstring, because that makes it hard to identify the test from the output of running the test @@ -145,7 +157,7 @@ can label it with ``pytest.mark.slow``:: @pytest.mark.slow def test_big(self): - print 'Big, slow test' + print('Big, slow test') Similarly for methods:: @@ -162,21 +174,21 @@ name; thus:: def setup(): """Module-level setup""" - print 'doing setup' + print('doing setup') def teardown(): """Module-level teardown""" - print 'doing teardown' + print('doing teardown') class TestMe: def setup(): """Class-level setup""" - print 'doing setup' + print('doing setup') def teardown(): """Class-level teardown""" - print 'doing teardown' + print('doing teardown') Setup and teardown functions to functions and methods are known as "fixtures", @@ -187,7 +199,7 @@ Parametric tests One very nice feature of testing is allowing easy testing across a range of parameters - a nasty problem for standard unit tests. Use the -``dec.paramaterize`` decorator. +``pytest.mark.parametrize`` decorator. Doctests -------- @@ -278,16 +290,16 @@ minor variations, it can be helpful to create a base class containing all the common tests, and then create a subclass for each variation. Several examples of this technique exist in NumPy; below are excerpts from one in `numpy/linalg/tests/test_linalg.py -`__:: +`__:: class LinalgTestCase: def test_single(self): - a = array([[1.,2.], [3.,4.]], dtype=single) + a = array([[1., 2.], [3., 4.]], dtype=single) b = array([2., 1.], dtype=single) self.do(a, b) def test_double(self): - a = array([[1.,2.], [3.,4.]], dtype=double) + a = array([[1., 2.], [3., 4.]], dtype=double) b = array([2., 1.], dtype=double) self.do(a, b) @@ -296,14 +308,14 @@ from one in `numpy/linalg/tests/test_linalg.py class TestSolve(LinalgTestCase): def do(self, a, b): x = linalg.solve(a, b) - assert_almost_equal(b, dot(a, x)) - assert_(imply(isinstance(b, matrix), isinstance(x, matrix))) + assert_allclose(b, dot(a, x)) + assert imply(isinstance(b, matrix), isinstance(x, matrix)) class TestInv(LinalgTestCase): def do(self, a, b): a_inv = linalg.inv(a) - assert_almost_equal(dot(a, a_inv), identity(asarray(a).shape[0])) - assert_(imply(isinstance(a, matrix), isinstance(a_inv, matrix))) + assert_allclose(dot(a, a_inv), identity(asarray(a).shape[0])) + assert imply(isinstance(a, matrix), isinstance(a_inv, matrix)) In this case, we wanted to test solving a linear algebra problem using matrices of several data types, using ``linalg.solve`` and diff --git a/doc/changelog/1.17.0-changelog.rst b/doc/changelog/1.17.0-changelog.rst index 4177c848fd7e..8179c180bad3 100644 --- a/doc/changelog/1.17.0-changelog.rst +++ b/doc/changelog/1.17.0-changelog.rst @@ -159,7 +159,7 @@ names contributed a patch for the first time. Pull requests merged ==================== -A total of 531 pull requests were merged for this release. +A total of 532 pull requests were merged for this release. * `#4808 `__: ENH: Make the `mode` parameter of np.pad default to 'constant' * `#8131 `__: BUG: Fix help() formatting for deprecated functions. @@ -174,6 +174,7 @@ A total of 531 pull requests were merged for this release. * `#10855 `__: ENH: Adding a count parameter to np.unpackbits * `#11230 `__: MAINT: More cleanup of einsum * `#11233 `__: BUG: ensure i0 does not change the shape. +* `#11358 `__: MAINT: Rewrite numpy.pad without concatenate * `#11684 `__: BUG: Raise when unravel_index, ravel_multi_index are given empty... * `#11689 `__: DOC: Add ref docs for C generic types. * `#11721 `__: BUG: Make `arr.ctypes.data` hold onto a reference to the underlying... diff --git a/doc/changelog/1.18.0-changelog.rst b/doc/changelog/1.18.0-changelog.rst index b86b3614ad3a..266ff08077ac 100644 --- a/doc/changelog/1.18.0-changelog.rst +++ b/doc/changelog/1.18.0-changelog.rst @@ -123,20 +123,20 @@ names contributed a patch for the first time. Pull requests merged ==================== -A total of 406 pull requests were merged for this release. +A total of 413 pull requests were merged for this release. * `#9301 `__: DOC: added note to docstring of numpy.savez * `#10151 `__: BUG: Numpy scalar types sometimes have the same name * `#12129 `__: DOC: Improve axes shift description and example in np.tensordot -* `#12205 `__: MAINT: avoid relying on `np.generic.__name__` in `np.dtype.name` +* `#12205 `__: MAINT: avoid relying on ``np.generic.__name__`` in ``np.dtype.name`` * `#12284 `__: ENH: supply our version of numpy.pxd, requires cython>=0.29 * `#12633 `__: BUG: General fixes to f2py reference counts (dereferencing) * `#12658 `__: BUG: NaT now sorts to ends of arrays * `#12828 `__: DOC: Updates to nditer usage instructions -* `#13003 `__: BUG: Do not crash on recursive `.dtype` attribute lookup. +* `#13003 `__: BUG: Do not crash on recursive ``.dtype`` attribute lookup. * `#13368 `__: ENH: Use AVX for float32 implementation of np.sin & np.cos * `#13605 `__: DEP: Deprecate silent ignoring of bad data in fromfile/fromstring -* `#13610 `__: ENH: Always produce a consistent shape in the result of `argwhere` +* `#13610 `__: ENH: Always produce a consistent shape in the result of ``argwhere`` * `#13673 `__: DOC: array(obj, dtype=dt) can downcast * `#13698 `__: DOC: Document ma.filled behavior with non-scalar fill_value * `#13710 `__: DOC: Add note to irfft-like functions about the default sizes @@ -144,6 +144,7 @@ A total of 406 pull requests were merged for this release. * `#13766 `__: MAINT: Update NEP template. * `#13794 `__: ENH: random: Add the multivariate hypergeometric distribution. * `#13799 `__: DOC: Fix unrendered links +* `#13802 `__: BUG: Fixed maximum relative error reporting in assert_allclose * `#13812 `__: MAINT: Rewrite Floyd algorithm * `#13825 `__: DOC: Add missing macros to C-API documentation * `#13829 `__: ENH: Add axis argument to random.permutation and random.shuffle @@ -162,6 +163,7 @@ A total of 406 pull requests were merged for this release. * `#13892 `__: DOC : Refactor Array API documentation -- Array Structure and... * `#13895 `__: DOC: Fix typo in "make_mask" documentation * `#13896 `__: MAINT: Delete unused _aliased_types.py +* `#13899 `__: MAINT: Change the type of error raised in set_printoptions * `#13901 `__: BLD: Remove Trusty dist in Travis CI build * `#13907 `__: BUG: Handle weird bytestrings in dtype() * `#13908 `__: ENH: use towncrier to build the release note @@ -174,7 +176,7 @@ A total of 406 pull requests were merged for this release. * `#13926 `__: DOC: Remove explicit .next method calls with built-in next function... * `#13928 `__: DOC: Don't override MaskedArray.view documentation with the one... * `#13930 `__: BUG: Fix incorrect GIL release in array.nonzero -* `#13935 `__: MAINT: Warn if `_add_newdocs.py` is used to add docstrings to... +* `#13935 `__: MAINT: Warn if ``_add_newdocs.py`` is used to add docstrings to... * `#13943 `__: MAINT: Revert #13876, "MAINT,BUG,DOC: Fix errors in _add_newdocs" * `#13944 `__: MAINT,BUG,DOC: Fix errors in _add_newdocs * `#13945 `__: DOC, MAINT: emphasize random API changes, remove Generator.randint @@ -184,7 +186,7 @@ A total of 406 pull requests were merged for this release. * `#13950 `__: Fixing failure on Python 2.7 on Windows 7 * `#13952 `__: Fix a typo related to the range of indices * `#13959 `__: DOC: add space between words across lines -* `#13964 `__: BUG, DOC: add new recfunctions to `__all__` +* `#13964 `__: BUG, DOC: add new recfunctions to ``__all__`` * `#13967 `__: DOC: Change (old) range() to np.arange() * `#13968 `__: DOC: improve np.sort docstring * `#13970 `__: DOC: spellcheck numpy/doc/broadcasting.py @@ -226,7 +228,7 @@ A total of 406 pull requests were merged for this release. * `#14076 `__: TST: Add 3.8-dev to travisCI testing. * `#14085 `__: DOC: Add blank line above doctest for intersect1d * `#14086 `__: ENH: Propose standard policy for dropping support of old Python... -* `#14089 `__: DOC: Use `pip install .` where possible instead of calling setup.py +* `#14089 `__: DOC: Use ``pip install .`` where possible instead of calling setup.py * `#14091 `__: MAINT: adjustments to test_ufunc_noncontigous * `#14092 `__: MAINT: Improve NEP template * `#14096 `__: DOC: fix documentation of i and j for tri. @@ -236,6 +238,7 @@ A total of 406 pull requests were merged for this release. * `#14106 `__: MAINT: remove duplicate variable assignments * `#14108 `__: BUG: initialize variable that is passed by pointer * `#14110 `__: DOC: fix typo in c-api/array.rst doc +* `#14115 `__: DOC: fix markup of news fragment readme * `#14121 `__: BUG: Add gcd/lcm definitions to npy_math.h * `#14122 `__: MAINT: Mark umath accuracy test xfail. * `#14124 `__: MAINT: Use equality instead of identity check with literal @@ -249,36 +252,39 @@ A total of 406 pull requests were merged for this release. * `#14153 `__: TST: Allow fuss in testing strided/non-strided exp/log loops * `#14170 `__: NEP: Proposal for __duckarray__ protocol * `#14171 `__: BUG: Make advanced indexing result on read-only subclass writeable +* `#14174 `__: BUG: Check for existence of ``fromstr`` which used in ``fromstr_next_element`` * `#14178 `__: TST: Clean up of test_pocketfft.py * `#14181 `__: DEP: Deprecate np.alen +* `#14183 `__: DOC: Fix misleading ``allclose`` docstring for ``equal_nan`` * `#14185 `__: MAINT: Workaround for Intel compiler bug leading to failing test -* `#14190 `__: DOC: Fix hermitian argument docs in `svd` +* `#14190 `__: DOC: Fix hermitian argument docs in ``svd`` * `#14195 `__: MAINT: Fix a docstring typo. -* `#14196 `__: DOC: Fix links in `/.github/CONTRIBUTING.md`. +* `#14196 `__: DOC: Fix links in ``/.github/CONTRIBUTING.md``. * `#14197 `__: ENH: Multivariate normal speedups * `#14203 `__: MAINT: Improve mismatch message of np.testing.assert_array_equal * `#14204 `__: DOC,MAINT: Move towncrier files and fixup categories * `#14207 `__: BUG: Fixed default BitGenerator name * `#14209 `__: BUG: Fix uint-overflow if padding with linear_ramp and negative... * `#14216 `__: ENH: Enable huge pages in all Linux builds -* `#14217 `__: BUG: Fix leak in the f2py-generated module init and `PyMem_Del`... +* `#14217 `__: BUG: Fix leak in the f2py-generated module init and ``PyMem_Del``... * `#14219 `__: DOC: new nan_to_num keywords are from 1.17 onwards * `#14223 `__: TST: Add tests for deprecated C functions (PyArray_As1D, PyArray_As1D) -* `#14224 `__: DOC: mention `take_along_axis` in `choose` +* `#14224 `__: DOC: mention ``take_along_axis`` in ``choose`` * `#14227 `__: ENH: Parse complex number from string * `#14231 `__: DOC: update or remove outdated sourceforge links * `#14234 `__: MAINT: Better error message for norm * `#14235 `__: DOC: add backlinks to numpy.org * `#14240 `__: BUG: Don't fail when lexsorting some empty arrays. -* `#14241 `__: BUG: Fix segfault in `random.permutation(x)` when x is a string. +* `#14241 `__: BUG: Fix segfault in ``random.permutation(x)`` when x is a string. * `#14245 `__: Doc: fix a typo in NEP21 * `#14249 `__: DOC: set status of NEP 28 (website redesign) to Accepted * `#14250 `__: BLD: MAINT: change default behavior of build flag appending. * `#14252 `__: BUG: Fixes StopIteration error from 'np.genfromtext' for empty... -* `#14255 `__: BUG: fix inconsistent axes ordering for axis in function `unique` +* `#14255 `__: BUG: fix inconsistent axes ordering for axis in function ``unique`` * `#14256 `__: DEP: Deprecate load/dump functions in favour of pickle methods * `#14257 `__: MAINT: Update NEP-30 * `#14259 `__: DEP: Deprecate arrayprint formatting functions +* `#14263 `__: DOC: Make Py3K docs C code snippets RST literal blocks * `#14266 `__: DOC: remove scipy.org from the breadcrumb formattiong * `#14270 `__: BUG: Fix formatting error in exception message * `#14272 `__: DOC: Address typos in dispatch docs @@ -293,9 +299,9 @@ A total of 406 pull requests were merged for this release. * `#14313 `__: DOC: Clarify rules about broadcasting when empty arrays are involved. * `#14321 `__: TST, MAINT: bump to OpenBLAS 0.3.7 stable * `#14325 `__: DEP: numpy.testing.rand -* `#14335 `__: DEP: Deprecate class `SafeEval` +* `#14335 `__: DEP: Deprecate class ``SafeEval`` * `#14341 `__: BUG: revert detecting and raising error on ragged arrays -* `#14342 `__: DOC: Improve documentation of `isscalar`. +* `#14342 `__: DOC: Improve documentation of ``isscalar``. * `#14349 `__: MAINT: Fix bloated mismatch error percentage in array comparisons. * `#14351 `__: DOC: Fix a minor typo in dispatch documentation. * `#14352 `__: MAINT: Remove redundant deprecation checks @@ -344,7 +350,7 @@ A total of 406 pull requests were merged for this release. * `#14475 `__: DOC: add timedelta64 signature * `#14477 `__: MAINT: Extract raising of MemoryError to a helper function * `#14483 `__: BUG,MAINT: Some fixes and minor cleanup based on clang analysis -* `#14484 `__: MAINT: Add `NPY_UNUSED` and `const` qualified suggested by clang +* `#14484 `__: MAINT: Add ``NPY_UNUSED`` and ``const`` qualified suggested by clang * `#14485 `__: MAINT: Silence integer comparison build warnings in assert statements * `#14486 `__: MAINT: distutils: Add newline at the end of printed warnings. * `#14490 `__: BUG: random: Revert gh-14458 and refix gh-14557. @@ -383,7 +389,7 @@ A total of 406 pull requests were merged for this release. * `#14567 `__: DEP: remove deprecated (and private) numpy.testing submodules. * `#14568 `__: BLD, DOC: fix gh-14518, add release note * `#14570 `__: BUG: importing build_src breaks setuptools monkeypatch for msvc14 -* `#14572 `__: DOC: Note runtests.py `-- -s` method to use pytests `-s` +* `#14572 `__: DOC: Note runtests.py ``-- -s`` method to use pytests ``-s`` * `#14573 `__: DOC: update submodule docstrings, remove info.py files * `#14576 `__: DOC: Document the NPY_SCALARKIND values as C variables. * `#14582 `__: MAINT: Bump pytest from 5.1.2 to 5.1.3 @@ -405,30 +411,31 @@ A total of 406 pull requests were merged for this release. * `#14614 `__: MAINT: Bump pytest from 5.1.3 to 5.2.0 * `#14615 `__: MAINT: Add "MAINT" tag to dependabot commit msg * `#14616 `__: DOC: Updated sphinx directive formatting -* `#14620 `__: DEP: Finish deprecation of non-integer `num` in linspace +* `#14620 `__: DEP: Finish deprecation of non-integer ``num`` in linspace * `#14621 `__: DOC: s/OR/AND/ in np.logical_and docstring * `#14623 `__: DOC: misleading np.sinc() documentation * `#14629 `__: DOC: clarify residual in np.polyfit * `#14630 `__: BUILD: change to build_src --verbose-cfg, runtests.py --debug-info * `#14631 `__: BUG: always free clean_sep -* `#14634 `__: DOC: Create `class Extension` docstring and add it to documentation. -* `#14636 `__: DOC: add `printoptions` as a context manager to `set_printoptions` +* `#14634 `__: DOC: Create ``class Extension`` docstring and add it to documentation. +* `#14636 `__: DOC: add ``printoptions`` as a context manager to ``set_printoptions`` * `#14639 `__: DOC: Fix typo in NEP 29 * `#14643 `__: MAINT: Use scalar math power function directly * `#14649 `__: DOC: Add IPython to dependencies needed to build docs. * `#14652 `__: MAINT: Bump pytest-cov from 2.7.1 to 2.8.1 * `#14653 `__: MAINT: Bump pytest from 5.2.0 to 5.2.1 * `#14654 `__: MAINT: Bump pytz from 2019.2 to 2019.3 -* `#14656 `__: MAINT: Use `extract_unit` throughout datetime +* `#14656 `__: MAINT: Use ``extract_unit`` throughout datetime * `#14657 `__: BUG: fix fromfile behavior when reading sub-array dtypes * `#14662 `__: BUG: random: Use correct length when axis is given to shuffle. * `#14669 `__: BUG: Do not rely on undefined behaviour to cast from float to... * `#14674 `__: NEP: add default-dtype-object-deprecation nep 34 * `#14681 `__: MAINT: Remove unused boolean negative/subtract loops -* `#14682 `__: DEP: ufunc `out` argument must be a tuple for multiple outputs -* `#14693 `__: BUG: Fix `np.einsum` errors on Power9 Linux and z/Linux +* `#14682 `__: DEP: ufunc ``out`` argument must be a tuple for multiple outputs +* `#14693 `__: BUG: Fix ``np.einsum`` errors on Power9 Linux and z/Linux * `#14696 `__: DOC: Note release notes process changes on devdocs start page * `#14699 `__: Doc warnings +* `#14703 `__: TST: Adding CI stages, with one initial job to the Travis CI * `#14705 `__: DOC: Switch Markdown link to RST in NEP 29 * `#14709 `__: TST: Divide Azure CI Pipelines into stages. * `#14710 `__: DEP: Finish the out kwarg deprecation for ufunc calls @@ -526,7 +533,7 @@ A total of 406 pull requests were merged for this release. * `#15058 `__: API, DOC: change names to multivariate_hypergeometric, improve docs * `#15059 `__: REL: Prepare for NumPy 1.18.0 release. * `#15109 `__: TST: Check requires_memory immediately before the test -* `#15111 `__: ENH: Add support to sort timedelta64 `NaT` to end of the array +* `#15111 `__: ENH: Add support to sort timedelta64 ``NaT`` to end of the array * `#15112 `__: MAINT: follow-up cleanup for blas64 PR * `#15113 `__: ENH: f2py: add --f2cmap option for specifying the name of .f2py_f2cmap * `#15114 `__: ENH: add support for ILP64 OpenBLAS (without symbol suffix) diff --git a/doc/changelog/1.18.5-changelog.rst b/doc/changelog/1.18.5-changelog.rst new file mode 100644 index 000000000000..f0bc51e6f2a7 --- /dev/null +++ b/doc/changelog/1.18.5-changelog.rst @@ -0,0 +1,18 @@ + +Contributors +============ + +A total of 3 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Matti Picus +* Siyuan + + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#16439 `__: ENH: enable pickle protocol 5 support for python3.5 +* `#16441 `__: BUG: relpath fails for different drives on windows diff --git a/doc/changelog/1.19.0-changelog.rst b/doc/changelog/1.19.0-changelog.rst index bd743832a33b..bde00249972a 100644 --- a/doc/changelog/1.19.0-changelog.rst +++ b/doc/changelog/1.19.0-changelog.rst @@ -136,25 +136,28 @@ names contributed a patch for the first time. Pull requests merged ==================== -A total of 452 pull requests were merged for this release. +A total of 488 pull requests were merged for this release. * `#8255 `__: ENH: add identity kwarg to frompyfunc +* `#10600 `__: DOC: Do not complain about contiguity when mutating ``ndarray.shape`` * `#12646 `__: TST: check exception details in refguide_check.py * `#13421 `__: ENH: improve runtime detection of CPU features * `#14326 `__: TST: Add assert_array_equal test for big integer arrays. * `#14376 `__: MAINT: Remove unnecessary 'from __future__ import ...' statements * `#14530 `__: MAINT: Fix typos and copy edit NEP-0030. * `#14546 `__: DOC: NumPy for absolute beginners tutorial -* `#14715 `__: NEP: Proposal for array creation dispatching with `__array_function__` +* `#14715 `__: NEP: Proposal for array creation dispatching with ``__array_function__`` * `#14867 `__: ENH: Use AVX-512F for np.maximum and np.minimum * `#14924 `__: BUG: Fix numpy.random.dirichlet returns NaN for small 'alpha'... -* `#14933 `__: API: Use `ResultType` in `PyArray_ConvertToCommonType` +* `#14933 `__: API: Use ``ResultType`` in ``PyArray_ConvertToCommonType`` +* `#14940 `__: BUG: pickle the content of a scalar containing objects, not the... * `#14942 `__: MAINT,API: ignore and NULL fasttake/fastputmask ArrFuncs slots * `#14981 `__: BUG: Make ``ediff1d`` kwarg casting consistent * `#14988 `__: DOC: linalg: Include information about scipy.linalg. * `#14995 `__: BUG: Use ``__array__`` during dimension discovery * `#15011 `__: MAINT: cleanup compat.py3k.py * `#15022 `__: ENH: f2py: improve error messages +* `#15024 `__: DOC: clarify documentation for transpose() * `#15028 `__: [DOC] LaTeX: fix preamble (closes #15026) * `#15035 `__: BUG: add endfunction, endsubroutine to valid fortran end words * `#15040 `__: TST: Add test for object method (and general unary) loops @@ -165,6 +168,7 @@ A total of 452 pull requests were merged for this release. * `#15052 `__: MAINT: follow-up cleanup for blas64 PR * `#15054 `__: DOC: add docstrings to refguide-check * `#15066 `__: Revert "DEP: issue deprecation warning when creating ragged array... +* `#15068 `__: ENH: Add support to sort timedelta64 ``NaT`` to end of the array * `#15069 `__: ENH: add support for ILP64 OpenBLAS (without symbol suffix) * `#15070 `__: DOC: correct version for NaT sort * `#15072 `__: TST: Check requires_memory immediately before the test @@ -201,6 +205,7 @@ A total of 452 pull requests were merged for this release. * `#15187 `__: MAINT: unskip test on win32 * `#15189 `__: ENH: Add property-based tests using Hypothesis * `#15194 `__: BUG: test, fix for c++ compilation +* `#15195 `__: MAINT: refactoring in np.core.records * `#15196 `__: DOC: Adding instructions for building documentation to developer... * `#15197 `__: DOC: NEP 37: A dispatch protocol for NumPy-like modules * `#15203 `__: MAINT: Do not use private Python function in testing @@ -215,6 +220,8 @@ A total of 452 pull requests were merged for this release. * `#15227 `__: DOC: typo in release.rst * `#15228 `__: NEP: universal SIMD NEP 38 * `#15229 `__: MAINT: Remove unused int_asbuffer +* `#15230 `__: BUG: do not emit warnings for np.sign, np.equal when using nan +* `#15231 `__: MAINT: Remove Python2 specific C module setup [part2] * `#15232 `__: MAINT: Cleaning up PY_MAJOR_VERSION/PY_VERSION_HEX * `#15233 `__: MAINT: Clean up more PY_VERSION_HEX * `#15236 `__: MAINT: Remove implicit inheritance from object class @@ -224,14 +231,14 @@ A total of 452 pull requests were merged for this release. * `#15241 `__: MAINT: Remove references to non-existent sys.exc_clear() * `#15242 `__: DOC: Update HOWTO_RELEASE.rst * `#15248 `__: MAINT: cleanup use of sys.exc_info -* `#15249 `__: MAINT: Eliminate some calls to `eval` +* `#15249 `__: MAINT: Eliminate some calls to ``eval`` * `#15251 `__: MAINT: Improve const-correctness of shapes and strides * `#15253 `__: DOC: clarify the effect of None parameters passed to ndarray.view * `#15254 `__: MAINT: Improve const-correctness of string arguments * `#15255 `__: MAINT: Delete numpy.distutils.compat * `#15256 `__: MAINT: Implement keyword-only arguments as syntax * `#15260 `__: MAINT: Remove FIXME comments introduced in the previous commit -* `#15261 `__: MAINT: Work with unicode strings in `dtype('i8,i8')` +* `#15261 `__: MAINT: Work with unicode strings in ``dtype('i8,i8')`` * `#15262 `__: BUG: Use PyDict_GetItemWithError() instead of PyDict_GetItem() * `#15263 `__: MAINT: Remove python2 array_{get,set}slice * `#15264 `__: DOC: Add some missing functions in the list of available ufuncs. @@ -248,8 +255,9 @@ A total of 452 pull requests were merged for this release. * `#15280 `__: BENCH: Add basic benchmarks for take and putmask * `#15281 `__: MAINT: Cleanup most PY3K #ifdef guards * `#15282 `__: DOC: BLD: add empty release notes for 1.19.0 to fix doc build... +* `#15283 `__: MAINT: Cleanup more NPY_PY3K * `#15284 `__: MAINT: Use a simpler return convention for internal functions -* `#15285 `__: MAINT: Simplify np.int_ inheritance +* `#15285 `__: MAINT: Simplify ``np.int_`` inheritance * `#15286 `__: DOC" Update np.full docstring. * `#15287 `__: MAINT: Express PyArray_DescrAlignConverter in terms of _convert_from_any * `#15288 `__: MAINT: Push down declarations in _convert_from_* @@ -261,7 +269,7 @@ A total of 452 pull requests were merged for this release. * `#15304 `__: MAINT: Remove NPY_PY3K constant * `#15305 `__: MAINT: Remove sys.version checks in tests * `#15307 `__: MAINT: cleanup sys.version dependant code -* `#15310 `__: MAINT: Ensure `_convert_from_*` functions set errors +* `#15310 `__: MAINT: Ensure ``_convert_from_*`` functions set errors * `#15312 `__: MAINT: Avoid escaping unicode in error messages * `#15315 `__: MAINT: Change file extension of ma README to rst. * `#15319 `__: BUG: fix NameError in clip nan propagation tests @@ -271,22 +279,25 @@ A total of 452 pull requests were merged for this release. * `#15329 `__: TST: move _no_tracing to testing._private, remove testing.support * `#15333 `__: BUG: Add some missing C error handling * `#15335 `__: MAINT: Remove sys.version checks -* `#15336 `__: DEP: Deprecate `->f->fastclip` at registration time +* `#15336 `__: DEP: Deprecate ``->f->fastclip`` at registration time * `#15338 `__: DOC: document site.cfg.example * `#15350 `__: MAINT: Fix mistype in histogramdd docstring * `#15351 `__: DOC, BLD: reword release note, upgrade sphinx version * `#15353 `__: MAINT: Remove unnecessary calls to PyArray_DATA from binomial... * `#15354 `__: MAINT: Bump pytest from 5.3.2 to 5.3.3 +* `#15355 `__: MAINT: Const qualify UFunc inner loops * `#15358 `__: MAINT: Remove six * `#15361 `__: MAINT: Revise imports from collections.abc module * `#15362 `__: MAINT: remove internal functions required to handle Python2/3... * `#15364 `__: MAINT: Remove other uses of six module * `#15366 `__: MAINT: resolve pyflake F403 'from module import *' used +* `#15367 `__: DOC: Fix Multithreaded Generation example docs * `#15368 `__: MAINT: Update tox for supported Python versions * `#15369 `__: MAINT: simd: Avoid signed comparison warning -* `#15370 `__: DOC: Updating Chararry Buffer datatypes #15360 +* `#15370 `__: DOC: Updating Chararry Buffer datatypes +* `#15373 `__: MAINT: Remove sys.version checks * `#15374 `__: TST: Simplify unicode test -* `#15375 `__: MAINT: Use `with open` when possible +* `#15375 `__: MAINT: Use ``with open`` when possible * `#15377 `__: MAINT: Cleanup python2 references * `#15379 `__: MAINT: Python2 Cleanups * `#15381 `__: DEP: add PendingDeprecation to matlib.py funky namespace @@ -303,11 +314,14 @@ A total of 452 pull requests were merged for this release. * `#15407 `__: MAINT: Replace basestring with str. * `#15408 `__: ENH: Use AVX-512F for complex number arithmetic, absolute, square... * `#15414 `__: MAINT: Remove Python2 workarounds +* `#15415 `__: MAINT: Revert f2py Python 2.6 workaround * `#15417 `__: MAINT: Cleanup references to python2 * `#15418 `__: MAINT, DOC: Remove use of old Python __builtin__, now known as... * `#15421 `__: ENH: Make use of ExitStack in npyio.py * `#15422 `__: MAINT: Inline gentype_getreadbuf * `#15423 `__: MAINT: Use f-strings for clarity. +* `#15425 `__: MAINT: dir(numpy) returns duplicate "testing" +* `#15426 `__: MAINT: Use the PyArrayScalar_VAL macro where possible * `#15427 `__: DEP: Schedule unused C-API functions for removal/disabling * `#15428 `__: DOC: Improve ndarray.ctypes example * `#15429 `__: DOC: distutils: Add a docstring to show_config(). @@ -315,14 +329,15 @@ A total of 452 pull requests were merged for this release. * `#15434 `__: MAINT: Updated polynomial to use fstrings * `#15435 `__: DOC: Fix Incorrect document in Beginner Docs * `#15436 `__: MAINT: Update core.py with fstrings (issue #15420) -* `#15439 `__: DOC: fix docstrings so `python tools/refguide-check --rst ... +* `#15439 `__: DOC: fix docstrings so ``python tools/refguide-check --rst ``... * `#15441 `__: MAINT: Tidy macros in scalar_new * `#15444 `__: MAINT: use 'yield from ' for simple cases * `#15445 `__: MAINT: Bump pytest from 5.3.3 to 5.3.4 * `#15446 `__: BUG: Reject nonsense arguments to scalar constructors * `#15449 `__: DOC: Update refguide_check note on how to skip code -* `#15451 `__: MAINT: Simplify `np.object_.__new__` +* `#15451 `__: MAINT: Simplify ``np.object_.__new__`` * `#15452 `__: STY,MAINT: avoid 'multiple imports on one line' +* `#15463 `__: ENH: expose ``bit_generator`` and random C-API to cython * `#15464 `__: MAINT: Cleanup duplicate line in refguide_check * `#15465 `__: MAINT: cleanup unused imports; avoid redefinition of imports * `#15468 `__: BUG: Fix for SVD not always sorted with hermitian=True @@ -330,21 +345,22 @@ A total of 452 pull requests were merged for this release. * `#15474 `__: MAINT: Eliminate messy _WORK macro * `#15476 `__: update result of rng.random(3) to current rng output * `#15480 `__: DOC: Correct get_state doc -* `#15482 `__: MAINT: Use `.identifier = val` to fill type structs +* `#15482 `__: MAINT: Use ``.identifier = val`` to fill type structs * `#15483 `__: [DOC] Mention behaviour of np.squeeze with one element * `#15484 `__: ENH: fixing generic error messages to be more specific in multiarray/descriptor.c * `#15487 `__: BUG: Fixing result of np quantile edge case * `#15491 `__: TST: mark the top 3 slowest tests to save ~10 seconds * `#15493 `__: MAINT: Bump pytest from 5.3.4 to 5.3.5 * `#15500 `__: MAINT: Use True/False instead of 1/0 in np.dtype.__reduce__ -* `#15503 `__: MAINT: Do not allow `copyswap` and friends to fail silently +* `#15503 `__: MAINT: Do not allow ``copyswap`` and friends to fail silently * `#15504 `__: DOC: Remove duplicated code in true_divide docstring * `#15505 `__: NEP 40: Informational NEP about current DTypes +* `#15506 `__: NEP 41: First steps towards improved Datatype Support * `#15510 `__: DOC: Update unique docstring example * `#15511 `__: MAINT: Large overhead in some random functions * `#15516 `__: TST: Fix missing output in refguide-check * `#15521 `__: MAINT: Simplify arraydescr_richcompare -* `#15522 `__: MAINT: Fix internal misuses of `NPY_TITLE_KEY` +* `#15522 `__: MAINT: Fix internal misuses of ``NPY_TITLE_KEY`` * `#15524 `__: DOC: Update instructions for building/archiving docs. * `#15526 `__: BUG: Fix inline assembly that detects cpu features on x86(32bit) * `#15532 `__: update doctests, small bugs and changes of repr @@ -354,13 +370,15 @@ A total of 452 pull requests were merged for this release. * `#15543 `__: NEP: edit and move NEP 38 to accepted status * `#15547 `__: MAINT: Refresh Doxyfile and modernize numpyfilter.py * `#15549 `__: TST: Accuracy test float32 sin/cos/exp/log for AVX platforms -* `#15550 `__: DOC: Improve the `numpy.linalg.eig` docstring. +* `#15550 `__: DOC: Improve the ``numpy.linalg.eig`` docstring. +* `#15553 `__: BUG: Added missing error check in ``ndarray.__contains__`` * `#15554 `__: NEP 44 - Restructuring the NumPy Documentation * `#15556 `__: TST: (Travis CI) Use full python3-dbg path for virtual env creation * `#15560 `__: BUG, DOC: restore missing import * `#15566 `__: DOC: Removing bad practices from quick start + some PEP8 * `#15574 `__: TST: Do not create symbolic link named gfortran. * `#15575 `__: DOC: Document caveat in random.uniform +* `#15577 `__: TST: Test division by zero both with scalar and with array * `#15579 `__: DOC: numpy.clip is equivalent to minimum(..., maximum(...)) * `#15582 `__: MAINT: Bump cython from 0.29.14 to 0.29.15 * `#15583 `__: MAINT: Bump hypothesis from 5.3.0 to 5.5.4 @@ -369,6 +387,7 @@ A total of 452 pull requests were merged for this release. * `#15600 `__: TST: use manylinux2010 docker instead of ubuntu * `#15610 `__: TST: mask DeprecationWarning in xfailed test * `#15612 `__: BUG: Fix bug in AVX-512F np.maximum and np.minimum +* `#15614 `__: DOC: Reword docstring for assert_equal * `#15615 `__: BUG: Remove check requiring natural alignment of float/double... * `#15616 `__: DOC: Add missing imports, definitions and dummy file * `#15619 `__: DOC: Fix documentation for apply_along_axis @@ -376,9 +395,10 @@ A total of 452 pull requests were merged for this release. * `#15631 `__: MAINT: Pull identical line out of conditional. * `#15633 `__: DOC: remove broken link in f2py tutorial * `#15639 `__: BLD: update openblas download to new location, use manylinux2010-base +* `#15644 `__: DOC: Update to clarify actual behavior real_if_(all elements)_close * `#15648 `__: MAINT: AVX512 implementation with intrinsic for float64 input... * `#15653 `__: BLD: update OpenBLAS to pre-0.3.9 version -* `#15662 `__: DOC: Refactor `np.polynomial` docs using `automodule` +* `#15662 `__: DOC: Refactor ``np.polynomial`` docs using ``automodule`` * `#15665 `__: BUG: fix doctest exception messages * `#15672 `__: MAINT: Added comment pointing FIXME to relevant PR. * `#15673 `__: DOC: Make extension module wording more clear @@ -386,16 +406,18 @@ A total of 452 pull requests were merged for this release. * `#15680 `__: DOC: Improve Benchmark README with environment setup and more... * `#15682 `__: MAINT: Bump hypothesis from 5.5.4 to 5.6.0 * `#15683 `__: NEP: move NEP 44 to accepted status +* `#15685 `__: ENH: Add ``subok`` parameter to np.copy function (cf. #6509) * `#15694 `__: DOC: Fix indexing docs to pass refguide * `#15695 `__: MAINT: Test during import to detect bugs with Accelerate(MacOS)... * `#15696 `__: MAINT: Add a fast path to var for complex input * `#15701 `__: MAINT: Convert shebang from python to python3 (#15687) * `#15702 `__: MAINT: replace optparse with argparse for 'doc' and 'tools' scripts * `#15703 `__: DOC: Fix quickstart doc to pass refguide +* `#15705 `__: DOC: Change list to tuple in example description. * `#15706 `__: MAINT: Fixing typos in f2py comments and code. * `#15710 `__: DOC: fix SVD tutorial to pass refguide * `#15714 `__: MAINT: use list-based APIs to call subprocesses -* `#15715 `__: ENH: update numpy.linalg.multi_dot to accept an `out` argument +* `#15715 `__: ENH: update numpy.linalg.multi_dot to accept an ``out`` argument * `#15716 `__: TST: always use 'python -mpip' not 'pip' * `#15717 `__: DOC: update datetime reference to pass refguide * `#15718 `__: DOC: Fix coremath.rst to fix refguide_check @@ -403,6 +425,7 @@ A total of 452 pull requests were merged for this release. * `#15723 `__: BUG: fix logic error when nm fails on 32-bit * `#15724 `__: TST: Remove nose from the test_requirements.txt file. * `#15733 `__: DOC: Allow NEPs to link to python, numpy, scipy, and matplotlib... +* `#15735 `__: DOC: LICENSE 2019 -> 2020 * `#15736 `__: BUG: Guarantee array is in valid state after memory error occurs... * `#15738 `__: MAINT: Remove non-native byte order from _var check. * `#15740 `__: MAINT: Add better error handling in linalg.norm for vectors and... @@ -418,28 +441,30 @@ A total of 452 pull requests were merged for this release. * `#15769 `__: ENH: Allow toggling madvise hugepage and fix default * `#15771 `__: DOC: Fix runtests example in developer docs * `#15773 `__: DEP: Make issubdtype consistent for types and dtypes -* `#15774 `__: MAINT: remove useless `global` statements +* `#15774 `__: MAINT: remove useless ``global`` statements * `#15778 `__: BLD: Add requirements.txt file for building docs * `#15781 `__: BUG: don't add 'public' or 'private' if the other one exists -* `#15784 `__: ENH: Use TypeError in `np.array` for python consistency +* `#15784 `__: ENH: Use TypeError in ``np.array`` for python consistency * `#15794 `__: BUG: Add basic __format__ for masked element to fix incorrect... * `#15797 `__: TST: Add unit test for out=None of np.einsum * `#15799 `__: MAINT: Cleanups to np.insert and np.delete * `#15800 `__: BUG: Add error-checking versions of strided casts. -* `#15802 `__: DEP: Make `np.insert` and `np.delete` on 0d arrays with an axis... -* `#15803 `__: DOC: correct possible list lengths for `extobj` in ufunc calls +* `#15802 `__: DEP: Make ``np.insert`` and ``np.delete`` on 0d arrays with an axis... +* `#15803 `__: DOC: correct possible list lengths for ``extobj`` in ufunc calls * `#15804 `__: DEP: Make np.delete on out-of-bounds indices an error -* `#15805 `__: DEP: Forbid passing non-integral index arrays to `insert` and... +* `#15805 `__: DEP: Forbid passing non-integral index arrays to ``insert`` and... * `#15806 `__: TST: Parametrize sort test * `#15809 `__: TST: switch PyPy job with CPython * `#15812 `__: TST: Remove code that is not supposed to warn out of warning... * `#15815 `__: DEP: Do not cast boolean indices to integers in np.delete * `#15816 `__: MAINT: simplify code that assumes str/unicode and int/long are... +* `#15827 `__: BUG: Break on all errors when performing strided casts. * `#15830 `__: MAINT: pathlib and hashlib are in stdlib in Python 3.5+ -* `#15832 `__: ENH: improved error message `IndexError: too many indices for... +* `#15832 `__: ENH: improved error message ``IndexError: too many indices for``... +* `#15834 `__: NEP: Add paragraph to NEP 41 about no array-object use and fix... * `#15836 `__: BUG: Fix IndexError for illegal axis in np.mean * `#15839 `__: DOC: Minor fix to _hist_bin_fd documentation -* `#15840 `__: BUG,DEP: Make `scalar.__round__()` behave like pythons round +* `#15840 `__: BUG,DEP: Make ``scalar.__round__()`` behave like pythons round * `#15843 `__: DOC: First steps towards docs restructuring (NEP 44) * `#15848 `__: DOC, TST: enable refguide_check in circleci * `#15850 `__: DOC: fix typo in C-API reference @@ -447,8 +472,9 @@ A total of 452 pull requests were merged for this release. * `#15866 `__: MAINT: Bump cython from 0.29.15 to 0.29.16 * `#15867 `__: DEP: Deprecate ndarray.tostring() * `#15868 `__: TST: use draft OpenBLAS build +* `#15870 `__: ENH: Add keepdims argument to count_nonzero * `#15872 `__: BUG: Fix eigh and cholesky methods of numpy.random.multivariate_normal -* `#15876 `__: BUG: Check that `pvals` is 1D in `_generator.multinomial`. +* `#15876 `__: BUG: Check that ``pvals`` is 1D in ``_generator.multinomial``. * `#15877 `__: DOC: Add missing signature from nditer docstring * `#15881 `__: BUG: Fix empty_like to respect shape=() * `#15882 `__: BUG: Do not ignore empty tuple of strides in ndarray.__new__ @@ -456,29 +482,34 @@ A total of 452 pull requests were merged for this release. * `#15884 `__: BUG: Setting a 0d array's strides to themselves should be legal * `#15885 `__: BUG: Respect itershape=() in nditer * `#15887 `__: MAINT: Clean-up 'next = __next__' used for Python 2 compatibility +* `#15891 `__: DOC: Clarify docs on mixed advanced indexing and slicing * `#15893 `__: TST: Run test_large_zip in a child process * `#15894 `__: DOC: Add missing doc of numpy.ma.apply_over_axes in API list. * `#15899 `__: DOC: Improve record module documentation * `#15901 `__: DOC: Fixed order of items and link to mailing list in dev docs... * `#15903 `__: BLD: report clang version on macOS -* `#15904 `__: MAINT: records: Remove private `format_parser._descr` attribute +* `#15904 `__: MAINT: records: Remove private ``format_parser._descr`` attribute +* `#15907 `__: DOC: Update documentation w.r.t. NPY_RELAXED_STRIDES_CHECKING * `#15914 `__: BUG: random: Disallow p=0 in negative_binomial +* `#15920 `__: DOC: Improve docstring for numpy.linalg.lstsq * `#15921 `__: ENH: Use sysconfig instead of probing Makefile * `#15928 `__: DOC: Update np.copy docstring to include ragged case * `#15931 `__: DOC: Correct private function name to PyArray_AdaptFlexibleDType -* `#15936 `__: MAINT: Fix capitalization in error message in `mtrand.pyx` +* `#15936 `__: MAINT: Fix capitalization in error message in ``mtrand.pyx`` +* `#15938 `__: BUG: Add _LARGE_FILES to def_macros[] when platform is AIX. * `#15939 `__: DOC: Update np.rollaxis docstring * `#15949 `__: BUG: fix AttributeError on accessing object in nested MaskedArray. -* `#15951 `__: BUG: Alpha parameter must be 1D in `generator.dirichlet` +* `#15951 `__: BUG: Alpha parameter must be 1D in ``generator.dirichlet`` * `#15953 `__: NEP: minor maintenance, update filename and fix a cross-reference * `#15964 `__: MAINT: Bump hypothesis from 5.8.0 to 5.8.3 * `#15967 `__: TST: Add slow_pypy support * `#15968 `__: DOC: Added note to angle function docstring about angle(0) being... * `#15982 `__: MAINT/BUG: Cleanup and minor fixes to conform_reduce_result -* `#15985 `__: BUG: Avoid duplication in stack trace of `linspace(a, b, num=1.5)` -* `#15988 `__: BUG: Fix inf and NaN-warnings in half float `nextafter` +* `#15985 `__: BUG: Avoid duplication in stack trace of ``linspace(a, b, num=1.5)`` +* `#15988 `__: BUG: Fix inf and NaN-warnings in half float ``nextafter`` * `#15989 `__: MAINT: Remove 0d check for PyArray_ISONESEGMENT * `#15990 `__: DEV: Pass additional runtests.py args to ASV +* `#15991 `__: BUG: max/min of a masked array dtype fix * `#15993 `__: DOC: Fix method documentation of function sort in MaskedArray * `#16000 `__: NEP: Improve Value Based Casting paragraph in NEP 40 * `#16001 `__: DOC: add note on flatten ordering in matlab page @@ -496,6 +527,8 @@ A total of 452 pull requests were merged for this release. * `#16038 `__: MAINT,TST: Move _repr_latex tests to test_printing. * `#16041 `__: BUG: missing 'f' prefix for fstring * `#16042 `__: ENH: Fix exception causes in build_ext.py +* `#16043 `__: DOC: Add converters example to the loadtxt docstring +* `#16051 `__: DOC: Add missing bracket * `#16053 `__: DOC: Small typo fixes to NEP 40. * `#16054 `__: DOC, BLD: update release howto and walkthrough for ananconda.org... * `#16061 `__: ENH: Chained exceptions in linalg.py and polyutils.py @@ -508,6 +541,7 @@ A total of 452 pull requests were merged for this release. * `#16077 `__: BLD: fix path to libgfortran on macOS * `#16078 `__: DOC: Add axis to random module "new or different" docs * `#16079 `__: DOC,BLD: Limit timeit iterations in random docs. +* `#16080 `__: BUG: numpy.einsum indexing arrays now accept numpy int type * `#16081 `__: DOC: add note on type casting to numpy.left_shift(). * `#16083 `__: DOC: improve development debugging doc * `#16084 `__: DOC: tweak neps/scope.rst @@ -517,10 +551,12 @@ A total of 452 pull requests were merged for this release. * `#16097 `__: MAINT, DOC: Improve grammar on a comment in the quickstart * `#16100 `__: NEP 41: Accept NEP 41 and add DType<->scalar duplication paragraph * `#16101 `__: BLD: put openblas library in local directory on windows +* `#16102 `__: ENH: correct identity for logaddexp2 ufunc: -inf * `#16113 `__: MAINT: Fix random.PCG64 signature * `#16119 `__: DOC: Move misplaced news fragment for gh-13421 * `#16122 `__: DOC: Fix links for NEP 40 in NEP 41 * `#16125 `__: BUG: lib: Fix a problem with vectorize with default parameters. +* `#16128 `__: ENH: Add equal_nan keyword argument to array_equal * `#16129 `__: ENH: Better error message when ``bins`` has float value in ``histogramdd``. * `#16133 `__: MAINT: Unify casting error creation (outside the iterator) * `#16141 `__: BENCH: Default to building HEAD instead of master @@ -571,7 +607,7 @@ A total of 452 pull requests were merged for this release. * `#16344 `__: BUG: Allow attaching documentation twice in add_docstring * `#16355 `__: MAINT: Remove f-strings in setup.py. (gh-16346) * `#16356 `__: BUG: Indentation for docstrings -* `#16358 `__: BUG: Fix dtype leak in `PyArray_FromAny` error path +* `#16358 `__: BUG: Fix dtype leak in ``PyArray_FromAny`` error path * `#16383 `__: ENH: Optimize Cpu feature detect in X86, fix for GCC on macOS... * `#16398 `__: MAINT: core: Use a raw string for the fromstring docstring. * `#16399 `__: MAINT: Make ctypes optional on Windows diff --git a/doc/changelog/1.19.5-changelog.rst b/doc/changelog/1.19.5-changelog.rst new file mode 100644 index 000000000000..f7cbd5377190 --- /dev/null +++ b/doc/changelog/1.19.5-changelog.rst @@ -0,0 +1,32 @@ + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Christoph Gohlke +* Matti Picus +* Raghuveer Devulapalli +* Sebastian Berg +* Simon Graham + +* Veniamin Petrenko + +* Bernie Gray + + +Pull requests merged +==================== + +A total of 11 pull requests were merged for this release. + +* `#17756 `__: BUG: Fix segfault due to out of bound pointer in floatstatus... +* `#17774 `__: BUG: fix np.timedelta64('nat').__format__ throwing an exception +* `#17775 `__: BUG: Fixed file handle leak in array_tofile. +* `#17786 `__: BUG: Raise recursion error during dimension discovery +* `#17917 `__: BUG: Fix subarray dtype used with too large count in fromfile +* `#17918 `__: BUG: 'bool' object has no attribute 'ndim' +* `#17919 `__: BUG: ensure _UFuncNoLoopError can be pickled +* `#17924 `__: BLD: use BUFFERSIZE=20 in OpenBLAS +* `#18026 `__: BLD: update to OpenBLAS 0.3.13 +* `#18036 `__: BUG: make a variable volatile to work around clang compiler bug +* `#18114 `__: REL: Prepare for the NumPy 1.19.5 release. diff --git a/doc/changelog/1.20.0-changelog.rst b/doc/changelog/1.20.0-changelog.rst new file mode 100644 index 000000000000..f06bd8a8d22d --- /dev/null +++ b/doc/changelog/1.20.0-changelog.rst @@ -0,0 +1,913 @@ + +Contributors +============ + +A total of 184 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Aaron Meurer + +* Abhilash Barigidad + +* Abhinav Reddy + +* Abhishek Singh + +* Al-Baraa El-Hag + +* Albert Villanova del Moral + +* Alex Leontiev + +* Alex Rockhill + +* Alex Rogozhnikov +* Alexander Belopolsky +* Alexander Kuhn-Regnier + +* Allen Downey + +* Andras Deak +* Andrea Olivo + +* Andrew Eckart + +* Anirudh Subramanian +* Anthony Byuraev + +* Antonio Larrosa + +* Ashutosh Singh + +* Bangcheng Yang + +* Bas van Beek + +* Ben Derrett + +* Ben Elliston + +* Ben Nathanson + +* Bernie Gray + +* Bharat Medasani + +* Bharat Raghunathan +* Bijesh Mohan + +* Bradley Dice + +* Brandon David + +* Brandt Bucher +* Brian Soto + +* Brigitta Sipocz +* Cameron Blocker + +* Carl Leake + +* Charles Harris +* Chris Brown + +* Chris Vavaliaris + +* Christoph Gohlke +* Chunlin Fang +* CloseChoice + +* Daniel G. A. Smith + +* Daniel Hrisca +* Daniel Vanzo + +* David Pitchford + +* Davide Dal Bosco + +* Derek Homeier +* Dima Kogan + +* Dmitry Kutlenkov + +* Douglas Fenstermacher + +* Dustin Spicuzza + +* E. Madison Bray + +* Elia Franzella + +* Enrique Matías Sánchez + +* Erfan Nariman | Veneficus + +* Eric Larson +* Eric Moore +* Eric Wieser +* Erik M. Bray +* EthanCJ-git + +* Etienne Guesnet + +* FX Coudert + +* Felix Divo +* Frankie Robertson + +* Ganesh Kathiresan +* Gengxin Xie +* Gerry Manoim + +* Guilherme Leobas +* Hassan Kibirige +* Hugo Mendes + +* Hugo van Kemenade +* Ian Thomas + +* InessaPawson + +* Isabela Presedo-Floyd + +* Isuru Fernando +* Jakob Jakobson + +* Jakub Wilk +* James Myatt + +* Jesse Li + +* John Hagen + +* John Zwinck +* Joseph Fox-Rabinovitz +* Josh Wilson +* Jovial Joe Jayarson + +* Julia Signell + +* Jun Kudo + +* Karan Dhir + +* Kaspar Thommen + +* Kerem Hallaç +* Kevin Moore + +* Kevin Sheppard +* Klaus Zimmermann + +* LSchroefl + +* Laurie + +* Laurie Stephey + +* Levi Stovall + +* Lisa Schwetlick + +* Lukas Geiger + +* Madhulika Jain Chambers + +* Matthias Bussonnier +* Matti Picus +* Melissa Weber Mendonça +* Michael Hirsch +* Nick R. Papior +* Nikola Forró +* Noman Arshad + +* Paul YS Lee + +* Pauli Virtanen +* Paweł Redzyński + +* Peter Andreas Entschev +* Peter Bell +* Philippe Ombredanne + +* Phoenix Meadowlark + +* Piotr Gaiński +* Raghav Khanna + +* Raghuveer Devulapalli +* Rajas Rade + +* Rakesh Vasudevan +* Ralf Gommers +* Raphael Kruse + +* Rashmi K A + +* Robert Kern +* Rohit Sanjay + +* Roman Yurchak +* Ross Barnowski +* Royston E Tauro + +* Ryan C Cooper + +* Ryan Soklaski +* Safouane Chergui + +* Sahil Siddiq + +* Sarthak Vineet Kumar + +* Sayed Adel +* Sebastian Berg +* Sergei Vorfolomeev + +* Seth Troisi +* Sidhant Bansal + +* Simon Gasse +* Simon Graham + +* Stefan Appelhoff + +* Stefan Behnel + +* Stefan van der Walt +* Steve Dower +* Steve Joachim + +* Steven Pitman + +* Stuart Archibald +* Sturla Molden +* Susan Chang + +* Takanori H + +* Tapajyoti Bose + +* Thomas A Caswell +* Tina Oberoi +* Tirth Patel +* Tobias Pitters + +* Tomoki, Karatsu + +* Tyler Reddy +* Veniamin Petrenko + +* Wansoo Kim + +* Warren Weckesser +* Wei Yang + +* Wojciech Rzadkowski +* Yang Hau + +* Yogesh Raisinghani + +* Yu Feng +* Yuya Unno + +* Zac Hatfield-Dodds +* Zuhair Ali-Khan + +* @abhilash42 + +* @danbeibei + +* @dojafrat +* @dpitch40 + +* @forfun + +* @iamsoto + +* @jbrockmendel + +* @leeyspaul + +* @mitch + +* @prateek arora + +* @serge-sans-paille + +* @skywalker + +* @stphnlyd + +* @xoviat +* @谭九鼎 + +* @JMFT + +* @Jack + +* @Neal C + + +Pull requests merged +==================== + +A total of 716 pull requests were merged for this release. + +* `#13516 `__: ENH: enable multi-platform SIMD compiler optimizations +* `#14779 `__: NEP 36 (fair play) +* `#14882 `__: DEP: Deprecate aliases of builtin types in python 3.7+ +* `#15037 `__: BUG: ``np.resize`` negative shape and subclasses edge case fixes +* `#15121 `__: ENH: random: Add the method ``permuted`` to Generator. +* `#15162 `__: BUG,MAINT: Fix issues with non-reduce broadcasting axes +* `#15471 `__: BUG: Ensure PyArray_FromScalar always returns the requested dtype +* `#15507 `__: NEP 42: Technical decisions for new DTypes +* `#15508 `__: API: Create Preliminary DTypeMeta class and np.dtype subclasses +* `#15551 `__: DOC: Simd optimization documentation +* `#15604 `__: MAINT: Avoid exception in NpzFile destructor if constructor raises... +* `#15666 `__: ENH: Improved ``__str__`` for polynomials +* `#15759 `__: BUILD: Remove Accelerate support +* `#15791 `__: [DOC] Added tutorial about the numpy.ma module. +* `#15852 `__: ENH: Add where argument to np.mean +* `#15886 `__: DEP: Deprecate passing shape=None to mean shape=() +* `#15900 `__: DEP: Ensure indexing errors will be raised even on empty results +* `#15997 `__: ENH: improve printing of arrays with multi-line reprs +* `#16056 `__: DEP: Deprecate inexact matches for mode, searchside +* `#16130 `__: DOC: Correct documentation of ``__array__`` when used as output... +* `#16134 `__: ENH: Implement concatenate dtype and casting keyword arguments +* `#16156 `__: DEP: Deprecate ``numpy.dual``. +* `#16161 `__: BUG: Potential fix for divmod(1.0, 0.0) to raise divbyzero and... +* `#16167 `__: DOC: Increase guidance and detail of np.polynomial docstring +* `#16174 `__: DOC: Add transition note to all lib/poly functions +* `#16200 `__: ENH: Rewrite of array-coercion to support new dtypes +* `#16205 `__: ENH: Add ``full_output`` argument to ``f2py.compile``. +* `#16207 `__: DOC: Add PyArray_ContiguousFromObject C docs +* `#16232 `__: DEP: Deprecate ufunc.outer with matrix inputs +* `#16237 `__: MAINT: precompute ``log(2.0 * M_PI)`` in ``random_loggam`` +* `#16238 `__: MAINT: Unify cached (C-level static) imports +* `#16239 `__: BUG,DOC: Allow attach docs twice but error if wrong +* `#16242 `__: BUG: Fix default fallback in genfromtxt +* `#16247 `__: ENH:Umath Replace raw SIMD of unary float point(32-64) with NPYV... +* `#16248 `__: MRG, ENH: added edge keyword argument to digitize +* `#16253 `__: DOC: Clarify tiny/xmin in finfo and machar +* `#16254 `__: MAINT: Chain exceptions in generate_umath.py +* `#16257 `__: DOC: Update the f2py section of the "Using Python as Glue" page. +* `#16260 `__: DOC: Improve ``rec.array`` function documentation +* `#16266 `__: ENH: include dt64/td64 isinstance checks in ``__init__.pxd`` +* `#16267 `__: DOC: Clarifications for np.std +* `#16273 `__: BUG: Order percentile monotonically +* `#16274 `__: MAINT: cleanups to quantile +* `#16275 `__: REL: Update master after 1.19.x branch. +* `#16276 `__: BUG: Ensure out argument is returned by identity for 0d arrays +* `#16278 `__: DOC: Clarifications for ``np.var``. +* `#16283 `__: DOC: Add a note about performance of isclose compared to math.isclose +* `#16284 `__: MAINT: Clean up the implementation of quantile +* `#16285 `__: MAINT: Bump hypothesis from 5.12.0 to 5.14.0 +* `#16288 `__: BLD: Avoid "visibility attribute not supported" warning +* `#16291 `__: DOC: Improve "tobytes" docstring. +* `#16292 `__: BUG: Fix tools/download-wheels.py. +* `#16295 `__: BUG: Require Python >= 3.6 in setup.py +* `#16296 `__: DOC: Fix malformed docstrings in ma. +* `#16297 `__: ENH: Optimize Cpu feature detect in X86, fix for GCC on macOS +* `#16298 `__: BUG: np.info does not show keyword-only arguments +* `#16300 `__: DOC: Fix bad reference in ``numpy.ma`` +* `#16304 `__: TST, MAINT: Fix detecting and testing armhf features +* `#16305 `__: DOC: Fix packbits documentation rendering, +* `#16306 `__: DOC: Fix troubleshooting code snippet when env vars are empty +* `#16308 `__: BUG: relpath fails for different drives on windows +* `#16311 `__: DOC: Fix ``np.ma.core.doc_note`` +* `#16316 `__: MAINT: Bump numpydoc version +* `#16318 `__: MAINT: Stop Using PyEval_Call* and simplify some uses +* `#16321 `__: ENH: Improve the ARM cpu feature detection by parsing /proc/cpuinfo +* `#16323 `__: DOC: Reconstruct Testing Guideline. +* `#16327 `__: BUG: Don't segfault on bad __len__ when assigning. +* `#16329 `__: MAINT: Cleanup 'tools/download-wheels.py' +* `#16332 `__: DOC: link np.interp to SciPy's interpolation functions (closes... +* `#16333 `__: DOC: Fix spelling typo - homogenous to homogeneous. (#16324) +* `#16334 `__: ENH: Use AVX-512 for np.isnan, np.infinite, np.isinf and np.signbit +* `#16336 `__: BUG: Fix refcounting in add_newdoc +* `#16337 `__: CI: Create a link for the circleCI artifact +* `#16346 `__: MAINT: Remove f-strings in setup.py. +* `#16348 `__: BUG: Fix dtype leak in ``PyArray_FromAny`` error path +* `#16349 `__: BUG: Indentation for docstrings +* `#16350 `__: BUG: Set readonly flag in array interface +* `#16351 `__: BUG: Fix small leaks in error path and ``empty_like`` with shape +* `#16362 `__: MAINT: Streamline download-wheels. +* `#16365 `__: DOC: Fix an obvious mistake in a message printed in doc/Makefile. +* `#16367 `__: MAINT: Bump cython from 0.29.17 to 0.29.19 +* `#16368 `__: MAINT: Bump hypothesis from 5.14.0 to 5.15.1 +* `#16369 `__: MAINT: Bump pytest-cov from 2.8.1 to 2.9.0 +* `#16371 `__: ENH: Use AVX-512 for np.frexp and np.ldexp +* `#16373 `__: MAINT, DOC: add index for user docs. +* `#16375 `__: ENH: ARM Neon implementation with intrinsic for np.argmax. +* `#16385 `__: DOC: Tighten howto-docs guide #16259 +* `#16387 `__: MAINT: Make ctypes optional on Windows +* `#16389 `__: ENH: Hardcode buffer handling for simple scalars +* `#16392 `__: MAINT: Stop uploading wheels to Rackspace. +* `#16393 `__: MAINT: Use a raw string for the fromstring docstring. +* `#16395 `__: ENH: Validate and disable CPU features in runtime +* `#16397 `__: ENH: Implement the NumPy C SIMD vectorization interface +* `#16404 `__: DOC,BLD: Update make dist html target. +* `#16408 `__: DOC,BLD: Update sphinx conf to use xelatex. +* `#16409 `__: TST, CI: turn on codecov patch diffs +* `#16411 `__: BUG: endpoints of array returned by geomspace() should match... +* `#16417 `__: MAINT: support python 3.10 +* `#16418 `__: MAINT: Chain some exceptions. +* `#16420 `__: DOC: Improve intersect1d docstring +* `#16422 `__: DOC: Update assert_warns parameter list +* `#16423 `__: TST: Simplify assert_warns in test_io.py +* `#16427 `__: DOC: make NEP 18 status Final +* `#16428 `__: DOC: Add style guide to howto_document +* `#16430 `__: DOC: NEP for C style guide +* `#16433 `__: DOC: Fix description of dtype default in linspace +* `#16435 `__: BUG: Add extern to PyArrayDTypeMeta_Type declaration +* `#16436 `__: DOC: Add a reference into NEP 29, +* `#16438 `__: MAINT: Catch remaining cases of Py_SIZE and Py_TYPE as lvalues +* `#16442 `__: ENH: Fix deprecated warn for Intel/Apple/Clang Compiler +* `#16444 `__: DOC: make clearer that sinc is normalized by a factor pi +* `#16445 `__: DOC: update roadmap +* `#16446 `__: BUG: fixes einsum output order with optimization (#14615) +* `#16447 `__: DOC: add a "make show" command to doc/Makefile +* `#16450 `__: DOC: Add a NEP link to all neps. +* `#16452 `__: DOC,ENH: extend error message when Accelerate is detected +* `#16454 `__: TST: Add tests for PyArray_IntpConverter +* `#16463 `__: DOC: Improve assert_warns docstring with example +* `#16464 `__: MAINT: Bump hypothesis from 5.15.1 to 5.16.0 +* `#16465 `__: DOC: Fix development_workflow links +* `#16468 `__: BUG: fix GCC 10 major version comparison +* `#16471 `__: BLD: install mingw32 v7.3.0 for win32 +* `#16472 `__: DOC: Fixes for 18 broken links +* `#16474 `__: MAINT: use zip instead of range in piecewise +* `#16476 `__: ENH: add ``norm=forward,backward`` to numpy.fft functions +* `#16482 `__: SIMD: Optimize the performace of np.packbits in ARM-based machine. +* `#16485 `__: BUG: Fix result when a gufunc output broadcasts the inputs. +* `#16500 `__: DOC: Point Contributing page to new NEP 45 +* `#16501 `__: MAINT: make Py_SET_SIZE and Py_SET_TYPE macros a bit safer +* `#16503 `__: BUG:random: Error when ``size`` is smaller than broadcast input... +* `#16504 `__: DOC: Correct MV Normal sig +* `#16505 `__: BUG: raise IEEE exception on AIX +* `#16506 `__: DOC: only single-polynomial fitting in np.polynomial.Polynomial.fit() +* `#16510 `__: DOC: Minor rounding correction in Generator.binomial +* `#16514 `__: STY: trivial doc style fix in NEP 45. +* `#16515 `__: ENH: add type stubs from numpy-stubs +* `#16519 `__: BUG: f2py: make callbacks threadsafe +* `#16520 `__: STY: f2py: replace \t by whitespace for readability +* `#16522 `__: MAINT:ARMHF Fix detecting feature groups NEON_HALF and NEON_VFPV4 +* `#16523 `__: MAINT: Improve buffer speed +* `#16524 `__: MAINT: f2py: move thread-local declaration definition to common... +* `#16529 `__: BUG: Fix cython warning in random/_common.pyx. +* `#16530 `__: MAINT: Bump pytest from 5.4.2 to 5.4.3 +* `#16532 `__: BUG: Remove non-threadsafe sigint handling from fft calculation +* `#16540 `__: SIMD: SSE2 intrinsic implementation for float64 input of np.enisum +* `#16551 `__: BUG: Ensure SeedSequence 0-padding does not collide with spawn... +* `#16554 `__: DEP: Remove deprecated numeric types and deprecate remaining +* `#16555 `__: CI: drop win32 3.7, 3.6 builds +* `#16556 `__: MAINT: simplifying annotations for np.core.from_numeric +* `#16558 `__: ENH: make typing module available at runtime +* `#16570 `__: ENH: Throw TypeError on operator concat on Numpy Arrays +* `#16571 `__: TST: Add new tests for array coercion +* `#16572 `__: BUG: fix sin/cos bug when input is strided array +* `#16574 `__: MAINT: fix name of first parameter to dtype constructor in type... +* `#16581 `__: DOC: Added an example for np.transpose(4d_array) +* `#16583 `__: MAINT: changed np.generic arguments to positional-only +* `#16589 `__: MAINT: Remove nickname from polynomial classes. +* `#16590 `__: DOC: Clarify dtype default for logspace and geomspace +* `#16591 `__: DOC: Disallow complex args in arange +* `#16592 `__: BUG: Raise TypeError for float->timedelta promotion +* `#16594 `__: ENH: Add ``__f2py_numpy_version__`` attribute to Fortran modules. +* `#16596 `__: BUG: Fix reference count leak in mapping.c +* `#16601 `__: MAINT: Move and improve ``test_ignore_nan_ulperror``. +* `#16603 `__: DOC: make addition of types a "new feature" in release notes +* `#16605 `__: MAINT: Avx512 intrinsics implementation for float64 input np.log +* `#16606 `__: MAINT: Bump pytest-cov from 2.9.0 to 2.10.0 +* `#16607 `__: MAINT: Bump hypothesis from 5.16.0 to 5.16.1 +* `#16613 `__: MAINT: bump mypy version to 0.780 +* `#16617 `__: BLD: Openblas 0.3.10 +* `#16618 `__: ENH: add annotation for abs +* `#16619 `__: BLD: check if std=c99 is really required +* `#16620 `__: MAINT, CI: disable Shippable cache +* `#16621 `__: BENCH: Expand array-creation benchmarks +* `#16622 `__: MAINT: Implemented two dtype-related TODO's +* `#16623 `__: BUG: Initialize stop-reading in array_from_text +* `#16627 `__: DOC: Updated documentation for numpy.squeeze +* `#16629 `__: ENH: add tool to find functions missing types +* `#16630 `__: ENH,BUG:distutils Remove the origins from the implied features +* `#16633 `__: MAINT: lib: Some code clean up in loadtxt +* `#16635 `__: BENCH: remove obsolete goal_time param +* `#16639 `__: BUG: Fix uint->timedelta promotion to raise TypeError +* `#16642 `__: MAINT: Replace ``PyUString_GET_SIZE`` with ``PyUnicode_GetLength``. +* `#16643 `__: REL: Fix outdated docs link +* `#16644 `__: MAINT: Improve performance of np.full +* `#16646 `__: TST: add a static typing test for memoryviews as ArrayLikes +* `#16647 `__: ENH: Added annotations to 8 functions from np.core.fromnumeric +* `#16648 `__: REL: Update master after 1.19.0 release. +* `#16650 `__: ENH: Allow genfromtxt to unpack structured arrays +* `#16651 `__: MAINT: Prefer generator expressions over list comprehensions... +* `#16653 `__: DOC: cross-reference numpy.dot and numpy.linalg.multi_dot +* `#16658 `__: MAINT: Bump hypothesis from 5.16.1 to 5.16.3 +* `#16659 `__: MAINT: Bump mypy from 0.780 to 0.781 +* `#16664 `__: DOC: Add lib.format.open_memmap to autosummary. +* `#16666 `__: BUG: Fix bug in AVX complex absolute while processing array of... +* `#16669 `__: MAINT: remove blacklist/whitelist terms +* `#16671 `__: DOC: Simplify and update git setup page +* `#16674 `__: TST: Add extra debugging information to CPU features detection +* `#16675 `__: ENH: Add support for file like objects to np.core.records.fromfile +* `#16683 `__: DOC: updated gcc minimum recommend version to build from source +* `#16684 `__: MAINT: Allow None to be passed to certain generic subclasses +* `#16690 `__: DOC: fixed docstring for descr_to_dtype +* `#16691 `__: DOC: Remove "matrix" from ``triu`` docstring. +* `#16696 `__: MAINT: add py.typed sentinel to package manifest +* `#16699 `__: MAINT: Fixup quantile tests to not use ``np.float`` +* `#16702 `__: BLD: Add CPU entry for Emscripten / WebAssembly +* `#16704 `__: TST: Disable Python 3.9-dev testing. +* `#16706 `__: DOC: Add instruction about stable symlink +* `#16708 `__: MAINT: Disable use_hugepages in case of ValueError +* `#16709 `__: DOC: Add dep directive to alen docstring. +* `#16710 `__: ENH, BLD: Add RPATH support for AIX +* `#16718 `__: DOC: fix typo +* `#16720 `__: BUG: Fix PyArray_SearchSorted signature. +* `#16723 `__: NEP: Initial draft for NEP 43 for extensible ufuncs +* `#16729 `__: ENH: Add annotations to the last 8 functions in numpy.core.fromnumeric +* `#16730 `__: ENH: Use f90 compiler specified in f2py command line args for... +* `#16731 `__: DOC: reword random c-api introduction, cython is documented in... +* `#16735 `__: DOC: Tweak a sentence about broadcasting. +* `#16736 `__: DOC: Prepend ``ma.`` to references in ``numpy.ma`` +* `#16738 `__: DOC: Remove redundant word +* `#16742 `__: DOC: add unique() to See Also of repeat() +* `#16743 `__: DOC: add example to unique() and make connection to repeat() +* `#16747 `__: MAINT: Chaining exceptions in numpy/core/_internal.py +* `#16752 `__: BLD: add manylinux1 OpenBlAS 0.3.10 hashes and test for them +* `#16757 `__: DOC: Add Matti Picus to steering council page +* `#16759 `__: ENH: make dtype generic over scalar type +* `#16760 `__: DOC: Added a section in the 'Iterating over arrays' doc page... +* `#16761 `__: MAINT: Tidy exception chaining in _datasource.py +* `#16762 `__: MAINT: Fixes for deprecated functions in scalartypes.c.src +* `#16764 `__: MAINT: Bump mypy from 0.781 to 0.782 +* `#16765 `__: MAINT: Bump hypothesis from 5.16.3 to 5.19.0 +* `#16767 `__: ENH: Update NumPy logos +* `#16770 `__: MAINT: Remove unneeded call to PyUnicode_READY +* `#16771 `__: MAINT: Fix deprecated functions in scalarapi.c +* `#16775 `__: DOC: switch to logo with text +* `#16777 `__: BUG: Added missing return after raising error in methods.c +* `#16778 `__: NEP: Update NEP 42 to note the issue of circular references +* `#16782 `__: ENH, TST: Bring the NumPy C SIMD vectorization interface "NPYV"... +* `#16786 `__: BENCH: Add basic benchmarks for scalar indexing and assignment +* `#16789 `__: BUG: fix decode error when building and get rid of warn +* `#16792 `__: DOC: Minor RST formatting. +* `#16793 `__: BLD, MAINT: update cython to 0.29.21 +* `#16794 `__: TST: Upgrade to Python 3.8 for DEBUG testing. +* `#16798 `__: DOC: Fix RST/numpydoc standard. +* `#16800 `__: MAINT: Move typing tests +* `#16802 `__: MAINT: Explicitly disallow object user dtypes +* `#16805 `__: DOC: add example to corrcoef function +* `#16806 `__: DOC: adding docs on passing dimensions as tuple to ndindex +* `#16807 `__: BUG, MAINT: Remove overzealous automatic RST link +* `#16811 `__: DOC: Add explanation of 'K' and 'A' layout options to 'asarray*'... +* `#16814 `__: DOC: Add a reST label to /user/building.rst +* `#16815 `__: BUG: fix mgrid output for lower precision float inputs +* `#16816 `__: BLD: temporarily disable OpenBLAS hash checks +* `#16817 `__: BUG: Do not inherit flags from the structured part of a union... +* `#16819 `__: DOC: replace dec.slow with pytest.mark.slow +* `#16820 `__: MAINT: Make void scalar to array creation copy when dtype is... +* `#16821 `__: DOC: fix inconsistent parameter name in np.ndindex docstring +* `#16822 `__: MAINT: setuptools 49.2.0 emits a warning, avoid it +* `#16824 `__: DOC: add examples to random number generator pages +* `#16826 `__: DOC: describe ufunc copy behavior when input and output overlap +* `#16827 `__: MAINT: Fix ``runtest.py`` warning. +* `#16829 `__: DOC,BLD: Add pandas to doc_requirements.txt +* `#16831 `__: MAINT: fix sphinx deprecation +* `#16834 `__: Avoid using uninitialized bytes in getlimits.py. +* `#16835 `__: DOC: Explaining why datetime64 doesn't work for allclose + isclose +* `#16836 `__: DOC: improve SIMD features tables +* `#16837 `__: BLD: update openblas hashes, re-enable check +* `#16838 `__: MAINT: Remove code that will never run +* `#16840 `__: MAINT: Bump hypothesis from 5.19.0 to 5.19.1 +* `#16841 `__: BUG: linspace should round towards -infinity +* `#16845 `__: TST: Disable shippable until we can fix it. +* `#16847 `__: MAINT: Remove Duplicated Code (function extract rmap) +* `#16848 `__: MAINT: Remove Duplicated Code +* `#16849 `__: MAINT: Change for loop (range -> for each) +* `#16850 `__: DEP: Deprecate NumPy object scalars +* `#16854 `__: DOC: clarify whats required for new features see #13924 +* `#16857 `__: MAINT: fix new compiler warnings on clang +* `#16858 `__: BUG: fix the search dir of dispatch-able sources +* `#16860 `__: MAINT: Remove deprecated python function 'file()' +* `#16868 `__: BUG: Validate output size in bin- and multinomial +* `#16870 `__: BLD, MAINT: Pin setuptools +* `#16871 `__: BUG: Update compiler check for AVX-512F +* `#16874 `__: TST, MAINT: fix the test for ``np.ones`` +* `#16878 `__: DOC: edit to the documentation of lib/polynomial.py/polyfit +* `#16879 `__: MAINT: Configure hypothesis in ``np.test()`` for determinism,... +* `#16882 `__: BLD: Remove unused pip install +* `#16883 `__: BUG,DOC: Fix bad MPL kwarg in docs +* `#16886 `__: DOC: Fix types including curly braces +* `#16887 `__: DOC: Remove the links for ``True`` and ``False`` +* `#16888 `__: ENH: Integrate the new CPU dispatcher with umath generator +* `#16890 `__: TST, BUG: Re-raise MemoryError exception in test_large_zip's... +* `#16894 `__: DOC: Fix wrong markups in ``arrays.dtypes`` +* `#16896 `__: DOC: Remove links for C codes +* `#16897 `__: DOC: Fix the declarations of C fuctions +* `#16899 `__: MNT: also use Py_SET_REFCNT instead of Py_REFCNT +* `#16900 `__: MAINT: Chaining exceptions in numpy/__init__.py +* `#16907 `__: DOC: update val to be scalar or array like optional closes #16901 +* `#16910 `__: MAINT: Bump hypothesis from 5.19.1 to 5.20.2 +* `#16911 `__: ENH: Speed up trim_zeros +* `#16914 `__: BUG: Fix string/bytes to complex assignment +* `#16917 `__: DOC: Add correctness vs strictness consideration for np.dtype +* `#16919 `__: DOC: Add ufunc docstring to generated docs. +* `#16925 `__: REL: Update master after 1.19.1 release. +* `#16931 `__: Revert "Merge pull request #16248 from alexrockhill/edge" +* `#16935 `__: ENH: implement NEP-35's ``like=`` argument +* `#16936 `__: BUG: Fix memory leak of buffer-info cache due to relaxed strides +* `#16938 `__: ENH,API: Store exported buffer info on the array +* `#16940 `__: BLD: update OpenBLAS build +* `#16941 `__: BUG: Allow array-like types to be coerced as object array elements +* `#16943 `__: DEP: Deprecate size-one ragged array coercion +* `#16944 `__: Change the name of the folder "icons" to "logo". +* `#16949 `__: ENH: enable colors for ``runtests.py --ipython`` +* `#16950 `__: DOC: Clarify input to irfft/irfft2/irfftn +* `#16952 `__: MAINT: Bump hypothesis from 5.20.2 to 5.23.2 +* `#16953 `__: update numpy/lib/arraypad.py with appropriate chain exception +* `#16957 `__: MAINT: Use arm64 instead of aarch64 on travisCI. +* `#16962 `__: MAINT: Chain exception in ``distutils/fcompiler/environment.py``. +* `#16966 `__: MAINT: Added the ``order`` parameter to ``np.array()`` +* `#16969 `__: ENH: Add Neon SIMD implementations for add, sub, mul, and div +* `#16973 `__: DOC: Fixed typo in lib/recfunctions.py +* `#16974 `__: TST: Add pypy win32 CI testing. +* `#16982 `__: ENH: Increase the use of ``Literal`` types +* `#16986 `__: ENH: Add NumPy declarations to be used by Cython 3.0+ +* `#16988 `__: DOC: Add the new NumPy logo to Sphinx pages +* `#16991 `__: MAINT: Bump hypothesis from 5.23.2 to 5.23.9 +* `#16992 `__: MAINT: Bump pytest from 5.4.3 to 6.0.1 +* `#16993 `__: BLD: pin setuptools < 49.2.0 +* `#16996 `__: DOC: Revise glossary page +* `#17002 `__: DOC: clip() allows arguments. +* `#17009 `__: NEP: Updated NEP-35 with keyword-only instruction +* `#17010 `__: BUG: Raise correct errors in boolean indexing fast path +* `#17013 `__: MAINT: Simplify scalar power +* `#17014 `__: MAINT: Improve error handling in umathmodule setup +* `#17022 `__: DOC: Fix non-matching pronoun. +* `#17028 `__: DOC: Disclaimer for FFT library +* `#17029 `__: MAINT: Add error return to all casting functionality and NpyIter +* `#17033 `__: BUG: fix a compile and a test warning +* `#17036 `__: DOC: Clarify that ``np.char`` comparison functions always return... +* `#17039 `__: DOC: Use a less ambiguous example for array_split +* `#17041 `__: MAINT: Bump hypothesis from 5.23.9 to 5.23.12 +* `#17048 `__: STY: core._internal style fixups +* `#17050 `__: MAINT: Remove _EXTRAFLAGS variable +* `#17050 `__: MAINT: change ``for line in open()`` to ``with open() as f`` +* `#17052 `__: MAINT: Delete obsolete conversion to list +* `#17053 `__: BUG: fix typo in polydiv that prevented promotion to poly1d +* `#17055 `__: MAINT: Replace lambda function by list comprehension +* `#17058 `__: MAINT: Revert boolean casting back to elementwise comparisons... +* `#17059 `__: BUG: fix pickling of arrays larger than 2GiB +* `#17062 `__: API, BUG: Raise error on complex input to i0 +* `#17063 `__: MAINT: Remove obsolete conversion to set +* `#17067 `__: DEP: lib: Remove the deprecated financial functions. +* `#17068 `__: MAINT, BUG: Remove uses of PyString_FromString. +* `#17074 `__: DOC: use the pydata_sphinx_theme +* `#17078 `__: DOC: Fixes duplication of toctree content (Closes #17077) +* `#17091 `__: MAINT: Bump pytest-cov from 2.10.0 to 2.10.1 +* `#17092 `__: MAINT: Bump hypothesis from 5.23.12 to 5.26.0 +* `#17093 `__: NEP: Adjust NEP-35 to make it more user-accessible +* `#17104 `__: ENH: Add placeholder stubs for all sub-modules +* `#17109 `__: MAINT: Split einsum into multiple files +* `#17112 `__: BUG: Handle errors from the PyCapsule API +* `#17115 `__: DOC: Fix spacing in vectorize doc +* `#17116 `__: API: Remove ``np.ctypeslib.ctypes_load_library`` +* `#17119 `__: DOC: make spacing consistent in NEP 41 bullet points +* `#17121 `__: BUG: core: fix ilp64 blas dot/vdot/... for strides > int32 max +* `#17123 `__: ENH: allow running mypy through runtests.py +* `#17127 `__: MAINT: Remove duplicated symbols from link step +* `#17129 `__: BLD: Check for reduce intrinsics and AVX512BW mask operations +* `#17132 `__: MAINT: Chain some exceptions in arraysetops. +* `#17133 `__: MAINT: Chain ValueError in ma.timer_comparison +* `#17137 `__: API,MAINT: Rewrite promotion using common DType and common instance +* `#17141 `__: MAINT: Make arrayprint str and repr the ndarray defaults. +* `#17142 `__: DOC: NEP-42: Fix a few typos. +* `#17143 `__: MAINT: Change handling of the expired financial functions. +* `#17144 `__: ENH: Add annotations to 3 functions in ``np.core.function_base`` +* `#17145 `__: MAINT, BUG: Replace uses of PyString_AsString. +* `#17146 `__: MAINT: ``Replace PyUString_*`` by ``PyUnicode_*`` equivalents. +* `#17149 `__: MAINT: Replace PyInt macros with their PyLong replacement +* `#17150 `__: ENH: Add support for the abstract scalars to cython code +* `#17151 `__: BUG: Fix incorrect cython definition of npy_cfloat +* `#17152 `__: MAINT: Clean up some ``Npy_`` vs ``Py_`` macro usage +* `#17154 `__: DOC: Remove references to PyCObject +* `#17159 `__: DOC: Update numpy4matlab +* `#17160 `__: Clean up some more bytes vs unicode handling +* `#17161 `__: BUG: Remove Void special case for "safe casting" +* `#17163 `__: MAINT: Remove redundant headers +* `#17164 `__: MAINT: Remove NPY_COPY_PYOBJECT_PTR +* `#17167 `__: BLD: Merge the npysort library into multiarray +* `#17168 `__: TST: Add tests mapping out the rules for metadata in promotion +* `#17171 `__: BUG: revert trim_zeros changes from gh-16911 +* `#17172 `__: ENH: Make ``np.complexfloating`` generic w.r.t. ``np.floating`` +* `#17176 `__: MAINT/ENH: datetime: remove calls to PyUnicode_AsASCIIString,... +* `#17180 `__: ENH: Added missing methods to ``np.flatiter`` +* `#17181 `__: DOC: Correct error in description of ndarray.base +* `#17182 `__: DOC: Document ``dtype.metadata`` +* `#17186 `__: MAINT: Use utf8 strings in more of datetime +* `#17188 `__: MAINT: Add placeholder stubs for ``ndarray`` and ``generic`` +* `#17191 `__: MAINT: Bump hypothesis from 5.26.0 to 5.30.0 +* `#17193 `__: MAINT: Remove some callers of functions in numpy.compat +* `#17195 `__: ENH: Make the window functions exactly symmetric +* `#17197 `__: MAINT: Improve error handling in npy_cpu_init +* `#17199 `__: DOC: Fix the documented signatures of four ``ufunc`` methods +* `#17201 `__: MAINT: Make the ``NPY_CPU_DISPATCH_CALL`` macros expressions not... +* `#17204 `__: DOC: Fixed headings for tutorials so they appear at new theme... +* `#17210 `__: DOC: Canonical_urls +* `#17214 `__: MAINT: Fix various issues with the ``np.generic`` annotations +* `#17215 `__: DOC: Use official MATLAB spelling in numpy-for-matlab-users.rst +* `#17219 `__: BLD: enabled negation of library choices in NPY_*_ORDER +* `#17220 `__: BUG, DOC: comment out metadata added via javascript +* `#17222 `__: MAINT, DOC: move informational files from numpy.doc.*.py to their... +* `#17223 `__: MAINT: use sysconfig not distutils.sysconfig where possible +* `#17225 `__: BUG: Fix dimension discovery of within array ragged cases +* `#17227 `__: DOC: Added templates for different types of issues. +* `#17233 `__: DEP: Deprecated ndindex.ndincr +* `#17235 `__: MAINT: Remove old PY_VERSION_HEX and sys.version_info code +* `#17237 `__: BUG: Avoid using ``np.random`` in typing tests. +* `#17238 `__: DOC: Use SPDX license expressions with correct license +* `#17239 `__: DOC: Fix link quick-start in old random API functions +* `#17240 `__: MAINT: added exception chaining in shape_base.py +* `#17241 `__: MAINT: ``__array_interface__`` data address cannot be bytes +* `#17242 `__: MAINT: Run slow CI jobs earlier so builds finishes sooner +* `#17247 `__: ENH: Add tool to help speed up Travis CI +* `#17250 `__: DOC: Fix docstring cross-referencing +* `#17252 `__: DOC: Added a PR "Reviewer guidelines" document. +* `#17257 `__: DOC: work around a bug in the new theme +* `#17258 `__: SIMD: add fused multiply subtract/add intrinics for all supported... +* `#17259 `__: MAINT: Bump hypothesis from 5.30.0 to 5.33.0 +* `#17260 `__: MAINT: Bump pydata-sphinx-theme from 0.3.2 to 0.4.0 +* `#17263 `__: DOC: add new glossary terms +* `#17264 `__: DOC: remove some glosssary terms +* `#17267 `__: TST: Fix the path to ``mypy.ini`` in ``runtests.py`` +* `#17268 `__: BUG: sysconfig attributes/distutils issue +* `#17273 `__: ENH: Annotate the arithmetic operations of ``ndarray`` and ``generic`` +* `#17278 `__: MAINT: Merge together index page content into a single file +* `#17279 `__: DOC: Fix a typo in shape_base. +* `#17284 `__: ENH: Pass optimizations arguments to asv build +* `#17285 `__: DEP: Change the financial name access warning to DeprecationWarning +* `#17288 `__: REL: Update master after 1.19.2 release. +* `#17289 `__: MAINT: Simplify ufunc pickling +* `#17290 `__: MAINT: Cleanup some pystring macros +* `#17292 `__: MAINT: Replace remaining PyString macros. +* `#17293 `__: MAINT: Replace PyUString_Check by PyUnicode_Check. +* `#17295 `__: BUG,ENH: fix pickling user-scalars by allowing non-format buffer... +* `#17296 `__: MAINT: Replace some ``pyint_*`` macros defined in ``npy_3kcompat``. +* `#17297 `__: BLD: set upper versions for build dependencies +* `#17299 `__: MAINT: (dtype-transfer) make copyswapn and legacy cast wrapper... +* `#17300 `__: MAINT: Replace PyBaseString_Check by PyUnicode_Check +* `#17302 `__: MAINT: Replace a couple of missed npy_3kcompat macros +* `#17304 `__: BUILD: pin pygments to 2.6.1, 2.7.0 breaks custom NumPyC lexer +* `#17307 `__: MAINT: Bump hypothesis from 5.33.0 to 5.35.1 +* `#17308 `__: MAINT: Bump pytest from 6.0.1 to 6.0.2 +* `#17309 `__: MAINT: Move the ``fromnumeric`` annotations to their own stub file +* `#17312 `__: MAINT: Syntax-highlight .src files on github +* `#17313 `__: MAINT: Mark vendored/generated files in .gitattributes +* `#17315 `__: MAINT: Cleanup f2py/cfuncs.py +* `#17319 `__: BUG: Set deprecated fields to null in PyArray_InitArrFuncs +* `#17320 `__: BUG: allow registration of hard-coded structured dtypes +* `#17326 `__: ENH: Add annotations for five array construction functions +* `#17329 `__: DOC: Fix incorrect ``.. deprecated::`` syntax that led to this... +* `#17330 `__: DOC: improve ``issubdtype`` and scalar type docs +* `#17331 `__: DOC: Remove the tables of scalar types, and use ``..autoclass``... +* `#17332 `__: DOC, BLD: update lexer highlighting and make numpydocs a regular... +* `#17334 `__: MAINT: Chaining exceptions in npyio.py +* `#17337 `__: NEP: Regenerate table in NEP 29 (add numpy 1.18 and 1.19 to list) +* `#17338 `__: DOC: Fix syntax errors in docstrings for versionchanged, versionadded +* `#17340 `__: SIMD: Add partial/non-contig load and store intrinsics for 32/64-bit +* `#17344 `__: ENH, BLD: Support for the NVIDIA HPC SDK nvfortran compiler +* `#17346 `__: BLD,BUG: Fix a macOS build failure when ``NPY_BLAS_ORDER=""`` +* `#17350 `__: DEV: Add PR prefix labeler and numpy prefix mapping +* `#17352 `__: DOC: Guide to writing how-tos +* `#17353 `__: DOC: How-to guide for I/O +* `#17354 `__: DOC: clarify residuals return param +* `#17356 `__: ENH: Add Npy__PyLong_AsInt function. +* `#17357 `__: MAINT: Bump hypothesis from 5.35.1 to 5.35.3 +* `#17364 `__: MAINT: Finish replacing PyInt_Check +* `#17369 `__: DOC: distutils: Remove an obsolete paragraph. +* `#17370 `__: NEP: Edit nep-0042 for more clarity +* `#17372 `__: ENH: Add annotations for remaining ``ndarray`` / ``generic`` non-magic... +* `#17373 `__: BUG: Fixes module data docstrings. +* `#17375 `__: DOC: Fix default_rng docstring +* `#17377 `__: BUG: ensure _UFuncNoLoopError can be pickled +* `#17380 `__: Minor grammatical correction in quickstart doc. +* `#17382 `__: DOC: NumPy restyling for pydata theme +* `#17383 `__: MAINT: Fix docstring for np.matmul +* `#17386 `__: MAINT: Bump hypothesis from 5.35.3 to 5.36.1 +* `#17388 `__: MAINT: Remove old debug print statement. +* `#17391 `__: DOC: Replace "About NumPy" with "Document conventions" +* `#17392 `__: DOC: Update info on doc style rules +* `#17393 `__: BUG: Fix default void, datetime, and timedelta in array coercion +* `#17394 `__: ENH: Implement sliding window +* `#17396 `__: MAINT: Replace append_metastr_to_string function. +* `#17399 `__: BLD: Fixed ARGOUTVIEWM memory deallocation. Closes #17398. +* `#17400 `__: DOC: rm incorrect alias from recarray user article. +* `#17401 `__: MAINT: Rewrite can-cast logic in terms of NEP 42 +* `#17402 `__: DOC: Add arraysetops to an autosummary +* `#17404 `__: MAINT: Replace PyUString_ConcatAndDel in nditer_constr.c. +* `#17405 `__: MAINT: Replace PyUString_ConcatAndDel in mapping.c. +* `#17406 `__: ENH: Replace the module-level ``__getattr__`` with explicit type... +* `#17407 `__: DOC: in PR template, set expectations for PR review timeline +* `#17409 `__: MAINT: Cleanup remaining PyUString_ConcatAndDel use. +* `#17410 `__: API: Special case how numpy scalars are coerced to signed integer +* `#17411 `__: TST: Mark the typing tests as slow +* `#17412 `__: DOC: Fix a parameter type in the ``putmask`` docs +* `#17418 `__: DOC: adding operational form documentation for array ops +* `#17419 `__: DEP: Deprecate coercion to subarray dtypes +* `#17421 `__: BUG: Fix memory leak in array-coercion error paths +* `#17422 `__: MAINT: chains nested try-except in numpy/ma/core.py +* `#17423 `__: DOC: Remove bogus reference to _a_ +* `#17424 `__: DOC: Fix formatting issues in description of .c.src files +* `#17427 `__: NEP: nep-0029 typo correction +* `#17429 `__: MAINT: Move aliases for common scalar unions to ``numpy.typing`` +* `#17430 `__: BUG: Fix memoryleaks related to NEP 37 function overrides +* `#17431 `__: DOC: Fix the links for ``Ellipsis`` +* `#17432 `__: DOC: add references to einops and opt_einsum +* `#17433 `__: MAINT : Disable 32 bit PyPy CI testing on Windows. +* `#17435 `__: DOC: Security warning for issues template +* `#17436 `__: DOC: Fix "Feature request" spelling in issue templates +* `#17438 `__: MAINT: Chaining exception in numpy\numpy\ma\mrecords.py +* `#17440 `__: DOC: Cleaner template for PRs +* `#17442 `__: MAINT: fix exception chaining in format.py +* `#17443 `__: ENH: Warn on unsupported Python 3.10+ +* `#17444 `__: ENH: Add ``Typing :: Typed`` to the PyPi classifier +* `#17445 `__: DOC: Fix the references for macros +* `#17447 `__: NEP: update NEP 42 with discussion of type hinting applications +* `#17448 `__: DOC: Remove CoC pages from Sphinx +* `#17453 `__: MAINT: Chain exceptions in "_polybase.py" +* `#17455 `__: MAINT: Bump hypothesis from 5.36.1 to 5.37.0 +* `#17456 `__: ENH: add dtype option to numpy.lib.function_base.cov and corrcoef +* `#17457 `__: BUG: Fixes incorrect error message in numpy.ediff1d +* `#17459 `__: DOC: update code of conduct URL +* `#17464 `__: DOC: Add some entries for C types and macros +* `#17465 `__: ENH: Add annotations for bitwise operations +* `#17468 `__: DOC: add some missing scalar aliases +* `#17472 `__: TST: Fix doctest for full_like +* `#17473 `__: MAINT: py3k: remove os.fspath and os.PathLike backports +* `#17474 `__: MAINT: Move the ``np.core.numeric`` annotations to their own stub... +* `#17479 `__: ENH: type ``np.unicode_`` as ``np.str_`` +* `#17481 `__: DOC: Fix the entries for members of structures +* `#17483 `__: DOC: Fix the references for ``random.*`` +* `#17485 `__: BLD: circleCI- merge before build, add -n to sphinx +* `#17487 `__: MAINT: Remove duplicate placeholder annotations +* `#17493 `__: DOC: New round of NEP 42 edits +* `#17497 `__: DOC: Use consistent lowercase on docs landing page +* `#17498 `__: MAINT: fix incompatible type comparison in numpy.lib.utils.info +* `#17501 `__: BUG: Fix failures in master related to userdtype registeration +* `#17502 `__: BUG: remove ``sys`` from the type stubs +* `#17503 `__: DOC: Fix empty 'C style guide' page +* `#17504 `__: DOC: Rename 'Quickstart tutorial' +* `#17508 `__: ENH: Added the Final feature for all constants +* `#17510 `__: DOC: Fewer blank lines in PR template +* `#17520 `__: DOC: Display real license on license page +* `#17521 `__: DOC: Add docstrings for some scalar types +* `#17523 `__: DOC: Update top links in landing page +* `#17525 `__: CI: Make merge ref grabbing conditional on the PR being active +* `#17527 `__: DOC: Fix Bool types in C functions +* `#17528 `__: Doc: Fix some links and typos +* `#17529 `__: MAINT: Cleanup compatibility code for pathlib +* `#17534 `__: DOC: Fix a typo +* `#17535 `__: ENH: add function to get broadcast shape from a given set of... +* `#17536 `__: BUG: Fixed crash on self-referential dtypes +* `#17537 `__: MAINT: Bump hypothesis from 5.37.0 to 5.37.1 +* `#17538 `__: MAINT: Bump pydata-sphinx-theme from 0.4.0 to 0.4.1 +* `#17539 `__: MAINT: Bump mypy from 0.782 to 0.790 +* `#17540 `__: ENH: Make ``np.number`` generic with respect to its precision +* `#17541 `__: CI: fix conditional for PR merge command +* `#17546 `__: MAINT: explicit disabling ``CCompilerOpt`` in F2PY +* `#17548 `__: BUG: Cygwin Workaround for #14787 on affected platforms +* `#17549 `__: DOC: Fix the entries of C functions +* `#17555 `__: DOC: Fix wrong blockquotes +* `#17558 `__: DOC: MAINT: Add NEP 43 links to NEP 42 +* `#17559 `__: DOC: Remove directives for some constants +* `#17564 `__: MAINT: Update the annotations in ``np.core.numeric`` +* `#17570 `__: DOC: Add the entry for ``NPY_FEATURE_VERSION`` +* `#17571 `__: DOC: Fix typos +* `#17572 `__: ENH: Add annotations for three new constants +* `#17576 `__: DOC: Fix Boolean array indexing typo +* `#17577 `__: BUG: Respect dtype of all-zero argument to poly1d +* `#17578 `__: NEP36: include additional feedback +* `#17580 `__: MAINT: Cleanup swig for Python 3. +* `#17581 `__: MAINT: Move the ``np.core.numerictypes`` annotations to their own... +* `#17583 `__: MAINT: Bump hypothesis from 5.37.1 to 5.37.3 +* `#17584 `__: ENH: Add annotations for ``np.core._type_aliases`` +* `#17594 `__: DOC: Typo in lexsort docstring +* `#17596 `__: DEP,BUG: Coercion/cast of array to a subarray dtype will be fixed +* `#17597 `__: TST: Clean up the errors of the typing tests +* `#17598 `__: BUG: Fixed file handle leak in array_tofile. +* `#17601 `__: TST: Fix a broken ``np.core.numeric`` test +* `#17603 `__: MAINT: Mark dead code as intentional for clang. +* `#17607 `__: DOC: removed old references to submodule licenses +* `#17608 `__: DOC: Fix typos (general documentation) +* `#17610 `__: Fully qualify license trove classifier +* `#17611 `__: BUG: mac dylib treated as part of extra objects by f2py +* `#17613 `__: ENH: Add annotations for 9 ``ndarray``/``generic`` magic methods +* `#17614 `__: DOC: Fix the document for arrays interface +* `#17618 `__: MAINT: Conversion of some strings to f-strings +* `#17619 `__: DOC: Fix some references +* `#17621 `__: TST: Valid docstring for config_py function show() +* `#17622 `__: MAINT: Conversion of some strings to fstrings, part II +* `#17623 `__: MAINT: Conversion of some strings to fstrings, part III +* `#17624 `__: DOC: Tidy up references to ``str_`` / ``bytes_`` +* `#17625 `__: MAINT: Conversion of some strings to fstrings, part iv +* `#17627 `__: DOC: Fix the references for ``__array_*__`` +* `#17628 `__: DOC: Add entries for macros +* `#17629 `__: DOC: Add ``identity_value`` to ``PyUFuncObject`` +* `#17630 `__: DOC: Replace ``PyCObject`` with ``PyCapsule`` +* `#17633 `__: DOC: Don't use Python highlighting for non-python code +* `#17638 `__: DOC: Fix some references +* `#17639 `__: MAINT: Bump hypothesis from 5.37.3 to 5.38.0 +* `#17641 `__: MAINT, BLD: update to OpenBLAS v0.3.12 +* `#17642 `__: DOC: Fix reference to atleast_1d +* `#17643 `__: ENH: Add annotations for ``np.core._ufunc_config`` +* `#17644 `__: ENH: Add annotations for ``np.core.shape_base`` +* `#17645 `__: BUG: fix np.timedelta64('nat').__format__ throwing an exception +* `#17654 `__: BUG: f2py incorrectly translates dimension declarations. +* `#17655 `__: BLD: Fix installing Numpy on z/OS +* `#17657 `__: NEP: Ensure inner loop signature is complete everywhere +* `#17658 `__: TST: simplify source path names in compilation test +* `#17662 `__: TST: f2py: Add a doctest for ``getlincoef`` +* `#17666 `__: REL: Update master after 1.19.3 release. +* `#17668 `__: TST: Make test suite work in FIPS (140-2) Mode +* `#17670 `__: DOC: f2py: Add a docstring for getarrlen +* `#17672 `__: DOC: Update README badge for travis-ci.com +* `#17673 `__: MAINT: Refine a number of ``np.generic`` annotations +* `#17675 `__: MAINT: Update release documentation and software +* `#17681 `__: SIMD: Add sum intrinsics for float/double. +* `#17682 `__: BUG: (nditer_impl.h) Use ``intp`` instead of ``char *`` for offset... +* `#17689 `__: BUG: Fix small bug in ``make_lite.py``. +* `#17691 `__: DOC: Modify Templates +* `#17692 `__: MAINT: Bump hypothesis from 5.38.0 to 5.41.0 +* `#17693 `__: MAINT: Bump pytz from 2020.1 to 2020.4 +* `#17695 `__: TST: use a more standard workflow for PyPy +* `#17696 `__: REL: Update master after 1.19.4 release. +* `#17699 `__: MAINT: Rename ``DtypeLike`` to ``DTypeLike`` +* `#17700 `__: Fix small typos. +* `#17701 `__: BUG: Fixed an issue where ``.pyi`` files were ignored by numpy... +* `#17703 `__: Fix Doc Typos & Added Example +* `#17706 `__: BUG: Raise promotion error if a DType was provided in array coercion +* `#17708 `__: Improve the einsum bench by adding new bench cases and variable... +* `#17711 `__: ENH: adds type hints to numpy.version +* `#17715 `__: REV: Revert gh-17654 - f2py incorrectly translates dimension... +* `#17717 `__: MAINT: Add more files to ``.gitgnore`` +* `#17720 `__: API: Do not import sliding_window_view to main namespace +* `#17723 `__: MAINT: Do not override ``sliding_window_view`` module to ``numpy`` +* `#17725 `__: NEP: Add NEP-35 instructions on reading like= downstream +* `#17729 `__: BLD: Use importlib to find numpy root directory in distutils +* `#17733 `__: MAINT: ma: Remove unused ``**options`` from MaskedArray ``__new__``... +* `#17735 `__: TST: Remove Python 3.6 CI testing. +* `#17738 `__: BLD, TST: move linux jobs to github actions +* `#17740 `__: MAINT: Bump hypothesis from 5.41.0 to 5.41.2 +* `#17743 `__: BLD, BUG: Fix cblas detection on windows +* `#17745 `__: TST: add pypy3.7 +* `#17748 `__: BLD: compare platform.architecture() correctly +* `#17749 `__: DOC: Add "performance" category to the release notes +* `#17751 `__: BUG: Fix segfault due to out of bound pointer in floatstatus... +* `#17753 `__: BUG: Fix buffer export dtype references +* `#17755 `__: BUG: Fix memory leaks found using valgrind +* `#17758 `__: BLD: Lazy load f2py test utilities +* `#17759 `__: BLD: use BUFFERSIZE=20 in OpenBLAS +* `#17763 `__: SIMD, BUG: fix reuses the previous values during the fallback... +* `#17768 `__: MAINT: update link to website in FUNDING.yml +* `#17773 `__: MAINT: Add BLD and STY to labeler prefixes. +* `#17776 `__: MAINT: Simplify Hypothesis configuration +* `#17787 `__: NEP: Make like= argument added in NEP-35 strict +* `#17788 `__: DOC: Fix up links, code blocks of release note fragments +* `#17796 `__: MAINT: Minor touchups in npyio +* `#17802 `__: MAINT: Update mailmap. +* `#17805 `__: MAINT: Set the ufunc and ndarray ops return type to ``Any`` +* `#17812 `__: Update linalg.py +* `#17815 `__: DOC: Fix empty_like docstring +* `#17823 `__: DOC: Add missing release fragments to ``upcoming_changes``. +* `#17828 `__: BUG: Fix incorrectly passed size in masked processing +* `#17829 `__: MAINT: Bump hypothesis from 5.41.2 to 5.41.3 +* `#17830 `__: TST: Add back durations flag for DEBUG builds. +* `#17832 `__: BUG: Fix subarray dtype used with too large count in fromfile +* `#17833 `__: BUG: Fix pickling of scalars with NPY_LISTPICKLE +* `#17838 `__: DOC: Update the ``numpy.typing`` documentation +* `#17841 `__: DOC: Fixing boilerplate code example +* `#17844 `__: MAINT: Add ``__all__`` to ``numpy.typing`` +* `#17848 `__: DOC: Add release note for gh-16161. +* `#17855 `__: BUG: Fix incorrect C function prototypes/declarations. +* `#17857 `__: MAINT: Prepare for the NumPy 1.20.x branch. +* `#17869 `__: BUG, TST: use python-version not PYTHON_VERSION +* `#17879 `__: BUG: Fix buffer readflag errors and small leaks +* `#17893 `__: DOC: Prepare for 1.20.0 release +* `#17898 `__: MAINT: Remove remaining uses of Python 3.6. +* `#17899 `__: TST: use latest pypy37 not pypy36 +* `#17901 `__: MAINT: clean up a spurious warning in numpy/typing/setup.py +* `#17904 `__: ENH: Speed up default ``where`` in the reduce-like method +* `#17915 `__: TST: remove stray '+' from f-string upgrade +* `#17916 `__: ENH: add support for fujitsu compiler to numpy. +* `#17922 `__: BUG: 'bool' object has no attribute 'ndim' +* `#17931 `__: DOC: Update release notes to mention ``type(dtype) is not np.dtype`` +* `#17990 `__: BUG: Replace f-string in setup.py +* `#18015 `__: BUG: Ignore fewer errors during array-coercion +* `#18016 `__: BUG: Fix a MacOS build failure +* `#18017 `__: TST: Fix crosstalk issues with polynomial str tests. +* `#18018 `__: TST: Ensure tests are not sensitive to execution order +* `#18019 `__: BLD: update to OpenBLAS 0.3.13 +* `#18024 `__: DEP: Futurewarn on requiring __len__ on array-likes +* `#18035 `__: BUG: make a variable volatile to work around clang compiler bug +* `#18049 `__: TST: add back sdist test run +* `#18063 `__: BUG: Fix concatenation when the output is "S" or "U" +* `#18064 `__: BLD, BUG: Fix detecting aarch64 on macOS +* `#18068 `__: REL: Prepare for 1.20.0rc2 release. +* `#18108 `__: BUG, BLD: Generate the main dispatcher config header into the... +* `#18120 `__: BUG, SIMD: Fix _simd module build for 64bit ARM/NEON clang +* `#18127 `__: REL: Update 1.20.x after 1.19.5 release. +* `#18130 `__: BUG: Fix promotion of half and string +* `#18146 `__: BUG, MAINT: improve avx512 mask logical operations +* `#18154 `__: BUG: Promotion between strings and objects was assymetric +* `#18192 `__: MAINT: Use explicit reexports for numpy.typing objects +* `#18201 `__: BUG: Keep ignoring most errors during array-protocol lookup +* `#18219 `__: MAINT: random shuffle: warn on unrecognized objects, fix empty... +* `#18231 `__: BLD: update OpenBLAS to af2b0d02 +* `#18237 `__: DOC: Clarify the type alias deprecation message +* `#18257 `__: BUG: Ensure too many advanced indices raises an exception +* `#18258 `__: MAINT: add an 'apt update' +* `#18259 `__: DOC: Prepare for the NumPy 1.20.0 release. diff --git a/doc/changelog/1.20.1-changelog.rst b/doc/changelog/1.20.1-changelog.rst new file mode 100644 index 000000000000..215cdca3c5e0 --- /dev/null +++ b/doc/changelog/1.20.1-changelog.rst @@ -0,0 +1,36 @@ + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Bas van Beek +* Charles Harris +* Nicholas McKibben + +* Pearu Peterson +* Ralf Gommers +* Sebastian Berg +* Tyler Reddy +* @Aerysv + + +Pull requests merged +==================== + +A total of 15 pull requests were merged for this release. + +* `#18306 `__: MAINT: Add missing placeholder annotations +* `#18310 `__: BUG: Fix typo in ``numpy.__init__.py`` +* `#18326 `__: BUG: don't mutate list of fake libraries while iterating over... +* `#18327 `__: MAINT: gracefully shuffle memoryviews +* `#18328 `__: BUG: Use C linkage for random distributions +* `#18336 `__: CI: fix when GitHub Actions builds trigger, and allow ci skips +* `#18337 `__: BUG: Allow unmodified use of isclose, allclose, etc. with timedelta +* `#18345 `__: BUG: Allow pickling all relevant DType types/classes +* `#18351 `__: BUG: Fix missing signed_char dependency. Closes #18335. +* `#18352 `__: DOC: Change license date 2020 -> 2021 +* `#18353 `__: CI: CircleCI seems to occasionally time out, increase the limit +* `#18354 `__: BUG: Fix f2py bugs when wrapping F90 subroutines. +* `#18356 `__: MAINT: crackfortran regex simplify +* `#18357 `__: BUG: threads.h existence test requires GLIBC > 2.12. +* `#18359 `__: REL: Prepare for the NumPy 1.20.1 release. diff --git a/doc/changelog/1.20.2-changelog.rst b/doc/changelog/1.20.2-changelog.rst new file mode 100644 index 000000000000..831cf03324de --- /dev/null +++ b/doc/changelog/1.20.2-changelog.rst @@ -0,0 +1,40 @@ + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Bas van Beek +* Charles Harris +* Christoph Gohlke +* Mateusz Sokół + +* Michael Lamparski +* Sebastian Berg + +Pull requests merged +==================== + +A total of 20 pull requests were merged for this release. + +* `#18382 `__: MAINT: Update f2py from master. +* `#18459 `__: BUG: ``diagflat`` could overflow on windows or 32-bit platforms +* `#18460 `__: BUG: Fix refcount leak in f2py ``complex_double_from_pyobj``. +* `#18461 `__: BUG: Fix tiny memory leaks when ``like=`` overrides are used +* `#18462 `__: BUG: Remove temporary change of descr/flags in VOID functions +* `#18469 `__: BUG: Segfault in nditer buffer dealloc for Object arrays +* `#18485 `__: BUG: Remove suspicious type casting +* `#18486 `__: BUG: remove nonsensical comparison of pointer < 0 +* `#18487 `__: BUG: verify pointer against NULL before using it +* `#18488 `__: BUG: check if PyArray_malloc succeeded +* `#18546 `__: BUG: incorrect error fallthrough in nditer +* `#18559 `__: CI: Backport CI fixes from main. +* `#18599 `__: MAINT: Add annotations for ``dtype.__getitem__``, ``__mul__`` and... +* `#18611 `__: BUG: NameError in numpy.distutils.fcompiler.compaq +* `#18612 `__: BUG: Fixed ``where`` keyword for ``np.mean`` & ``np.var`` methods +* `#18617 `__: CI: Update apt package list before Python install +* `#18636 `__: MAINT: Ensure that re-exported sub-modules are properly annotated +* `#18638 `__: BUG: Fix ma coercion list-of-ma-arrays if they do not cast to... +* `#18661 `__: BUG: Fix small valgrind-found issues +* `#18671 `__: BUG: Fix small issues found with pytest-leaks diff --git a/doc/changelog/1.20.3-changelog.rst b/doc/changelog/1.20.3-changelog.rst new file mode 100644 index 000000000000..df7f1056521a --- /dev/null +++ b/doc/changelog/1.20.3-changelog.rst @@ -0,0 +1,35 @@ + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Anne Archibald +* Bas van Beek +* Charles Harris +* Dong Keun Oh + +* Kamil Choudhury + +* Sayed Adel +* Sebastian Berg + +Pull requests merged +==================== + +A total of 15 pull requests were merged for this release. + +* `#18763 `__: BUG: Correct ``datetime64`` missing type overload for ``datetime.date``... +* `#18764 `__: MAINT: Remove ``__all__`` in favor of explicit re-exports +* `#18768 `__: BLD: Strip extra newline when dumping gfortran version on MacOS +* `#18769 `__: BUG: fix segfault in object/longdouble operations +* `#18794 `__: MAINT: Use towncrier build explicitly +* `#18887 `__: MAINT: Relax certain integer-type constraints +* `#18915 `__: MAINT: Remove unsafe unions and ABCs from return-annotations +* `#18921 `__: MAINT: Allow more recursion depth for scalar tests. +* `#18922 `__: BUG: Initialize the full nditer buffer in case of error +* `#18923 `__: BLD: remove unnecessary flag ``-faltivec`` on macOS +* `#18924 `__: MAINT, CI: treats _SIMD module build warnings as errors through... +* `#18925 `__: BUG: for MINGW, threads.h existence test requires GLIBC > 2.12 +* `#18941 `__: BUG: Make changelog recognize gh- as a PR number prefix. +* `#18948 `__: REL, DOC: Prepare for the NumPy 1.20.3 release. +* `#18953 `__: BUG: Fix failing mypy test in 1.20.x. diff --git a/doc/changelog/1.21.0-changelog.rst b/doc/changelog/1.21.0-changelog.rst new file mode 100644 index 000000000000..947da4da740e --- /dev/null +++ b/doc/changelog/1.21.0-changelog.rst @@ -0,0 +1,769 @@ + +Contributors +============ + +A total of 175 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* @8bitmp3 + +* @DWesl + +* @Endolith +* @Illviljan + +* @Lbogula + +* @Lisa + +* @Patrick + +* @Scian + +* @h-vetinari + +* @h6197627 + +* @jbCodeHub + +* @legoffant + +* @sfolje0 + +* @tautaus + +* @yetanothercheer + +* Abhay Raghuvanshi + +* Adrian Price-Whelan + +* Aerik Pawson + +* Agbonze Osazuwa + +* Aitik Gupta + +* Al-Baraa El-Hag +* Alex Henrie +* Alexander Hunt + +* Alizé Papp + +* Allan Haldane +* Amarnath1904 + +* Amrit Krishnan + +* Andras Deak +* AngelGris + +* Anne Archibald +* Anthony Vo + +* Antony Lee +* Atharva-Vidwans + +* Ayush Verma + +* Bas van Beek +* Bharat Raghunathan +* Bhargav V + +* Brian Soto +* Carl Michal + +* Charles Harris +* Charles Stern + +* Chiara Marmo + +* Chris Barnes + +* Chris Vavaliaris +* Christina Hedges + +* Christoph Gohlke +* Christopher Dahlin + +* Christos Efstathiou + +* Chunlin Fang +* Constanza Fierro + +* Daniel Evans + +* Daniel Montes + +* Dario Mory + +* David Carlier + +* David Stansby +* Deepyaman Datta + +* Derek Homeier +* Dong Keun Oh + +* Dylan Cutler + +* Eric Larson +* Eric Wieser +* Eva Jau + +* Evgeni Burovski +* FX Coudert + +* Faris A Chugthai + +* Filip Ter + +* Filip Trojan + +* François Le Lay + +* Ganesh Kathiresan +* Giannis Zapantis + +* Giulio Procopio + +* Greg Lucas + +* Hollow Man + +* Holly Corbett + +* I-Shen Leong + +* Inessa Pawson +* Isabela Presedo-Floyd +* Ismael Jimenez + +* Isuru Fernando +* Jakob Jakobson +* James Gerity + +* Jamie Macey + +* Jasmin Classen + +* Jody Klymak + +* Joseph Fox-Rabinovitz +* Jérome Eertmans + +* Jérôme Kieffer + +* Kamil Choudhury + +* Kasia Leszek + +* Keller Meier + +* Kenichi Maehashi +* Kevin Sheppard +* Kulin Seth + +* Kumud Lakara + +* Laura Kopf + +* Laura Martens + +* Leo Singer + +* Leonardus Chen + +* Lima Tango + +* Lumir Balhar + +* Maia Kaplan + +* Mainak Debnath + +* Marco Aurélio da Costa + +* Marta Lemanczyk + +* Marten van Kerkwijk +* Mary Conley + +* Marysia Winkels + +* Mateusz Sokół + +* Matt Haberland +* Matt Hall + +* Matt Ord + +* Matthew Badin + +* Matthias Bussonnier +* Matthias Geier +* Matti Picus +* Matías Ríos + +* Maxim Belkin + +* Melissa Weber Mendonça +* Meltem Eren Copur + +* Michael Dubravski + +* Michael Lamparski +* Michal W. Tarnowski + +* Michał Górny + +* Mike Boyle + +* Mike Toews +* Misal Raj + +* Mitchell Faas + +* Mukulikaa Parhari + +* Neil Girdhar + +* Nicholas McKibben + +* Nico Schlömer +* Nicolas Hug + +* Nilo Kruchelski + +* Nirjas Jakilim + +* Ohad Ravid + +* Olivier Grisel +* Pamphile ROY + +* Panos Mavrogiorgos + +* Patrick T. Komiske III + +* Pearu Peterson +* Peter Hawkins + +* Raghuveer Devulapalli +* Ralf Gommers +* Raúl Montón Pinillos + +* Rin Arakaki + +* Robert Kern +* Rohit Sanjay +* Roman Yurchak +* Ronan Lamy +* Ross Barnowski +* Ryan C Cooper +* Ryan Polley + +* Ryan Soklaski +* Sabrina Simao + +* Sayed Adel +* Sebastian Berg +* Shen Zhou + +* Stefan van der Walt +* Sylwester Arabas + +* Takanori Hirano +* Tania Allard + +* Thomas J. Fan + +* Thomas Orgis + +* Tim Hoffmann +* Tomoki, Karatsu + +* Tong Zou + +* Touqir Sajed + +* Tyler Reddy +* Wansoo Kim +* Warren Weckesser +* Weh Andreas + +* Yang Hau +* Yashasvi Misra + +* Zolboo Erdenebaatar + +* Zolisa Bleki + +Pull requests merged +==================== + +A total of 581 pull requests were merged for this release. + +* `#13578 `__: DEP: Deprecate `data_type.dtype` if attribute is not already... +* `#15269 `__: ENH: Implement faster keyword argument parsing capable of ``METH_FASTCALL`` +* `#15271 `__: ENH: Optimize and cleanup ufunc calls and ufunc CheckOverrides +* `#15392 `__: BUG: Remove temporary change of descr/flags in VOID functions +* `#16164 `__: DOC: Add more information about poly1d -> polynomial to reference... +* `#16241 `__: ENH: Warn when reloading numpy or using numpy in sub-interpreter +* `#16370 `__: DOC: Fix for building with sphinx 3 +* `#16588 `__: DOC: unify the docs for np.transpose and ndarray.transpose +* `#16818 `__: DOC: added examples section for rfft2 and irfft2 docstring +* `#16855 `__: DOC: Fix Typo (Wrong argument name) +* `#16987 `__: ENH: Phase unwrapping generalized to arbitrary interval size +* `#17102 `__: SIMD: Optimize the performance of np.packbits in AVX2/AVX512F/VSX. +* `#17122 `__: MAINT: Use numpy version for f2py version. +* `#17492 `__: DEP: Shift correlate mode parsing to C and deprecate inexact... +* `#17586 `__: DEP: Formally deprecate `np.typeDict` +* `#17587 `__: SIMD: Replace raw SIMD of sin/cos with NPYV(universal intrinsics) +* `#17636 `__: MAINT: Bump pydata-sphinx-theme and set logo link to index +* `#17637 `__: DOC: Add module template +* `#17719 `__: ENH: Make `ndarray` generic w.r.t. its shape and dtype +* `#17727 `__: ENH: Added libdivide for floor divide +* `#17736 `__: BUG, Benchmark: fix passing optimization build options to asv +* `#17737 `__: MAINT, Benchmark: print the supported CPU features during the... +* `#17778 `__: ENH: Add annotations for comparison operations +* `#17782 `__: SIMD: Optimize the performance of einsum's submodule multiply... +* `#17789 `__: ENH, SIMD: Add new NPYV intrinsics pack(0) +* `#17790 `__: ENH, SIMD: Add new NPYV intrinsics pack(1) +* `#17791 `__: BLD: Enable Werror=undef in travis +* `#17792 `__: ENH: add support for fujitsu compiler to numpy. +* `#17795 `__: ENH: Add two new `_Like` unions +* `#17817 `__: BUG: Ignore fewer errors during array-coercion +* `#17836 `__: MAINT: Add git rules to ignore all SIMD generated files +* `#17843 `__: ENH: Add a mypy plugin for inferring platform-specific `np.number`... +* `#17847 `__: TST: use latest pypy37 not pypy36 +* `#17852 `__: DOC: Doc for deprecate_with_doc +* `#17853 `__: DOC: Clarify docs of np.resize(). +* `#17861 `__: MAINT: Update master after 1.20.x branch. +* `#17862 `__: Make it clearer that np.interp input must be monotonically increasing +* `#17863 `__: MAINT: Implement new casting loops based on NEP 42 and 43 +* `#17866 `__: DOC: fix typo in glossary.rst +* `#17868 `__: BUG, TST: use python-version not PYTHON_VERSION +* `#17872 `__: DOC: update the release howto for oldest-supported-numpy +* `#17874 `__: MAINT: clean up a spurious warning in numpy/typing/setup.py +* `#17875 `__: DOC: Prepare for 1.20.0 release +* `#17876 `__: DOC: fixed typo in np-indexing.png explaining [-2:] slice in... +* `#17877 `__: BUG: Fix buffer readflag errors and small leaks +* `#17878 `__: BUG: np.arange: Allow `stop` not `start` as sole kwargs. +* `#17881 `__: MAINT: Bump hypothesis from 5.41.3 to 5.41.4 +* `#17883 `__: MAINT: Remove duplicate dictionary entry +* `#17884 `__: BUG: numpy.putmask not respecting writeable flag +* `#17886 `__: ENH: Timestamp development versions. +* `#17887 `__: DOC: Update arraycreation +* `#17888 `__: DOC: Correct sentence/statement composition +* `#17889 `__: DOC: Rename basics to fundamentals + added description +* `#17895 `__: MAINT: Remove remaining uses of Python 3.6. +* `#17896 `__: ENH: Speed up default `where` in the reduce-like method +* `#17897 `__: BUG: merging PR to use -Werror=undef broke another PR +* `#17900 `__: DEP: Finalize unravel_index `dims` alias for `shape` keyword +* `#17906 `__: BUG: Fix a MacOS build failure +* `#17907 `__: BUG: 'bool' object has no attribute 'ndim' +* `#17912 `__: BUG: remove stray '+' from f-string upgrade in numba/extending.py +* `#17914 `__: DOC: Update release notes to mention `type(dtype) is not np.dtype` +* `#17920 `__: NEP: Update NEP 42 and 43 according to the current implementation +* `#17921 `__: BUG: Enforce high >= low on uniform number generators +* `#17929 `__: MAINT: Replace `contextlib_nullcontext` with `contextlib.nullcontext` +* `#17934 `__: DOC: Add information about leak checking and valgrind +* `#17936 `__: TST: Fixed an issue where the typing tests would fail for comparison... +* `#17942 `__: DOC: Clarify savez documentation of naming arrays in output file +* `#17943 `__: [DOC]: Wrong length for underline in docstring. +* `#17945 `__: MAINT: Bump hypothesis from 5.41.4 to 5.41.5 +* `#17950 `__: BUG: Removed empty String from Nag Compiler's Flags +* `#17953 `__: NEP: Accept NEP 42 -- New and extensible DTypes +* `#17955 `__: DOC: Replace {var} in docstrings type annotation with `scalar... +* `#17956 `__: ENH: Use versioneer to manage numpy versions. +* `#17957 `__: TST: Fix crosstalk issues with polynomial str tests. +* `#17958 `__: MAINT: Optimize the performance of count_nonzero by using universal... +* `#17960 `__: TST, BUILD: Add a native x86 baseline build running on ubuntu-20.04 +* `#17962 `__: TST: Ensure tests are not sensitive to execution order +* `#17966 `__: BUG: Add missing decref to arange +* `#17968 `__: ENH: Use more typevars in `np.dtype` +* `#17971 `__: BUG, SIMD: Fix direactive check for AVX512BW of intrinsics npyv_tobits_* +* `#17973 `__: DEP: Futurewarn on requiring __len__ on array-likes +* `#17974 `__: BLD: Fixes for versioneer and setup.py sdist. +* `#17976 `__: DOC: Add/remove spaces in snippets and re-format here and there +* `#17978 `__: MAINT: Update test_requirements and release_requirements. +* `#17981 `__: ENH: Add proper dtype-support to `np.flatiter` +* `#17985 `__: ENH, SIMD: Ditching the old CPU dispatcher(Arithmetic) +* `#17992 `__: DOC: Replace verbatim with reference to local parameter +* `#17993 `__: [DOC] np.kron use double backticks for non-references +* `#17994 `__: SIMD: Optimize the performance of einsum's submodule dot . +* `#17995 `__: MAINT: Bump pytest from 6.0.2 to 6.2.0 +* `#17996 `__: MAINT: Update wheel requirement from <=0.35.1 to <0.36.3 +* `#17997 `__: MAINT: Bump hypothesis from 5.41.5 to 5.43.3 +* `#17998 `__: TST: ignore pytest warning +* `#17999 `__: Replace Numpy with NumPy +* `#18001 `__: BLD, BUG: Fix detecting aarch64 on macOS +* `#18002 `__: DOC: Fix and extend the docstring for np.inner +* `#18007 `__: DOC: Add a brief explanation of float printing +* `#18008 `__: DOC: fix for doctests +* `#18011 `__: BLD: update to OpenBLAS 0.3.13 +* `#18012 `__: SIMD: Optimize the performance of einsum's submodule sum. +* `#18014 `__: DOC: random: add some examples for SeedSequence +* `#18027 `__: DOC, MAINT: Minor fixes to refguide_check.py documentation. +* `#18030 `__: BUG: make a variable volatile to work around clang compiler bug +* `#18031 `__: DOC: Parameter name typo axes -> axis in numpy.fft._pocketfft. +* `#18032 `__: ENH: Add annotations for `np.core.arrayprint` +* `#18034 `__: DOC: Fix a couple of reference to verbatim and vice versa +* `#18042 `__: MAINT: Add dist_info to "other" setup.py commands. +* `#18045 `__: MAINT: Bump pytest from 6.2.0 to 6.2.1 +* `#18046 `__: TST: add back sdist test run +* `#18047 `__: BLD,DOC: pin sphinx to 3.3.1 +* `#18048 `__: DOC: Update TESTS.rst.txt +* `#18050 `__: MAINT: Add aliases for commonly used `ArrayLike` objects +* `#18051 `__: DEP: deprecate np.testing.dec +* `#18052 `__: BUG: Fix concatenation when the output is "S" or "U" +* `#18054 `__: DOC: Update stack docstrings +* `#18057 `__: BLD: ensure we give the right error message for old Python versions +* `#18062 `__: DOC: add missing details to linalg.lstsq docstring +* `#18065 `__: MAINT: CPUs that support unaligned access. +* `#18066 `__: TST: Allow mypy output types to be specified via aliases +* `#18067 `__: MAINT: Remove obsolete workaround to set ndarray.__hash__ = None +* `#18070 `__: BUG: Fix unique handling of nan entries. +* `#18072 `__: MAINT: crackfortran regex simplify +* `#18074 `__: MAINT: exprtype regex simplify +* `#18075 `__: ENH, SIMD: Dispatch for unsigned floor division +* `#18077 `__: NEP: mark NEP 28 on website redesign as final +* `#18078 `__: Fix build warnings in NEPs +* `#18079 `__: MAINT: Bump sphinx from 3.3.1 to 3.4.1 +* `#18080 `__: MAINT: Bump pytz from 2020.4 to 2020.5 +* `#18081 `__: MAINT: Bump hypothesis from 5.43.3 to 5.43.4 +* `#18082 `__: DOC: roadmap update +* `#18083 `__: MAINT: regex char class improve +* `#18084 `__: NEP: NumPy sponsorship guidelines (NEP 46) +* `#18085 `__: DOC: replace 'this platform' with the actual platform in the... +* `#18086 `__: BUG, SIMD: Fix _simd module build for 64bit Arm/neon clang +* `#18088 `__: DOC: Update reference to verbatim in a few location. +* `#18090 `__: MAINT: multiline regex class simplify +* `#18091 `__: DOC: Avoid using "set of" when talking about an ordered list. +* `#18097 `__: NEP: update backwards compatibility and deprecation policy NEP +* `#18100 `__: BUG, BLD: Generate the main dispatcher config header into the... +* `#18101 `__: ENH: move exp, log, frexp, ldexp to SIMD dispatching +* `#18103 `__: TST: Avoid changing odd tempfile names in tests' site.cfg +* `#18104 `__: TST: Turn some tests with loops into parametrized tests. +* `#18109 `__: MAINT: Fix exception cause in mingw32ccompiler.py +* `#18110 `__: API: make piecewise subclass safe using use zeros_like. +* `#18111 `__: MAINT: Bump hypothesis from 5.43.4 to 5.46.0 +* `#18115 `__: BUG: Fix promotion of half and string +* `#18116 `__: DEP: Deprecate promotion of numbers and bool to string +* `#18118 `__: BUG, MAINT: improve avx512 mask logical operations +* `#18126 `__: REL: Update master after 1.19.5 release. +* `#18128 `__: ENH: Add dtype support to the array comparison ops +* `#18136 `__: ENH: Adding keyboard interrupt support for array creation +* `#18144 `__: BLD: add found Cython version to check in cythonize.py +* `#18148 `__: MAINT: Bump sphinx from 3.4.1 to 3.4.3 +* `#18149 `__: MAINT: Bump hypothesis from 5.46.0 to 6.0.0 +* `#18150 `__: BUG: Ensure too many advanced indices raises an exception +* `#18152 `__: BUG: Promotion between strings and objects was assymetric +* `#18156 `__: MAINT: Remove redundant null check before free +* `#18157 `__: BUG: Initialize value of no_castable_output used in ufunc_loop_matches +* `#18161 `__: MAINT: Make keyword arrays static +* `#18164 `__: TST: add a pypy37 windows 64-bit build +* `#18166 `__: Use sinus based formula for ``chebpts1`` +* `#18169 `__: ENH: cpu features detection implementation on FreeBSD ARM +* `#18173 `__: TST: Clear the mypy cache before running any typing tests +* `#18174 `__: MAINT: Changed the `NBitBase` variancy in `number` from co- to... +* `#18176 `__: ENH: Improve performance of tril_indices and triu_indices +* `#18178 `__: SIMD: add fast integer division intrinsics for all supported... +* `#18180 `__: BUG: threads.h existence test requires GLIBC > 2.12. +* `#18181 `__: ENH: [f2py] Add external attribute support. +* `#18182 `__: MAINT: Bump hypothesis from 6.0.0 to 6.0.2 +* `#18183 `__: MAINT: Optimize numpy.count_nonzero for int types using SIMD... +* `#18184 `__: BUG: Fix f2py bugs when wrapping F90 subroutines. +* `#18185 `__: MAINT: Give the `_Like` and `_ArrayLike` type aliases a... +* `#18187 `__: STY: unify imports in __init__.py +* `#18191 `__: STY: Use explicit reexports for numpy.typing objects +* `#18193 `__: MAINT: Fix typo in docstring example +* `#18194 `__: MAINT: einsum: Optimize the sub function two-operands by using... +* `#18196 `__: BLD: update OpenBLAS to af2b0d02 +* `#18197 `__: BUG: Keep ignoring most errors during array-protocol lookup +* `#18200 `__: ENH: Add new intrinsics sum_u8/u16/u64. +* `#18204 `__: TST: Speed up the typing tests +* `#18205 `__: MAINT: Update pavement.py to work with versioneer. +* `#18208 `__: TST: raise memory limit for test +* `#18210 `__: DOC: typo in post-loop return +* `#18211 `__: MAINT: random shuffle: warn on unrecognized objects, fix empty... +* `#18213 `__: DOC: Formatting consistency. +* `#18214 `__: DOC: Double backticks for inline code example. +* `#18217 `__: MAINT: Ignore ComplexWarning in ``test_iter_copy_casts``. +* `#18221 `__: DOC: Misc single to double backticks fixes. +* `#18223 `__: DOC: Improve doc for numpy.random.Generator.choice +* `#18224 `__: MAINT: Bump pydata-sphinx-theme from 0.4.1 to 0.4.2 +* `#18225 `__: MAINT: Bump mypy from 0.790 to 0.800 +* `#18226 `__: MAINT: Bump hypothesis from 6.0.2 to 6.0.3 +* `#18227 `__: MAINT: Bump pytest-cov from 2.10.1 to 2.11.1 +* `#18228 `__: ENH: Add dtype-support to the ufunc-based `ndarray` magic methods... +* `#18229 `__: MAINT: Clean up all module-level dunders +* `#18230 `__: DOC: Clarify the type alias deprecation message +* `#18232 `__: DOC: lib/shape_base numpydoc formatting. +* `#18233 `__: NEP: accept NEP 23 (backwards compatibility policy) +* `#18234 `__: NEP: accept NEP 46 (sponsorship guidelines) +* `#18235 `__: DOC: Fix command in "Writing custom array containers" guide +* `#18236 `__: ENH: Add aliases for commonly used dtype-like objects +* `#18238 `__: DOC: __array__ accepts a dtype argument +* `#18245 `__: BLD: fix issue with `bdist_egg`, which made `make dist` in doc/... +* `#18247 `__: DOC: Misc numpydoc format fixes +* `#18248 `__: DOC: See also -> See Also (casing) +* `#18251 `__: DOC: more misc fixes of syntax +* `#18252 `__: DOC: cleanup of numpy/polynomial. +* `#18253 `__: DOC: improve description of `_NoValue` +* `#18255 `__: MAINT: add an 'apt update' +* `#18262 `__: REL: Update master after 1.20.0 release. +* `#18263 `__: ENH: Added sanity check to printoptions +* `#18264 `__: BUG: Use C linkage for random distributions +* `#18269 `__: DOC: Numpydoc format space before `:` in Parameters +* `#18272 `__: DOC: Numpydoc warning incorrect underline length. +* `#18274 `__: MAINT: Chain exceptions in linalg +* `#18275 `__: MAINT: Bump hypothesis from 6.0.3 to 6.1.1 +* `#18276 `__: MAINT: Bump pytest from 6.2.1 to 6.2.2 +* `#18277 `__: MAINT: Bump pydata-sphinx-theme from 0.4.2 to 0.4.3 +* `#18278 `__: MAINT: defer the import of shutil +* `#18282 `__: MAINT: gracefully shuffle memoryviews +* `#18284 `__: ENH: Add annotations for the remaining `np.generic` aliases +* `#18285 `__: TST: Pin `typing_extensions` to the latest version +* `#18289 `__: MAINT: Move transferdata into buffer-wise struct +* `#18293 `__: BUG: Fix typo in ``numpy.__init__.py`` +* `#18295 `__: BUG: don't mutate list of fake libraries while iterating over... +* `#18301 `__: MAINT: avoid chaining exceptions in conv_template.py +* `#18302 `__: MAINT: Add missing placeholder annotations +* `#18303 `__: MAINT: Fix typo in PyArray_RegisterDataType error +* `#18307 `__: DOC: Corrected numpy.power example. +* `#18313 `__: Numpy logo fix on README +* `#18315 `__: CI: rearrange Azure build jobs +* `#18317 `__: MAINT: Fixed chain exception for array_split func +* `#18320 `__: DOC: add links to polynomial function/class listing +* `#18322 `__: ENH: Add a mypy plugin for exposing platform-specific extended-precision... +* `#18323 `__: ENH: Add dtype-support to the ufunc-based `ndarray` magic methods... +* `#18324 `__: MAINT: Avoid moveaxis overhead in median. +* `#18329 `__: BUG: Allow unmodified use of isclose, allclose, etc. with timedelta +* `#18331 `__: MAINT: Update openblas_support for macosx-arm64 +* `#18332 `__: BUG: Allow pickling all relevant DType types/classes +* `#18333 `__: CI: fix when GitHub Actions builds trigger, and allow ci skips +* `#18334 `__: TST: use setup-python action for pypy, disable win64 pypy +* `#18338 `__: DOC: Fix whitespace before "last updated" on overview page +* `#18339 `__: DOC: Discussion on the @ operator and the matrix class +* `#18340 `__: DOC: remove pygments_style from conf.py +* `#18342 `__: DOC: Specified all possible return types for trapz function #18140 +* `#18344 `__: DOC: Added sentence to docstring of histogram_bin_edges to explain... +* `#18346 `__: DOC: Change license date 2020 -> 2021 +* `#18347 `__: MAINT: Delete unused "dst" clearing functions +* `#18348 `__: DEP: doc-deprecate BLAS_SRC/LAPACK_SRC +* `#18349 `__: CI: CircleCI seems to occasionally time out, increase the limit +* `#18350 `__: BUG: Fix missing signed_char dependency. +* `#18361 `__: ENH: Share memory of read-only intent(in) arrays. +* `#18362 `__: REL: Update master after 1.20.1 release. +* `#18364 `__: DOC: Update landing page to match table of contents +* `#18366 `__: MAINT: Disable TravisCI git clone depth. +* `#18367 `__: MAINT: Bump pytz from 2020.5 to 2021.1 +* `#18369 `__: BUG: np.in1d bug on the object array (issue 17923) +* `#18372 `__: DOC: improve standard_t example in numpy.random. +* `#18374 `__: TST: Add a test for nditer write masked with references +* `#18375 `__: BUG: fix regression in a hidden callback use case in f2py. +* `#18377 `__: ENH: Add annotations for `np.lib.ufunclike` +* `#18379 `__: DOC: Fix docstring of _median_nancheck. +* `#18384 `__: BUG: improve the interface of `tofile` method +* `#18389 `__: MAINT: Fix version of wheel to support Python 3.10 +* `#18390 `__: ENH: Add annotations for `np.core.einsumfunc` +* `#18392 `__: BUG: Remove check in shuffle for non-ndarrays +* `#18394 `__: MAINT: Added Chain exceptions where appropriate +* `#18395 `__: ENH: Initial typing of random +* `#18396 `__: MAINT: Threading and Unicode strings +* `#18397 `__: ENH: Add annotations for `np.lib.index_tricks` +* `#18398 `__: MAINT: Fix casting signatures to align with NEP 43 signature +* `#18400 `__: MAINT: Added Chain exceptions where appropriate +* `#18402 `__: BUG: Fix typo in char_codes +* `#18404 `__: BUG: Fix iterator shape in advanced index assignment broadcast... +* `#18405 `__: DOC: Mention `scipy.signal.correlate` and FFT method in `np.correlate`closes... +* `#18413 `__: MAINT: Bump sphinx from 3.4.3 to 3.5.0 +* `#18414 `__: MAINT: Bump hypothesis from 6.1.1 to 6.2.0 +* `#18415 `__: MAINT: Update END statements parsing for recent Fortran standards. +* `#18416 `__: BUG: Fix f2py parsing continued lines that follow comment lines. +* `#18417 `__: ENH: Add dtype-support to the ufunc-based `ndarray` magic methods... +* `#18418 `__: DOC: remove layout overrides for headers +* `#18420 `__: BUG: Fix tiny memory leaks when ``like=`` overrides are used +* `#18423 `__: ENH: Lint checks for PR diffs +* `#18428 `__: DOC: remove explanations.rst +* `#18429 `__: DOC: point intersphinx to matplotlib/stable... +* `#18432 `__: MAINT: Correct code producing warnings +* `#18433 `__: ENH: Add typing for RandomState +* `#18436 `__: BUG: Fix refcount leak in f2py `complex_double_from_pyobj` +* `#18437 `__: TST: Fix some uninitialized memory in the tests +* `#18438 `__: BUG: Correct shuffling of objects in 1-d array likes +* `#18439 `__: MAINT: random: Use 'from exc' when raising a ValueError in choice. +* `#18443 `__: BUG: fix stacklevel in warning within random.shuffle +* `#18448 `__: DOC: Remove unfinished Linear Algebra section from Quickstart... +* `#18450 `__: BUG: Segfault in nditer buffer dealloc for Object arrays +* `#18454 `__: NEP: add Spending NumPy Project Funds (NEP 48) +* `#18455 `__: BUG: ``diagflat`` could overflow on windows or 32-bit platforms +* `#18456 `__: NEP: array API standard adoption (NEP 47) +* `#18458 `__: DOC: update NEP status for accepted/finished NEPs +* `#18463 `__: MAINT: Bump mypy from 0.800 to 0.812 +* `#18464 `__: MAINT: Bump sphinx from 3.5.0 to 3.5.1 +* `#18465 `__: MAINT: Bump cython from 0.29.21 to 0.29.22 +* `#18466 `__: MAINT: Bump hypothesis from 6.2.0 to 6.3.0 +* `#18475 `__: ENH: Added type annotations to eye() function +* `#18476 `__: BUG: Remove suspicious type casting +* `#18477 `__: BUG: remove nonsensical comparison of pointer < 0 +* `#18478 `__: BUG: verify pointer against NULL before using it +* `#18479 `__: BUG: check if PyArray_malloc succeeded +* `#18481 `__: DOC: Generator and RandomState doc improvements +* `#18482 `__: ENH: Improve error message in multinomial +* `#18489 `__: DOC: Rename "Ones and zeros" section in array-creation documentation. +* `#18493 `__: BUG: Fix non-versioneer uses of numpy.distutils +* `#18497 `__: TST: Remove the `einsum` typing tests reliance on issuing a `ComplexWarning` +* `#18498 `__: BUG: Fixed Von Mises distribution for big values of kappa +* `#18499 `__: TST: Branch coverage improvement for `np.polynomial` +* `#18502 `__: DOC: Fix links to landing page +* `#18505 `__: DOC: add guide for downstream package authors +* `#18509 `__: DOC: trunc, floor, ceil, rint, fix should all link to each other +* `#18513 `__: BLD: add _2_24 to valid manylinux names +* `#18515 `__: MAINT: Improve error message when common type not found. +* `#18517 `__: MAINT: Bump hypothesis from 6.3.0 to 6.3.4 +* `#18518 `__: DOC Improve formatting in the depending_on_numpy documentation +* `#18522 `__: BUG: remove extraneous ARGOUTVIEWM dim. 4 typemaps +* `#18526 `__: MAINT: Specify color in RGB in the docs about the new NumPy logo +* `#18530 `__: BUG: incorrect error fallthrough in nditer +* `#18531 `__: CI: Use Ubuntu 18.04 to run "full" test. +* `#18537 `__: [BLD] use the new openblas lib +* `#18538 `__: Fix the numpy Apple M1 build +* `#18539 `__: BUG: NameError in numpy.distutils.fcompiler.compaq +* `#18544 `__: MAINT: Update master to main after branch rename +* `#18545 `__: ENH: Add annotations for `np.lib.arrayterator` +* `#18554 `__: CI: Pin docker image for Linux_Python_38_32bit_full_with_asserts... +* `#18560 `__: BUG: Fixed ``where`` keyword for ``np.mean`` & ``np.var`` methods +* `#18566 `__: CI: another master -> main fix +* `#18567 `__: CI: skip lint check on merges with main +* `#18569 `__: CI: Ensure that doc-build uses "main" as branch name +* `#18570 `__: CI: Use `git branch -m` instead of `--initial-branch=main` +* `#18571 `__: BUG: Fix overflow warning on apple silicon +* `#18572 `__: CI: Set git default branch to "main" in CircleCI. +* `#18574 `__: MAINT: Update the Call for Contributions section +* `#18575 `__: MAINT: Bump sphinx from 3.5.1 to 3.5.2 +* `#18576 `__: MAINT: Bump hypothesis from 6.3.4 to 6.6.0 +* `#18578 `__: MAINT: Bump pycodestyle from 2.5.0 to 2.6.0 +* `#18579 `__: MAINT: OrderedDict is no longer necessary from Python 3.7 +* `#18582 `__: BLD, TST: use pypy nightly to work around bug +* `#18583 `__: DOC: Clarify docs for fliplr() / flipud() +* `#18584 `__: DOC: Added documentation for linter (#18423) +* `#18593 `__: MAINT: Do not claim input to binops is `self` (array object) +* `#18594 `__: MAINT: Remove strange `op == NULL` check +* `#18596 `__: MAINT: Chain exceptions in index_tricks.py and mrecords.py +* `#18598 `__: MAINT: Add annotations for `dtype.__getitem__`, `__mul__` and... +* `#18602 `__: CI: Do not fail CI on lint error +* `#18605 `__: BUG: Fix ma coercion list-of-ma-arrays if they do not cast to... +* `#18614 `__: MAINT: Bump pycodestyle from 2.6.0 to 2.7.0 +* `#18615 `__: MAINT: Bump hypothesis from 6.6.0 to 6.8.1 +* `#18616 `__: CI: Update apt package list before Python install +* `#18618 `__: MAINT: Ensure that re-exported sub-modules are properly annotated +* `#18622 `__: DOC: Consistently use rng as variable name for random generators +* `#18629 `__: BUG, ENH: fix array2string rounding bug by adding min_digits... +* `#18630 `__: DOC: add note to numpy.rint() docstrings +* `#18634 `__: BUG: Use npy_log1p where appropriate in random generation +* `#18635 `__: ENH: Improve the exception for default low in Generator.integers +* `#18641 `__: MAINT: Remove useless declarations in `bad_commands` +* `#18642 `__: ENH: Use new argument parsing for array creation functions +* `#18643 `__: DOC: Remove mention of nose from README +* `#18645 `__: DOC: Minor fix in inline code example of ufunc reference +* `#18648 `__: MAINT: use super() as described by PEP 3135 +* `#18649 `__: MAINT: Add missing type to cdef statement +* `#18651 `__: BUG: Fix small valgrind-found issues +* `#18652 `__: DOC: Update some plotting code to current Matplotlib idioms +* `#18657 `__: ENH: Improve performance of `np.save` for small arrays +* `#18658 `__: BLD: remove /usr/include from default include dirs +* `#18659 `__: DEV: add a conda environment.yml with all development dependencies +* `#18660 `__: DOC: add release note for removal of /usr/include from include... +* `#18664 `__: MAINT: Bump sphinx from 3.5.2 to 3.5.3 +* `#18666 `__: ENH: Use exponentials in place of inversion in Rayleigh and geometric +* `#18670 `__: BUG: Fix small issues found with pytest-leaks +* `#18676 `__: MAINT: Implement new style promotion for `np.result_type`, etc. +* `#18679 `__: BUG: Changed METH_VARARGS to METH_NOARGS +* `#18680 `__: Docs: simd-optimizations.rst: fix typo (basline ~> baseline) +* `#18685 `__: REL: Update main after 1.20.2 release. +* `#18686 `__: BUG: Fix test_ccompiler_opt when path contains dots +* `#18689 `__: DOC: Change matrix size in absolute beginners doc. +* `#18690 `__: BUG: Correct datetime64 missing type overload for datetime.date... +* `#18691 `__: BUG: fix segfault in object/longdouble operations +* `#18692 `__: MAINT: Bump pydata-sphinx-theme from 0.5.0 to 0.5.2 +* `#18693 `__: MAINT: Bump hypothesis from 6.8.1 to 6.8.3 +* `#18694 `__: TST: pin pypy version to 7.3.4rc1 +* `#18695 `__: ENH: Support parsing Fortran abstract interface blocks. +* `#18697 `__: DEP: Disable PyUFunc_GenericFunction and PyUFunc_SetUsesArraysAsData +* `#18698 `__: MAINT: Specify the color space in all new NumPy logo files +* `#18701 `__: BLD: Strip extra newline when dumping gfortran version on MacOS +* `#18705 `__: DOC: update Steering Council membership and people on governance... +* `#18706 `__: DOC: Add release notes to upcoming_changes +* `#18708 `__: TST: add tests for using np.meshgrid for higher dimensional grids. +* `#18712 `__: DOC: Simplifies Mandelbrot set plot in Quickstart guide +* `#18718 `__: API, DEP: Move ufunc signature parsing to the start +* `#18722 `__: DOC: deduplicate dtype basic types (2) +* `#18725 `__: MAINT: Bump pytest from 6.2.2 to 6.2.3 +* `#18726 `__: MAINT: Bump hypothesis from 6.8.3 to 6.8.4 +* `#18728 `__: MAINT: Add exception chaining where appropriate +* `#18731 `__: BUG: Check out requirements and raise when not satisfied +* `#18733 `__: DEV: Adds gitpod to numpy +* `#18737 `__: BLD: introduce use of BLAS_LIBS and LAPACK_LIBS in distutils/system_info +* `#18739 `__: MAINT: Add exception chaining where appropriate +* `#18741 `__: DOC: Emphasize distinctions between np.copy and ndarray.copy +* `#18745 `__: CI: remove shippable CI +* `#18750 `__: MAINT: Allow more recursion depth for scalar tests. +* `#18751 `__: BUG: Regression #18075 | Fixing Ufunc TD generation order +* `#18753 `__: BLD: Negative zero handling with ifort +* `#18755 `__: MAINT: Bump sphinx from 3.5.3 to 3.5.4 +* `#18757 `__: MAINT: Bump hypothesis from 6.8.4 to 6.9.1 +* `#18758 `__: DOC: Update howto-docs with link to NumPy tutorials. +* `#18761 `__: DOC: Small fixes (including formatting) for NEP 43 +* `#18765 `__: ENH: Improve the placeholder annotations for the main numpy namespace +* `#18766 `__: ENH, SIMD: Replace libdivide functions of signed integer division... +* `#18770 `__: DOC: More concise "How to import NumPy" description +* `#18771 `__: DOC: Use: from numpy.testing import ... +* `#18772 `__: CI: Use informational mode for codecov +* `#18773 `__: CI: Fixing typo in Azure job run +* `#18777 `__: DOC: update random and asserts in test guidelines +* `#18778 `__: MAINT: Relax the integer-type-constraint of `npt._ShapeLike` +* `#18779 `__: DOC: fix spelling of "reccomended" ("recommended") +* `#18780 `__: ENH: Improve the placeholder annotations for the main numpy namespace... +* `#18781 `__: ENH: Add `__all__` to a number of public modules +* `#18785 `__: DOC: change `dec.parametrize` to `pytest.mark.parametrize` +* `#18786 `__: DOC: add note for clip() special case a_min > a_max See #18782 +* `#18787 `__: DOC: Document newer pytest conventions +* `#18789 `__: DEV: Pin pydata-sphinx-theme to 0.5.2. +* `#18790 `__: CI: Use `towncrier build` explicitly +* `#18791 `__: DOC: Fixes small things in the genfromtext docstring +* `#18792 `__: MAINT: Use recent towncrier releases on PyPI. +* `#18795 `__: SIMD, TEST: Workaround for misaligned stack GCC BUG ABI on WIN64 +* `#18796 `__: DOC: Misc Numpydoc and formatting for proper parsing. +* `#18797 `__: DOC: Update random c-api documentation +* `#18799 `__: MAINT: Improve the placeholder annotations for the main numpy... +* `#18800 `__: MAINT: Relax miscellaneous integer-type constraints +* `#18801 `__: DOC: fix typo in frexp docstring +* `#18802 `__: DOC: Improve random.choice() documentation +* `#18805 `__: NEP: propose new nep for allocator policies +* `#18806 `__: MAINT: Bump hypothesis from 6.9.1 to 6.10.0 +* `#18807 `__: MAINT: Bump cython from 0.29.22 to 0.29.23 +* `#18809 `__: MAINT: runtests help text cleanup +* `#18812 `__: DOC: Document howto build documentation in a virtual environment +* `#18813 `__: BUG: Initialize the full nditer buffer in case of error +* `#18818 `__: ENH: Add annotations for 4 objects in `np.core.numerictypes` +* `#18820 `__: MAINT: Remove incorrect inline +* `#18822 `__: DEV: general Gitpod enhancements +* `#18823 `__: MAINT: Minor fix to add reference link to numpy.fill_diagonal... +* `#18825 `__: MAINT: Update README.md +* `#18831 `__: BUG: Prevent nan being used in percentile +* `#18834 `__: DOC: Fix typo in random docs +* `#18836 `__: MAINT: Generalize and shorten the ufunc "trivially iterable"... +* `#18837 `__: ENH, SIMD: Add support for dispatching C++ sources +* `#18839 `__: DOC: Add Gitpod development documentation +* `#18841 `__: DOC: Add favicon +* `#18842 `__: ENH: Improve the placeholder annotations within sub-modules +* `#18843 `__: DOC: Clarify isreal docstring +* `#18845 `__: DOC: Move Sphinx numpy target in reference index. +* `#18851 `__: MAINT: Disable pip version check for azure lint check. +* `#18853 `__: ENH: Improve the placeholder annotations within sub-modules (part... +* `#18855 `__: STY: change CRLF line terminators to Unix +* `#18856 `__: MAINT: Fix the typo "implment" +* `#18862 `__: TST: Skip f2py TestSharedMemory for LONGDOUBLE on macos/arm64 +* `#18863 `__: ENH: Add max values comparison for floating point +* `#18864 `__: MAINT: Remove dead codepath in generalized ufuncs +* `#18868 `__: Upgrade to GitHub-native Dependabot +* `#18869 `__: MAINT: Fix azure linter problems with pip 21.1 +* `#18871 `__: MAINT: Bump hypothesis from 6.10.0 to 6.10.1 +* `#18874 `__: BLD, ENH: Enable Accelerate Framework +* `#18877 `__: MAINT: Update PyPy version used by CI +* `#18880 `__: API: Ensure that casting does not affect ufunc loop +* `#18882 `__: ENH: Add min values comparison for floating point +* `#18885 `__: MAINT: Remove unsafe unions and ABCs from return-annotations +* `#18889 `__: ENH: Add SIMD operations for min and max value comparision +* `#18890 `__: MAINT: ssize_t -> Py_ssize_t and other fixes for Python v3.10.0 +* `#18891 `__: MAINT: Bump typing-extensions from 3.7.4.3 to 3.10.0.0 +* `#18893 `__: DOC: Add a set of standard replies. +* `#18895 `__: DOC: Improve cumsum documentation +* `#18896 `__: MAINT: Explicitly mark text files in .gitattributes. +* `#18897 `__: MAINT: Add ".csv" some data file names. +* `#18899 `__: BLD, BUG: Fix compiler optimization log AttributeError +* `#18900 `__: BLD: remove unnecessary flag `-faltivec` on macOS +* `#18903 `__: MAINT, CI: treats _SIMD module build warnings as errors through... +* `#18906 `__: ENH: Add PCG64DXSM BitGenerator +* `#18908 `__: MAINT: Adjust NumPy float hashing to Python's slightly changed... +* `#18909 `__: ENH: Improve the placeholder annotations within sub-modules (part... +* `#18910 `__: BUG : for MINGW, threads.h existence test requires GLIBC > 2.12 +* `#18911 `__: BLD, BUG: Fix bdist_wheel duplicate building +* `#18912 `__: CI: fix the GitHub Actions trigger in docker.yml +* `#18918 `__: DOC: fix documentation of cloning over ssh +* `#18919 `__: ENH: Add placeholder annotations for two missing `np.testing`... +* `#18920 `__: BUG: Report underflow condition in AVX implementation of np.exp +* `#18927 `__: NEP: add mailing list thread, fixes from review +* `#18930 `__: BUG: Make changelog recognize ``gh-`` as a PR number prefix. +* `#18931 `__: BUG: Fix refcounting in string-promotion deprecation code path +* `#18933 `__: BUG: Fix underflow error in AVX512 implementation of ufunc exp/f64 +* `#18934 `__: DOC: Add a release note for the improved placeholder annotations +* `#18935 `__: API: Add `npt.NDArray`, a runtime-subscriptable alias for `np.ndarray` +* `#18936 `__: DOC: Update performance for new PRNG +* `#18940 `__: ENH: manually inline PCG64DXSM code for performance. +* `#18943 `__: TST: xfail `TestCond.test_nan` unconditionally +* `#18944 `__: ENH: Add annotations for `np.lib.utils` +* `#18954 `__: DOC: Update beginners docu for sum function with axis +* `#18955 `__: DOC: add an extra example in runtests.py help test +* `#18956 `__: DOC: change copyright SciPy to NumPy +* `#18957 `__: DOC: Improve datetime64 docs. +* `#18958 `__: MAINT: Do not use deprecated ``mktemp()`` +* `#18959 `__: DOC: improve numpy.histogram2d() documentation +* `#18960 `__: BUG: fixed ma.average ignoring masked weights +* `#18961 `__: DOC: add note and examples to `isrealobj` docstring +* `#18962 `__: DOC: Update a page title with proper case +* `#18963 `__: DEP: remove PolyBase from np.polynomial.polyutils +* `#18965 `__: DOC: Improve description of array scalar in glossary +* `#18967 `__: BUG: fix np.ma.masked_where(copy=False) when input has no mask +* `#18970 `__: MAINT, SIMD: Hardened the AVX compile-time tests +* `#18972 `__: ENH: Include co-authors in changelog. +* `#18973 `__: MAINT: Bump sphinx from 3.5.4 to 4.0.0 +* `#18974 `__: MAINT: Bump hypothesis from 6.10.1 to 6.12.0 +* `#18976 `__: MAINT: Bump pytest from 6.2.3 to 6.2.4 +* `#18980 `__: DOC: Gitpod documentation enhancements +* `#18982 `__: MAINT: Cleanup tools/changelog.py +* `#18983 `__: REL: Update main after 1.20.3 release. +* `#18985 `__: MAINT: Remove usage of the PEP 604 pipe operator +* `#18987 `__: BUG: Update coordinates in PyArray_ITER_GOTO1D +* `#18989 `__: BUG: fix potential buffer overflow(#18939) +* `#18990 `__: ENH: Add annotations for `np.lib.NumpyVersion` +* `#18996 `__: MAINT: Remove warning when checking AVX512f on MSVC +* `#18998 `__: ENH: Improve annotations of the `item`, `tolist`, `take` and... +* `#18999 `__: DEP: Ensure the string promotion FutureWarning is raised +* `#19001 `__: DEP: Deprecate error clearing for special method in array-coercion +* `#19002 `__: ENH: Add annotations for `np.broadcast` and `np.DataSource` +* `#19005 `__: ENH: Add dtype-support to 11 `ndarray` / `generic` methods +* `#19007 `__: BUG: fix potential use of null pointer in nditer buffers +* `#19008 `__: BUG: fix variable misprint in multiarray test code +* `#19009 `__: BUG: fix variable misprint checking wrong variable in umath tests +* `#19011 `__: BUG: fix ValueError in PyArray_Std on win_amd64 +* `#19012 `__: MAINT: Small cleanups in `PyArray_NewFromDescr_int` +* `#19014 `__: Revert "BUG: Update coordinates in PyArray_ITER_GOTO1D" +* `#19018 `__: DOC: "NumPy" <- "numpy" in NumPy Fundamentals - Indexing +* `#19021 `__: DOC: Add comment for ifdef macro guard +* `#19024 `__: MAINT: Bump pytest-cov from 2.11.1 to 2.12.0 +* `#19025 `__: MAINT: Bump sphinx from 4.0.0 to 4.0.1 +* `#19026 `__: DOC: Clarify minimum numpy version needed to use random c-api +* `#19029 `__: ENH: Improve the annotations of `np.core._internal` +* `#19031 `__: DEP: Deprecate 4 `ndarray.ctypes` methods +* `#19035 `__: MAINT: Python3 classes do not need to inherit from object +* `#19037 `__: BUG: do not use PyLong_FromLong for intp +* `#19041 `__: DOC: Improve trapz docstring +* `#19043 `__: DOC: Fix typo in release notes for v1.21 +* `#19046 `__: BUG, SIMD: Fix unexpected result of uint8 division on X86 +* `#19047 `__: BUG, SIMD: Fix NumPy build on ppc64le(IBM/Power) for old versions... +* `#19048 `__: BUG: Fix duplicate variable names in compiler check for AVX512_SKX +* `#19049 `__: BLD,API: (distutils) Force strict floating point error model... +* `#19052 `__: ENH: Improve the `np.ufunc` annotations +* `#19055 `__: DOC: Forward port missing 1.18.5 release note. +* `#19063 `__: ENH: Stubs for array_equal appear out of date. +* `#19066 `__: BUG: Fixed an issue wherein `nanmedian` could return an array... +* `#19068 `__: MAINT: Update mailmap +* `#19073 `__: REL: Prepare 1.21.0 release +* `#19074 `__: BUG: Fix compile-time test of POPCNT +* `#19075 `__: BUG: Fix test_numpy_version. +* `#19094 `__: BUG: Fixed an issue wherein `_GenericAlias.__getitem__` would... +* `#19100 `__: BUG: Linter should only run on pull requests. +* `#19120 `__: BUG: Fix setup.py to work in maintenance branches. +* `#19144 `__: BUG: expose short_version as previously in version.py +* `#19175 `__: API: Delay string and number promotion deprecation/future warning +* `#19178 `__: BUG, SIMD: Fix detect host/native CPU features on ICC at compile-time +* `#19180 `__: BUG: Add -std=c99 to intel icc compiler flags on linux +* `#19193 `__: NEP: Accept NEP 35 as final +* `#19194 `__: MAINT, BUG: Adapt `castingimpl.casting` to denote a minimal level +* `#19197 `__: REL: Prepare for NumPy 1.20.0rc2 release. +* `#19213 `__: MAINT: Add annotations for the missing `period` parameter to... +* `#19219 `__: MAINT: Add `complex` as allowed type for the `np.complexfloating`... +* `#19233 `__: TST: Ignore exp FP exceptions test for glibc ver < 2.17 +* `#19238 `__: MAINT: replace imgmath with mathjax for docs +* `#19239 `__: BUG: Fix out-of-bounds access in convert_datetime_divisor_to_multiple +* `#19240 `__: ENH: Support major version larger than 9 in NumpyVersion +* `#19268 `__: DOC: fix duplicate navbar in development documentation index +* `#19269 `__: BUG: Invalid dtypes comparison should not raise TypeError +* `#19280 `__: BUG: Add missing DECREF in new path +* `#19283 `__: REL: Prepare for 1.21.0 release diff --git a/doc/neps/_static/nep-0047-casting-rules-lattice.png b/doc/neps/_static/nep-0047-casting-rules-lattice.png new file mode 100644 index 000000000000..669d3047683d Binary files /dev/null and b/doc/neps/_static/nep-0047-casting-rules-lattice.png differ diff --git a/doc/neps/_static/nep-0047-library-dependencies.png b/doc/neps/_static/nep-0047-library-dependencies.png new file mode 100644 index 000000000000..4eab600a5382 Binary files /dev/null and b/doc/neps/_static/nep-0047-library-dependencies.png differ diff --git a/doc/neps/_static/nep-0047-scope-of-array-API.png b/doc/neps/_static/nep-0047-scope-of-array-API.png new file mode 100644 index 000000000000..55253288c3a2 Binary files /dev/null and b/doc/neps/_static/nep-0047-scope-of-array-API.png differ diff --git a/doc/neps/nep-0000.rst b/doc/neps/nep-0000.rst index 590976081723..7f841b7e28fb 100644 --- a/doc/neps/nep-0000.rst +++ b/doc/neps/nep-0000.rst @@ -278,7 +278,7 @@ References and Footnotes .. [1] This historical record is available by the normal git commands for retrieving older revisions, and can also be browsed on - `GitHub `_. + `GitHub `_. .. [2] The URL for viewing NEPs on the web is https://www.numpy.org/neps/. diff --git a/doc/neps/nep-0001-npy-format.rst b/doc/neps/nep-0001-npy-format.rst index 3a28247ab9ed..fdf4ae47ae26 100644 --- a/doc/neps/nep-0001-npy-format.rst +++ b/doc/neps/nep-0001-npy-format.rst @@ -292,7 +292,7 @@ included in the 1.9.0 release of numpy. Specifically, the file format.py in this directory implements the format as described here. - https://github.com/numpy/numpy/blob/master/numpy/lib/format.py + https://github.com/numpy/numpy/blob/main/numpy/lib/format.py References diff --git a/doc/neps/nep-0010-new-iterator-ufunc.rst b/doc/neps/nep-0010-new-iterator-ufunc.rst index 358018c46417..4e7fdfdf5769 100644 --- a/doc/neps/nep-0010-new-iterator-ufunc.rst +++ b/doc/neps/nep-0010-new-iterator-ufunc.rst @@ -575,12 +575,16 @@ ndim, and niter will produce slightly different layouts. intp shape; /* The current coordinate along this axis */ intp coord; - /* The operand and index strides for this axis + /* The operand and index strides for this axis */ intp stride[niter]; - {intp indexstride;} #if (flags&FLAGS_HASINDEX); + #if (flags&FLAGS_HASINDEX) + intp indexstride; + #endif /* The operand pointers and index values for this axis */ char* ptr[niter]; - {intp index;} #if (flags&FLAGS_HASINDEX); + #if (flags&FLAGS_HASINDEX) + intp index; + #endif }[ndim]; }; diff --git a/doc/neps/nep-0013-ufunc-overrides.rst b/doc/neps/nep-0013-ufunc-overrides.rst index 698e45738708..ceb8b23e9587 100644 --- a/doc/neps/nep-0013-ufunc-overrides.rst +++ b/doc/neps/nep-0013-ufunc-overrides.rst @@ -46,8 +46,8 @@ right behaviour, hence the change in name.) The ``__array_ufunc__`` as described below requires that any corresponding Python binary operations (``__mul__`` et al.) should be -implemented in a specific way and be compatible with Numpy's ndarray -semantics. Objects that do not satisfy this cannot override any Numpy +implemented in a specific way and be compatible with NumPy's ndarray +semantics. Objects that do not satisfy this cannot override any NumPy ufuncs. We do not specify a future-compatible path by which this requirement can be relaxed --- any changes here require corresponding changes in 3rd party code. @@ -132,7 +132,7 @@ However, this behavior is more confusing than useful, and having a :exc:`TypeError` would be preferable. This proposal will *not* resolve the issue with scipy.sparse matrices, -which have multiplication semantics incompatible with numpy arrays. +which have multiplication semantics incompatible with NumPy arrays. However, the aim is to enable writing other custom array types that have strictly ndarray compatible semantics. @@ -246,7 +246,7 @@ three groups: - *Incompatible*: neither above nor below A; types for which no (indirect) upcasting is possible. -Note that the legacy behaviour of numpy ufuncs is to try to convert +Note that the legacy behaviour of NumPy ufuncs is to try to convert unknown objects to :class:`ndarray` via :func:`np.asarray`. This is equivalent to placing :class:`ndarray` above these objects in the graph. Since we above defined :class:`ndarray` to return `NotImplemented` for @@ -454,7 +454,7 @@ implements the following behavior: A class wishing to modify the interaction with :class:`ndarray` in binary operations therefore has two options: -1. Implement ``__array_ufunc__`` and follow Numpy semantics for Python +1. Implement ``__array_ufunc__`` and follow NumPy semantics for Python binary operations (see below). 2. Set ``__array_ufunc__ = None``, and implement Python binary @@ -478,7 +478,7 @@ are not compatible, i.e., implementations should be something like:: except AttributeError: return False - class ArrayLike(object): + class ArrayLike: ... def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ... @@ -516,7 +516,7 @@ does not know how to deal with arrays and ufuncs, and thus has set ``__array_ufunc__`` to :obj:`None`, but does know how to do multiplication:: - class MyObject(object): + class MyObject: __array_ufunc__ = None def __init__(self, value): self.value = value @@ -678,7 +678,7 @@ NA ``abs`` :func:`absolute` Future extensions to other functions ------------------------------------ -Some numpy functions could be implemented as (generalized) Ufunc, in +Some NumPy functions could be implemented as (generalized) Ufunc, in which case it would be possible for them to be overridden by the ``__array_ufunc__`` method. A prime candidate is :func:`~numpy.matmul`, which currently is not a Ufunc, but could be relatively easily be diff --git a/doc/neps/nep-0014-dropping-python2.7-proposal.rst b/doc/neps/nep-0014-dropping-python2.7-proposal.rst index dfc09d2543c2..e14a173e2032 100644 --- a/doc/neps/nep-0014-dropping-python2.7-proposal.rst +++ b/doc/neps/nep-0014-dropping-python2.7-proposal.rst @@ -4,7 +4,7 @@ NEP 14 — Plan for dropping Python 2.7 support ============================================= -:Status: Accepted +:Status: Final :Resolution: https://mail.python.org/pipermail/numpy-discussion/2017-November/077419.html The Python core team plans to stop supporting Python 2 in 2020. The NumPy diff --git a/doc/neps/nep-0016-abstract-array.rst b/doc/neps/nep-0016-abstract-array.rst index 63ad600e9b3e..9d21abe6f4bb 100644 --- a/doc/neps/nep-0016-abstract-array.rst +++ b/doc/neps/nep-0016-abstract-array.rst @@ -141,7 +141,7 @@ Notes: In practice, either way we'd only do the full test after first checking for well-known types like ``ndarray``, ``list``, etc. `This is how NumPy currently checks for other double-underscore attributes -`__ +`__ and the same idea applies here to either approach. So these numbers won't affect the common case, just the case where we actually have an ``AbstractArray``, or else another third-party object that will end up diff --git a/doc/neps/nep-0023-backwards-compatibility.rst b/doc/neps/nep-0023-backwards-compatibility.rst index 158f46273f00..8b6f4cd1186a 100644 --- a/doc/neps/nep-0023-backwards-compatibility.rst +++ b/doc/neps/nep-0023-backwards-compatibility.rst @@ -5,10 +5,11 @@ NEP 23 — Backwards compatibility and deprecation policy ======================================================= :Author: Ralf Gommers -:Status: Draft +:Status: Final :Type: Process :Created: 2018-07-14 -:Resolution: (required for Accepted | Rejected | Withdrawn) +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2021-January/081423.html + Abstract -------- @@ -19,46 +20,222 @@ processes for individual cases where breaking backwards compatibility is considered. -Detailed description +Motivation and Scope -------------------- NumPy has a very large user base. Those users rely on NumPy being stable and the code they write that uses NumPy functionality to keep working. NumPy is also actively maintained and improved -- and sometimes improvements -require, or are made much easier, by breaking backwards compatibility. +require, or are made easier, by breaking backwards compatibility. Finally, there are trade-offs in stability for existing users vs. avoiding errors or having a better user experience for new users. These competing -needs often give rise to heated debates and delays in accepting or rejecting +needs often give rise to long debates and delay accepting or rejecting contributions. This NEP tries to address that by providing a policy as well as examples and rationales for when it is or isn't a good idea to break backwards compatibility. -General principles: - -- Aim not to break users' code unnecessarily. -- Aim never to change code in ways that can result in users silently getting - incorrect results from their previously working code. -- Backwards incompatible changes can be made, provided the benefits outweigh - the costs. -- When assessing the costs, keep in mind that most users do not read the mailing - list, do not look at deprecation warnings, and sometimes wait more than one or - two years before upgrading from their old version. And that NumPy has - many hundreds of thousands or even a couple of million users, so "no one will - do or use this" is very likely incorrect. -- Benefits include improved functionality, usability and performance (in order - of importance), as well as lower maintenance cost and improved future - extensibility. -- Bug fixes are exempt from the backwards compatibility policy. However in case - of serious impact on users (e.g. a downstream library doesn't build anymore), - even bug fixes may have to be delayed for one or more releases. -- The Python API and the C API will be treated in the same way. - - -Examples -^^^^^^^^ - -We now discuss a number of concrete examples to illustrate typical issues -and trade-offs. +In addition, this NEP can serve as documentation for users about how the NumPy +project treats backwards compatibility, and the speed at which they can expect +changes to be made. + +In scope for this NEP are: + +- Principles of NumPy's approach to backwards compatibility. +- How to deprecate functionality, and when to remove already deprecated + functionality. +- Decision making process for deprecations and removals. +- How to ensure that users are well informed about any change. + +Out of scope are: + +- Making concrete decisions about deprecations of particular functionality. +- NumPy's versioning scheme. + + +General principles +------------------ + +When considering proposed changes that are backwards incompatible, the +main principles the NumPy developers use when making a decision are: + +1. Changes need to benefit more than they harm users. +2. NumPy is widely used, so breaking changes should be assumed by default to be + harmful. +3. Decisions should be based on how they affect users and downstream packages + and should be based on usage data where possible. It does not matter whether + this use contradicts the documentation or best practices. +4. The possibility of an incorrect result is worse than an error or even crash. + +When assessing the costs of proposed changes, keep in mind that most users do +not read the mailing list, do not notice deprecation warnings, and sometimes +wait more than one or two years before upgrading from their old version. And +that NumPy has millions of users, so "no one will do or use this" is likely +incorrect. + +Benefits of proposed changes can include improved functionality, usability and +performance, as well as lower maintenance cost and improved future +extensibility. + +Fixes for clear bugs are exempt from this backwards compatibility policy. +However, in case of serious impact on users even bug fixes may have to be +delayed for one or more releases. For example, if a downstream library would no +longer build or would give incorrect results. + + +Strategies related to deprecations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Impact assessment +````````````````` + +Getting hard data on the impact of a deprecation of often difficult. Strategies +that can be used to assess such impact include: + +- Use a code search engine ([1]_, [2]_) or static ([3]_) or dynamic ([4]_) code + analysis tools to determine where and how the functionality is used. +- Test prominent downstream libraries against a development build of NumPy + containing the proposed change to get real-world data on its impact. +- Make a change on the main branch and revert it before release if it + causes problems. We encourage other packages to test against + NumPy's main branch and if that's too burdensome, then at least to + test pre-releases. This often turns up issues quickly. + +Alternatives to deprecations +```````````````````````````` + +If the impact is unclear or significant, it is often good to consider +alternatives to deprecations. For example, discouraging use in documentation +only, or moving the documentation for the functionality to a less prominent +place or even removing it completely. Commenting on open issues related to it +that they are low-prio or labeling them as "wontfix" will also be a signal to +users, and reduce the maintenance effort needing to be spent. + + +Implementing deprecations and removals +-------------------------------------- + +Deprecation warnings are necessary in all cases where functionality +will eventually be removed. If there is no intent to remove functionality, +then it should not be deprecated. A "please don't use this for new code" +in the documentation or other type of warning should be used instead, and the +documentation can be organized such that the preferred alternative is more +prominently shown. + +Deprecations: + +- shall include the version number of the release in which the functionality + was deprecated. +- shall include information on alternatives to the deprecated functionality, or a + reason for the deprecation if no clear alternative is available. Note that + release notes can include longer messages if needed. +- shall use ``DeprecationWarning`` by default, and ``VisibleDeprecation`` + for changes that need attention again after already having been deprecated or + needing extra attention for some reason. +- shall be listed in the release notes of the release where the deprecation is + first present. +- shall not be introduced in micro (bug fix) releases. +- shall set a ``stacklevel``, so the warning appears to come from the correct + place. +- shall be mentioned in the documentation for the functionality. A + ``.. deprecated::`` directive can be used for this. + +Examples of good deprecation warnings (also note standard form of the comments +above the warning, helps when grepping): + +.. code-block:: python + + # NumPy 1.15.0, 2018-09-02 + warnings.warn('np.asscalar(a) is deprecated since NumPy 1.16.0, use ' + 'a.item() instead', DeprecationWarning, stacklevel=3) + + # NumPy 1.15.0, 2018-02-10 + warnings.warn("Importing from numpy.testing.utils is deprecated " + "since 1.15.0, import from numpy.testing instead.", + DeprecationWarning, stacklevel=2) + + # NumPy 1.14.0, 2017-07-14 + warnings.warn( + "Reading unicode strings without specifying the encoding " + "argument is deprecated since NumPy 1.14.0. Set the encoding, " + "use None for the system default.", + np.VisibleDeprecationWarning, stacklevel=2) + +.. code-block:: C + + /* DEPRECATED 2020-05-13, NumPy 1.20 */ + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + matrix_deprecation_msg, ufunc->name, "first") < 0) { + return NULL; + } + +Removal of deprecated functionality: + +- shall be done after at least 2 releases assuming the current 6-monthly + release cycle; if that changes, there shall be at least 1 year between + deprecation and removal. +- shall be listed in the release notes of the release where the removal happened. +- can be done in any minor, but not bugfix, release. + +For backwards incompatible changes that aren't "deprecate and remove" but for +which code will start behaving differently, a ``FutureWarning`` should be +used. Release notes, mentioning version number and using ``stacklevel`` should +be done in the same way as for deprecation warnings. A ``.. versionchanged::`` +directive shall be used in the documentation after the behaviour change was +made to indicate when the behavior changed: + +.. code-block:: python + + def argsort(self, axis=np._NoValue, ...): + """ + Parameters + ---------- + axis : int, optional + Axis along which to sort. If None, the default, the flattened array + is used. + + .. versionchanged:: 1.13.0 + Previously, the default was documented to be -1, but that was + in error. At some future date, the default will change to -1, as + originally intended. + Until then, the axis should be given explicitly when + ``arr.ndim > 1``, to avoid a FutureWarning. + """ + ... + warnings.warn( + "In the future the default for argsort will be axis=-1, not the " + "current None, to match its documentation and np.argsort. " + "Explicitly pass -1 or None to silence this warning.", + MaskedArrayFutureWarning, stacklevel=3) + + +Decision making +--------------- + +In concrete cases where this policy needs to be applied, decisions are made according +to the `NumPy governance model +`_. + +All deprecations must be proposed on the mailing list in order to give everyone +with an interest in NumPy development a chance to comment. Removal of +deprecated functionality does not need discussion on the mailing list. + + +Functionality with more strict deprecation policies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- ``numpy.random`` has its own backwards compatibility policy with additional + requirements on top of the ones in this NEP, see + `NEP 19 `_. +- The file format of ``.npy`` and ``.npz`` files is strictly versioned + independent of the NumPy version; existing format versions must remain + backwards compatible even if a newer format version is introduced. + + +Example cases +------------- + +We now discuss a few concrete examples from NumPy's history to illustrate +typical issues and trade-offs. **Changing the behavior of a function** @@ -89,21 +266,6 @@ forces users to change their code more than once, which is almost never the right thing to do. Instead, a better approach here would have been to deprecate ``histogram`` and introduce a new function ``hist`` in its place. -**Returning a view rather than a copy** - -The ``ndarray.diag`` method used to return a copy. A view would be better for -both performance and design consistency. This change was warned about -(``FutureWarning``) in v.8.0, and in v1.9.0 ``diag`` was changed to return -a *read-only* view. The planned change to a writeable view in v1.10.0 was -postponed due to backwards compatibility concerns, and is still an open issue -(gh-7661). - -What should have happened instead: nothing. This change resulted in a lot of -discussions and wasted effort, did not achieve its final goal, and was not that -important in the first place. Finishing the change to a *writeable* view in -the future is not desired, because it will result in users silently getting -different results if they upgraded multiple versions or simply missed the -warnings. **Disallowing indexing with floats** @@ -117,131 +279,33 @@ introduced for v1.11.0 and turned into a hard error for v1.12.0. This change was disruptive, however it did catch real bugs in, e.g., SciPy and scikit-learn. Overall the change was worth the cost, and introducing it in -master first to allow testing, then removing it again before a release, is a -useful strategy. +the main branch first to allow testing, then removing it again before +a release, is a useful strategy. -Similar recent deprecations also look like good examples of +Similar deprecations that also look like good examples of cleanups/improvements: -- removing deprecated boolean indexing (gh-8312) -- deprecating truth testing on empty arrays (gh-9718) -- deprecating ``np.sum(generator)`` (gh-10670, one issue with this one is that - its warning message is wrong - this should error in the future). +- removing deprecated boolean indexing (in 2016, see `gh-8312 `__) +- deprecating truth testing on empty arrays (in 2017, see `gh-9718 `__) + **Removing the financial functions** -The financial functions (e.g. ``np.pmt``) are badly named, are present in the -main NumPy namespace, and don't really fit well within NumPy's scope. -They were added in 2008 after +The financial functions (e.g. ``np.pmt``) had short non-descriptive names, were +present in the main NumPy namespace, and didn't really fit well within NumPy's +scope. They were added in 2008 after `a discussion `_ on the mailing list where opinion was divided (but a majority in favor). -At the moment these functions don't cause a lot of overhead, however there are -multiple issues and PRs a year for them which cost maintainer time to deal -with. And they clutter up the ``numpy`` namespace. Discussion in 2013 happened -on removing them again (gh-2880). - -This case is borderline, but given that they're clearly out of scope, -deprecation and removal out of at least the main ``numpy`` namespace can be -proposed. Alternatively, document clearly that new features for financial -functions are unwanted, to keep the maintenance costs to a minimum. - -**Examples of features not added because of backwards compatibility** - -TODO: do we have good examples here? Possibly subclassing related? - - -Removing complete submodules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This year there have been suggestions to consider removing some or all of -``numpy.distutils``, ``numpy.f2py``, ``numpy.linalg``, and ``numpy.random``. -The motivation was that all these cost maintenance effort, and that they slow -down work on the core of Numpy (ndarrays, dtypes and ufuncs). - -The impact on downstream libraries and users would be very large, and -maintenance of these modules would still have to happen. Therefore this is -simply not a good idea; removing these submodules should not happen even for -a new major version of NumPy. - - -Subclassing of ndarray -^^^^^^^^^^^^^^^^^^^^^^ - -Subclassing of ``ndarray`` is a pain point. ``ndarray`` was not (or at least -not well) designed to be subclassed. Despite that, a lot of subclasses have -been created even within the NumPy code base itself, and some of those (e.g. -``MaskedArray``, ``astropy.units.Quantity``) are quite popular. The main -problems with subclasses are: - -- They make it hard to change ``ndarray`` in ways that would otherwise be - backwards compatible. -- Some of them change the behavior of ndarray methods, making it difficult to - write code that accepts array duck-types. - -Subclassing ``ndarray`` has been officially discouraged for a long time. Of -the most important subclasses, ``np.matrix`` will be deprecated (see gh-10142) -and ``MaskedArray`` will be kept in NumPy (`NEP 17 -`_). -``MaskedArray`` will ideally be rewritten in a way such that it uses only -public NumPy APIs. For subclasses outside of NumPy, more work is needed to -provide alternatives (e.g. mixins, see gh-9016 and gh-10446) or better support -for custom dtypes (see gh-2899). Until that is done, subclasses need to be -taken into account when making change to the NumPy code base. A future change -in NumPy to not support subclassing will certainly need a major version -increase. - - -Policy ------- - -1. Code changes that have the potential to silently change the results of a users' - code must never be made (except in the case of clear bugs). -2. Code changes that break users' code (i.e. the user will see a clear exception) - can be made, *provided the benefit is worth the cost* and suitable deprecation - warnings have been raised first. -3. Deprecation warnings are in all cases warnings that functionality will be removed. - If there is no intent to remove functionality, then deprecation in documentation - only or other types of warnings shall be used. -4. Deprecations for stylistic reasons (e.g. consistency between functions) are - strongly discouraged. - -Deprecations: - -- shall include the version numbers of both when the functionality was deprecated - and when it will be removed (either two releases after the warning is - introduced, or in the next major version). -- shall include information on alternatives to the deprecated functionality, or a - reason for the deprecation if no clear alternative is available. -- shall use ``VisibleDeprecationWarning`` rather than ``DeprecationWarning`` - for cases of relevance to end users (as opposed to cases only relevant to - libraries building on top of NumPy). -- shall be listed in the release notes of the release where the deprecation happened. - -Removal of deprecated functionality: +The financial functions didn't cause a lot of overhead, however there were +still multiple issues and PRs a year for them which cost maintainer time to +deal with. And they cluttered up the ``numpy`` namespace. Discussion on +removing them was discussed in 2013 (gh-2880, rejected) and in 2019 +(:ref:`NEP32`, accepted without significant complaints). -- shall be done after 2 releases (assuming a 6-monthly release cycle; if that changes, - there shall be at least 1 year between deprecation and removal), unless the - impact of the removal is such that a major version number increase is - warranted. -- shall be listed in the release notes of the release where the removal happened. - -Versioning: - -- removal of deprecated code can be done in any minor (but not bugfix) release. -- for heavily used functionality (e.g. removal of ``np.matrix``, of a whole submodule, - or significant changes to behavior for subclasses) the major version number shall - be increased. - -In concrete cases where this policy needs to be applied, decisions are made according -to the `NumPy governance model -`_. - -Functionality with more strict policies: - -- ``numpy.random`` has its own backwards compatibility policy, - see `NEP 19 `_. -- The file format for ``.npy`` and ``.npz`` files must not be changed in a backwards - incompatible way. +Given that they were clearly outside of NumPy's scope, moving them to a +separate ``numpy-financial`` package and removing them from NumPy after a +deprecation period made sense. That also gave users an easy way to update +their code by doing `pip install numpy-financial`. Alternatives @@ -257,34 +321,31 @@ ecosystem - being fairly conservative is required in order to not increase the extra maintenance for downstream libraries and end users to an unacceptable level. -**Semantic versioning.** - -This would change the versioning scheme for code removals; those could then -only be done when the major version number is increased. Rationale for -rejection: semantic versioning is relatively common in software engineering, -however it is not at all common in the Python world. Also, it would mean that -NumPy's version number simply starts to increase faster, which would be more -confusing than helpful. gh-10156 contains more discussion on this alternative. - Discussion ---------- -TODO - -This section may just be a bullet list including links to any discussions -regarding the NEP: - -- This includes links to mailing list threads or relevant GitHub issues. +- `Mailing list discussion on the first version of this NEP in 2018 `__ +- `Mailing list discussion on the Dec 2020 update of this NEP `__ +- `PR with review comments on the the Dec 2020 update of this NEP `__ References and Footnotes ------------------------ -.. [1] TODO +- `Issue requesting semantic versioning `__ + +- `PEP 387 - Backwards Compatibility Policy `__ + +.. [1] https://searchcode.com/ + +.. [2] https://sourcegraph.com/search + +.. [3] https://github.com/Quansight-Labs/python-api-inspect +.. [4] https://github.com/data-apis/python-record-api Copyright --------- -This document has been placed in the public domain. [1]_ +This document has been placed in the public domain. diff --git a/doc/neps/nep-0024-missing-data-2.rst b/doc/neps/nep-0024-missing-data-2.rst index 8e63629f32be..903ece1ba8e0 100644 --- a/doc/neps/nep-0024-missing-data-2.rst +++ b/doc/neps/nep-0024-missing-data-2.rst @@ -14,7 +14,7 @@ Abstract -------- *Context: this NEP was written as an alternative to NEP 12, which at the time of writing -had an implementation that was merged into the NumPy master branch.* +had an implementation that was merged into the NumPy main branch.* The principle of this NEP is to separate the APIs for masking and for missing values, according to diff --git a/doc/neps/nep-0025-missing-data-3.rst b/doc/neps/nep-0025-missing-data-3.rst index ffe208c98070..1756ce491188 100644 --- a/doc/neps/nep-0025-missing-data-3.rst +++ b/doc/neps/nep-0025-missing-data-3.rst @@ -14,7 +14,7 @@ Abstract *Context: this NEP was written as an additional alternative to NEP 12 (NEP 24 is another alternative), which at the time of writing had an implementation -that was merged into the NumPy master branch.* +that was merged into the NumPy main branch.* To try and make more progress on the whole missing values/masked arrays/... debate, it seems useful to have a more technical discussion of the pieces @@ -62,7 +62,7 @@ values, (4) it is compatible with the common practice of using NaN to indicate missingness when working with floating point numbers, (5) the dtype is already a place where "weird things can happen" -- there are a wide variety of dtypes that don't act like ordinary numbers (including structs, Python objects, -fixed-length strings, ...), so code that accepts arbitrary numpy arrays already +fixed-length strings, ...), so code that accepts arbitrary NumPy arrays already has to be prepared to handle these (even if only by checking for them and raising an error). Therefore adding yet more new dtypes has less impact on extension authors than if we change the ndarray object itself. @@ -96,7 +96,7 @@ for consistency. General strategy ================ -Numpy already has a general mechanism for defining new dtypes and slotting them +NumPy already has a general mechanism for defining new dtypes and slotting them in so that they're supported by ndarrays, by the casting machinery, by ufuncs, and so on. In principle, we could implement NA-dtypes just using these existing interfaces. But we don't want to do that, because defining all those new ufunc @@ -271,7 +271,7 @@ below. Casting ------- -FIXME: this really needs attention from an expert on numpy's casting rules. But +FIXME: this really needs attention from an expert on NumPy's casting rules. But I can't seem to find the docs that explain how casting loops are looked up and decided between (e.g., if you're casting from dtype A to dtype B, which dtype's loops are used?), so I can't go into details. But those details are tricky and @@ -338,7 +338,7 @@ Printing -------- FIXME: There should be some sort of mechanism by which values which are NA are -automatically repr'ed as NA, but I don't really understand how numpy printing +automatically repr'ed as NA, but I don't really understand how NumPy printing works, so I'll let someone else fill in this section. Indexing @@ -364,10 +364,10 @@ own global singleton.) So for now we stick to scalar indexing just returning Python API for generic NA support ================================= -NumPy will gain a global singleton called numpy.NA, similar to None, but with +NumPy will gain a global singleton called ``numpy.NA``, similar to None, but with semantics reflecting its status as a missing value. In particular, trying to treat it as a boolean will raise an exception, and comparisons with it will -produce numpy.NA instead of True or False. These basics are adopted from the +produce ``numpy.NA`` instead of True or False. These basics are adopted from the behavior of the NA value in the R project. To dig deeper into the ideas, http://en.wikipedia.org/wiki/Ternary_logic#Kleene_logic provides a starting point. @@ -453,8 +453,8 @@ The NEP also contains a proposal for a somewhat elaborate domain-specific-language for describing NA dtypes. I'm not sure how great an idea that is. (I have a bias against using strings as data structures, and find the already existing strings confusing enough as it is -- also, apparently the -NEP version of numpy uses strings like 'f8' when printing dtypes, while my -numpy uses object names like 'float64', so I'm not sure what's going on there. +NEP version of NumPy uses strings like 'f8' when printing dtypes, while my +NumPy uses object names like 'float64', so I'm not sure what's going on there. ``withNA(float64, arg1=value1)`` seems like a more pleasant way to print a dtype than "NA[f8,value1]", at least to me.) But if people want it, then cool. diff --git a/doc/neps/nep-0026-missing-data-summary.rst b/doc/neps/nep-0026-missing-data-summary.rst index f25ce9b91274..49d89d828449 100644 --- a/doc/neps/nep-0026-missing-data-summary.rst +++ b/doc/neps/nep-0026-missing-data-summary.rst @@ -17,7 +17,7 @@ The debate about how NumPy should handle missing data, a subject with many preexisting approaches, requirements, and conventions, has been long and contentious. There has been more than one proposal for how to implement support into NumPy, and there is a testable implementation which is -merged into NumPy's current master. The vast number of emails and differing +merged into NumPy's current main. The vast number of emails and differing points of view has made it difficult for interested parties to understand the issues and be comfortable with the direction NumPy is going. @@ -565,7 +565,7 @@ Recommendations for Moving Forward we're going to have to figure out how to experiment with such changes out-of-core if NumPy is to continue to evolve without forking -- might as well do it now. The existing code can live in - master, disabled, or it can live in a branch -- it'll still be there + the main branch, be disabled, or live its own branch -- it'll still be there once we know what we're doing. **Mark** thinks we should: @@ -576,8 +576,8 @@ Recommendations for Moving Forward A more detailed rationale for this recommendation is: * A solid preliminary NA-mask implementation is currently in NumPy - master. This implementation has been extensively tested - against scipy and other third-party packages, and has been in master + main. This implementation has been extensively tested + against scipy and other third-party packages, and has been in main in a stable state for a significant amount of time. * This implementation integrates deeply with the core, providing an interface which is usable in the same way R's NA support is. It diff --git a/doc/neps/nep-0027-zero-rank-arrarys.rst b/doc/neps/nep-0027-zero-rank-arrarys.rst index 2d152234cf44..cb39726759ba 100644 --- a/doc/neps/nep-0027-zero-rank-arrarys.rst +++ b/doc/neps/nep-0027-zero-rank-arrarys.rst @@ -57,7 +57,7 @@ However there are some important differences: Motivation for Array Scalars ---------------------------- -Numpy's design decision to provide 0-d arrays and array scalars in addition to +NumPy's design decision to provide 0-d arrays and array scalars in addition to native python types goes against one of the fundamental python design principles that there should be only one obvious way to do it. In this section we will try to explain why it is necessary to have three different ways to @@ -109,7 +109,7 @@ arrays to scalars were summarized as follows: are something like Python lists (which except for Object arrays they are not). -Numpy implements a solution that is designed to have all the pros and none of the cons above. +NumPy implements a solution that is designed to have all the pros and none of the cons above. Create Python scalar types for all of the 21 types and also inherit from the three that already exist. Define equivalent diff --git a/doc/neps/nep-0028-website-redesign.rst b/doc/neps/nep-0028-website-redesign.rst index e27da91c6c8b..592209a5ff89 100644 --- a/doc/neps/nep-0028-website-redesign.rst +++ b/doc/neps/nep-0028-website-redesign.rst @@ -7,7 +7,7 @@ NEP 28 — numpy.org website redesign :Author: Ralf Gommers :Author: Joe LaChance :Author: Shekhar Rajak -:Status: Accepted +:Status: Final :Type: Informational :Created: 2019-07-16 :Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-August/079889.html @@ -323,7 +323,9 @@ Alternatives we considered for the overall design of the website: Discussion ---------- -Mailing list thread discussing this NEP: TODO +- Pull request for this NEP (with a good amount of discussion): https://github.com/numpy/numpy/pull/14032 +- Email about NEP for review: https://mail.python.org/pipermail/numpy-discussion/2019-July/079856.html +- Proposal to accept this NEP: https://mail.python.org/pipermail/numpy-discussion/2019-August/079889.html References and Footnotes diff --git a/doc/neps/nep-0029-deprecation_policy.rst b/doc/neps/nep-0029-deprecation_policy.rst index 957674ee6e20..a50afcb98f9d 100644 --- a/doc/neps/nep-0029-deprecation_policy.rst +++ b/doc/neps/nep-0029-deprecation_policy.rst @@ -1,7 +1,7 @@ .. _NEP29: ================================================================================== -NEP 29 — Recommend Python and Numpy version support as a community policy standard +NEP 29 — Recommend Python and NumPy version support as a community policy standard ================================================================================== @@ -124,14 +124,14 @@ Drop Schedule :: On next release, drop support for Python 3.5 (initially released on Sep 13, 2015) - On Jan 07, 2020 drop support for Numpy 1.14 (initially released on Jan 06, 2018) + On Jan 07, 2020 drop support for NumPy 1.14 (initially released on Jan 06, 2018) On Jun 23, 2020 drop support for Python 3.6 (initially released on Dec 23, 2016) - On Jul 23, 2020 drop support for Numpy 1.15 (initially released on Jul 23, 2018) - On Jan 13, 2021 drop support for Numpy 1.16 (initially released on Jan 13, 2019) - On Jul 26, 2021 drop support for Numpy 1.17 (initially released on Jul 26, 2019) - On Dec 22, 2021 drop support for Numpy 1.18 (initially released on Dec 22, 2019) + On Jul 23, 2020 drop support for NumPy 1.15 (initially released on Jul 23, 2018) + On Jan 13, 2021 drop support for NumPy 1.16 (initially released on Jan 13, 2019) + On Jul 26, 2021 drop support for NumPy 1.17 (initially released on Jul 26, 2019) + On Dec 22, 2021 drop support for NumPy 1.18 (initially released on Dec 22, 2019) On Dec 26, 2021 drop support for Python 3.7 (initially released on Jun 27, 2018) - On Jun 21, 2022 drop support for Numpy 1.19 (initially released on Jun 20, 2020) + On Jun 21, 2022 drop support for NumPy 1.19 (initially released on Jun 20, 2020) On Apr 14, 2023 drop support for Python 3.8 (initially released on Oct 14, 2019) @@ -249,18 +249,18 @@ Code to generate support and drop schedule tables :: from datetime import datetime, timedelta - data = """Jan 15, 2017: Numpy 1.12 + data = """Jan 15, 2017: NumPy 1.12 Sep 13, 2015: Python 3.5 Dec 23, 2016: Python 3.6 Jun 27, 2018: Python 3.7 - Jun 07, 2017: Numpy 1.13 - Jan 06, 2018: Numpy 1.14 - Jul 23, 2018: Numpy 1.15 - Jan 13, 2019: Numpy 1.16 - Jul 26, 2019: Numpy 1.17 + Jun 07, 2017: NumPy 1.13 + Jan 06, 2018: NumPy 1.14 + Jul 23, 2018: NumPy 1.15 + Jan 13, 2019: NumPy 1.16 + Jul 26, 2019: NumPy 1.17 Oct 14, 2019: Python 3.8 - Dec 22, 2019: Numpy 1.18 - Jun 20, 2020: Numpy 1.19 + Dec 22, 2019: NumPy 1.18 + Jun 20, 2020: NumPy 1.19 """ releases = [] @@ -284,7 +284,7 @@ Code to generate support and drop schedule tables :: py_major,py_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'Python')[-1] minpy = f"{py_major}.{py_minor+1}+" - num_major,num_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'Numpy')[-1] + num_major,num_minor = sorted([int(x) for x in r[2].split('.')] for r in releases if r[1] == 'NumPy')[-1] minnum = f"{num_major}.{num_minor+1}+" toprint_drop_dates = [''] diff --git a/doc/neps/nep-0032-remove-financial-functions.rst b/doc/neps/nep-0032-remove-financial-functions.rst index 1c3722f46cb3..b57ae943fa96 100644 --- a/doc/neps/nep-0032-remove-financial-functions.rst +++ b/doc/neps/nep-0032-remove-financial-functions.rst @@ -5,7 +5,7 @@ NEP 32 — Remove the financial functions from NumPy ================================================== :Author: Warren Weckesser -:Status: Accepted +:Status: Final :Type: Standards Track :Created: 2019-08-30 :Resolution: https://mail.python.org/pipermail/numpy-discussion/2019-September/080074.html @@ -174,19 +174,19 @@ References and footnotes .. [1] Financial functions, https://numpy.org/doc/1.17/reference/routines.financial.html -.. [2] Numpy-discussion mailing list, "Simple financial functions for NumPy", +.. [2] NumPy-Discussion mailing list, "Simple financial functions for NumPy", https://mail.python.org/pipermail/numpy-discussion/2008-April/032353.html -.. [3] Numpy-discussion mailing list, "add xirr to numpy financial functions?", +.. [3] NumPy-Discussion mailing list, "add xirr to numpy financial functions?", https://mail.python.org/pipermail/numpy-discussion/2009-May/042645.html -.. [4] Numpy-discussion mailing list, "Definitions of pv, fv, nper, pmt, and rate", +.. [4] NumPy-Discussion mailing list, "Definitions of pv, fv, nper, pmt, and rate", https://mail.python.org/pipermail/numpy-discussion/2009-June/043188.html .. [5] Get financial functions out of main namespace, https://github.com/numpy/numpy/issues/2880 -.. [6] Numpy-discussion mailing list, "Deprecation of financial routines", +.. [6] NumPy-Discussion mailing list, "Deprecation of financial routines", https://mail.python.org/pipermail/numpy-discussion/2013-August/067409.html .. [7] ``component: numpy.lib.financial`` issues, @@ -198,7 +198,7 @@ References and footnotes .. [9] Quansight-Labs/python-api-inspect, https://github.com/Quansight-Labs/python-api-inspect/ -.. [10] Numpy-discussion mailing list, "NEP 32: Remove the financial functions +.. [10] NumPy-Discussion mailing list, "NEP 32: Remove the financial functions from NumPy" https://mail.python.org/pipermail/numpy-discussion/2019-September/079965.html @@ -206,7 +206,7 @@ References and footnotes remove the financial functions. https://mail.google.com/mail/u/0/h/1w0mjgixc4rpe/?&th=16d5c38be45f77c4&q=nep+32&v=c&s=q -.. [12] Numpy-discussion mailing list, "Proposal to accept NEP 32: Remove the +.. [12] NumPy-Discussion mailing list, "Proposal to accept NEP 32: Remove the financial functions from NumPy" https://mail.python.org/pipermail/numpy-discussion/2019-September/080074.html diff --git a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst index 5ec01081a9b0..3a689a4dcd24 100644 --- a/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst +++ b/doc/neps/nep-0035-array-creation-dispatch-with-array-function.rst @@ -5,11 +5,11 @@ NEP 35 — Array Creation Dispatching With __array_function__ =========================================================== :Author: Peter Andreas Entschev -:Status: Draft +:Status: Final :Type: Standards Track :Created: 2019-10-15 :Updated: 2020-11-06 -:Resolution: +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2021-May/081761.html Abstract -------- @@ -327,7 +327,7 @@ The first version of this proposal suggested the implementation above as one viable solution for NumPy functions implemented in C. However, due to the downsides pointed out above we have decided to discard any changes on the Python side and resolve those issues with a pure-C implementation. Please refer to -[implementation]_ for details. +[7]_ for details. Reading the Reference Array Downstream ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -344,6 +344,7 @@ An example of such use is to create a new Dask array while preserving its backend type: .. code:: python + # Returns dask.array np.asarray([1, 2, 3], like=da.array(cp.array(()))) @@ -357,6 +358,7 @@ not use the ``like=`` argument via the ``self`` attribute from instead: .. code:: python + # Returns dask.array np.asarray([1, 2, 3], like=da.array(cp.array(()))) @@ -379,6 +381,7 @@ implemented in Dask. The current relevant part of ``__array_function__`` definition in Dask is seen below: .. code:: python + def __array_function__(self, func, types, args, kwargs): # Code not relevant for this example here @@ -388,6 +391,7 @@ definition in Dask is seen below: And this is how the updated code would look like: .. code:: python + def __array_function__(self, func, types, args, kwargs): # Code not relevant for this example here @@ -419,8 +423,9 @@ by downstream libraries is beyond the scope of the present proposal. Discussion ---------- -.. [implementation] `Implementation's pull request on GitHub `_ -.. [discussion] `Further discussion on implementation and the NEP's content `_ +- `Further discussion on implementation and the NEP's content `_ +- `Decision to release an experimental implementation in NumPy 1.20.0 `__ + References ---------- @@ -437,6 +442,8 @@ References .. [6] `NEP 37 — A dispatch protocol for NumPy-like modules `_. +.. [7] `Implementation's pull request on GitHub `_ + Copyright --------- diff --git a/doc/neps/nep-0036-fair-play.rst b/doc/neps/nep-0036-fair-play.rst index 32e5e1e75c03..34c8f6eb8286 100644 --- a/doc/neps/nep-0036-fair-play.rst +++ b/doc/neps/nep-0036-fair-play.rst @@ -48,7 +48,7 @@ identifying the problematic release, locating alterations, and determining an appropriate course of action. During a community call on `October 16th, 2019 -`__ +`__ the community resolved to draft guidelines on the distribution of modified NumPy versions. @@ -129,7 +129,7 @@ Fair play rules discussion on the mailing list before monkeypatching NumPy. Questions and answers -------------------- +--------------------- **Q:** We would like to distribute an optimized version of NumPy that utilizes special instructions for our company's CPU. You recommend diff --git a/doc/neps/nep-0040-legacy-datatype-impl.rst b/doc/neps/nep-0040-legacy-datatype-impl.rst index 39889109de40..7ea7f6df35e9 100644 --- a/doc/neps/nep-0040-legacy-datatype-impl.rst +++ b/doc/neps/nep-0040-legacy-datatype-impl.rst @@ -6,7 +6,7 @@ NEP 40 — Legacy Datatype Implementation in NumPy :title: Legacy Datatype Implementation in NumPy :Author: Sebastian Berg -:Status: Draft +:Status: Final :Type: Informational :Created: 2019-07-17 diff --git a/doc/neps/nep-0042-new-dtypes.rst b/doc/neps/nep-0042-new-dtypes.rst index d1ddb7101706..bb85f1d10cba 100644 --- a/doc/neps/nep-0042-new-dtypes.rst +++ b/doc/neps/nep-0042-new-dtypes.rst @@ -8,10 +8,10 @@ NEP 42 — New and extensible DTypes :Author: Sebastian Berg :Author: Ben Nathanson :Author: Marten van Kerkwijk -:Status: Draft +:Status: Accepted :Type: Standard :Created: 2019-07-17 - +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2020-October/081038.html .. note:: @@ -203,7 +203,7 @@ Other elements of the casting implementation is the ``CastingImpl``: # Object describing and performing the cast casting : casting - def resolve_descriptors(self, Tuple[DType] : input) -> (casting, Tuple[DType]): + def resolve_descriptors(self, Tuple[DTypeMeta], Tuple[DType|None] : input) -> (casting, Tuple[DType]): raise NotImplementedError # initially private: @@ -213,6 +213,8 @@ Other elements of the casting implementation is the ``CastingImpl``: which describes the casting from one DType to another. In :ref:`NEP 43 ` this ``CastingImpl`` object is used unchanged to support universal functions. +Note that the name ``CastingImpl`` here will be generically called +``ArrayMethod`` to accomodate both casting and universal functions. ****************************************************************************** @@ -525,7 +527,8 @@ This means the implementation will work like this:: # Find what dtype1 is cast to when cast to the common DType # by using the CastingImpl as described below: castingimpl = get_castingimpl(type(dtype1), common) - safety, (_, dtype1) = castingimpl.resolve_descriptors((dtype1, None)) + safety, (_, dtype1) = castingimpl.resolve_descriptors( + (common, common), (dtype1, None)) assert safety == "safe" # promotion should normally be a safe cast if type(dtype2) is not common: @@ -652,7 +655,7 @@ and implements the following methods and attributes: * To report safeness, - ``resolve_descriptors(self, Tuple[DType] : input) -> casting, Tuple[DType]``. + ``resolve_descriptors(self, Tuple[DTypeMeta], Tuple[DType|None] : input) -> casting, Tuple[DType]``. The ``casting`` output reports safeness (safe, unsafe, or same-kind), and the tuple is used for more multistep casting, as in the example below. @@ -691,7 +694,7 @@ The full process is: 1. Call - ``CastingImpl[Int24, String].resolve_descriptors((int24, "S20"))``. + ``CastingImpl[Int24, String].resolve_descriptors((Int24, String), (int24, "S20"))``. This provides the information that ``CastingImpl[Int24, String]`` only implements the cast of ``int24`` to ``"S8"``. @@ -716,7 +719,7 @@ The full process is: to call - ``CastingImpl[Int24, String].resolve_descriptors((int24, None))``. + ``CastingImpl[Int24, String].resolve_descriptors((Int24, String), (int24, None))``. In this case the result of ``(int24, "S8")`` defines the correct cast: @@ -763,8 +766,8 @@ even if the user provides only an ``int16 -> int24`` cast. This proposal does not provide that, but future work might find such casts dynamically, or at least allow ``resolve_descriptors`` to return arbitrary ``dtypes``. -If ``CastingImpl[Int8, Int24].resolve_descriptors((int8, int24))`` returns -``(int16, int24)``, the actual casting process could be extended to include +If ``CastingImpl[Int8, Int24].resolve_descriptors((Int8, Int24), (int8, int24))`` +returns ``(int16, int24)``, the actual casting process could be extended to include the ``int8 -> int16`` cast. This adds a step. @@ -774,7 +777,7 @@ The implementation for casting integers to datetime would generally say that this cast is unsafe (because it is always an unsafe cast). Its ``resolve_descriptors`` function may look like:: - def resolve_descriptors(self, given_dtypes): + def resolve_descriptors(self, DTypes, given_dtypes): from_dtype, to_dtype = given_dtypes from_dtype = from_dtype.ensure_canonical() # ensure not byte-swapped if to_dtype is None: @@ -835,9 +838,10 @@ Its ``resolve_descriptors`` function may look like:: **Notes:** -The proposed ``CastingImpl`` is designed to be identical to the -``PyArrayMethod`` proposed in NEP43 as part of restructuring ufuncs to handle -new DTypes. +``CastingImpl`` is used as a name in this NEP to clarify that it implements +all functionality related to a cast. It is meant to be identical to the +``ArrayMethod`` proposed in NEP 43 as part of restructuring ufuncs to handle +new DTypes. All type definitions are expected to be named ``ArrayMethod``. The way dispatching works for ``CastingImpl`` is planned to be limited initially and fully opaque. In the future, it may or may not be moved into a @@ -1297,8 +1301,9 @@ The external API for ``CastingImpl`` will be limited initially to defining: instance if the second string is shorter. If neither type is parametric the ``resolve_descriptors`` must use it. -* ``resolve_descriptors(dtypes_in[2], dtypes_out[2], casting_out) -> int {0, - -1}`` The out +* ``resolve_descriptors(PyArrayMethodObject *self, PyArray_DTypeMeta *DTypes[2], + PyArray_Descr *dtypes_in[2], PyArray_Descr *dtypes_out[2], NPY_CASTING *casting_out) + -> int {0, -1}`` The out dtypes must be set correctly to dtypes which the strided loop (transfer function) can handle. Initially the result must have instances of the same DType class as the ``CastingImpl`` is defined for. The @@ -1307,9 +1312,12 @@ The external API for ``CastingImpl`` will be limited initially to defining: A new, additional flag, ``_NPY_CAST_IS_VIEW``, can be set to indicate that no cast is necessary and a view is sufficient to perform the cast. The cast should return - ``-1`` when a custom error is set and ``NPY_NO_CASTING`` to indicate - that a generic casting error should be set (this is in most cases - preferable). + ``-1`` when an error occurred. If a cast is not possible (but no error + occurred), a ``-1`` result should be returned *without* an error set. + *This point is under consideration, we may use ``-1`` to indicate + a general error, and use a different return value for an impossible cast.* + This means that it is *not* possible to inform the user about why a cast is + impossible. * ``strided_loop(char **args, npy_intp *dimensions, npy_intp *strides, ...) -> int {0, -1}`` (signature will be fully defined in :ref:`NEP 43 `) @@ -1326,7 +1334,7 @@ Although verbose, the API will mimic the one for creating a new DType: typedef struct{ int flags; /* e.g. whether the cast requires the API */ int nin, nout; /* Number of Input and outputs (always 1) */ - NPY_CASTING casting; /* The default casting level */ + NPY_CASTING casting; /* The "minimal casting level" */ PyArray_DTypeMeta *dtypes; /* input and output DType class */ /* NULL terminated slots defining the methods */ PyType_Slot *slots; @@ -1334,7 +1342,7 @@ Although verbose, the API will mimic the one for creating a new DType: The focus differs between casting and general ufuncs. For example, for casts ``nin == nout == 1`` is always correct, while for ufuncs ``casting`` is -expected to be usually `"safe"`. +expected to be usually `"no"`. **Notes:** We may initially allow users to define only a single loop. Internally NumPy optimizes far more, and this should be made public @@ -1349,6 +1357,11 @@ incrementally in one of two ways: * Or, more likely, expose the ``get_loop`` function which is passed additional information, such as the fixed strides (similar to our internal API). +* The casting level denotes the minimal guaranteed casting level and can be + ``-1`` if the cast may be impossible. For most non-parametric casts, this + value will be the casting level. NumPy may skip the ``resolve_descriptors`` + call for ``np.can_cast()`` when the result is ``True`` based on this level. + The example does not yet include setup and error handling. Since these are similar to the UFunc machinery, they will be defined in :ref:`NEP 43 ` and then incorporated identically into casting. diff --git a/doc/neps/nep-0043-extensible-ufuncs.rst b/doc/neps/nep-0043-extensible-ufuncs.rst index 96d4794f3774..cd73108e4fbb 100644 --- a/doc/neps/nep-0043-extensible-ufuncs.rst +++ b/doc/neps/nep-0043-extensible-ufuncs.rst @@ -150,6 +150,11 @@ The masked type resolvers specifically will *not* remain supported, but has no known users (including NumPy itself, which only uses the default version). +Further, no compatibility attempt will be made for *calling* as opposed +to providing either the normal or the masked type resolver. As NumPy +will use it only as a fallback. There are no known users of this +(undocumented) possibility. + While the above changes potentially break some workflows, we believe that the long-term improvements vastly outweigh this. Further, packages such as astropy and Numba are capable of adapting so that @@ -235,20 +240,30 @@ to define string equality, will be added to a ufunc. class StringEquality(BoundArrayMethod): nin = 1 nout = 1 + # DTypes are stored on the BoundArrayMethod and not on the internal + # ArrayMethod, to reference cyles. DTypes = (String, String, Bool) - def resolve_descriptors(context, given_descrs): + def resolve_descriptors(self: ArrayMethod, DTypes, given_descrs): """The strided loop supports all input string dtype instances and always returns a boolean. (String is always native byte order.) Defining this function is not necessary, since NumPy can provide it by default. + + The `self` argument here refers to the unbound array method, so + that DTypes are passed in explicitly. """ - assert isinstance(given_descrs[0], context.DTypes[0]) - assert isinstance(given_descrs[1], context.DTypes[1]) + assert isinstance(given_descrs[0], DTypes[0]) + assert isinstance(given_descrs[1], DTypes[1]) + assert given_descrs[2] is None or isinstance(given_descrs[2], DTypes[2]) - # The operation is always "safe" casting (most ufuncs are) - return (given_descrs[0], given_descrs[1], context.DTypes[2]()), "safe" + out_descr = given_descrs[2] # preserve input (e.g. metadata) + if given_descrs[2] is None: + out_descr = DTypes[2]() + + # The operation is always "no" casting (most ufuncs are) + return (given_descrs[0], given_descrs[1], out_descr), "no" def strided_loop(context, dimensions, data, strides, innerloop_data): """The 1-D strided loop, similar to those used in current ufuncs""" @@ -322,12 +337,12 @@ steps involved in a call to a universal function in NumPy. A UFunc call is split into the following steps: -1. *Handle ``__array_ufunc__`` protocol:* +1. Handle ``__array_ufunc__`` protocol: * For array-likes such as a Dask arrays, NumPy can defer the operation. This step is performed first, and unaffected by this NEP (compare :ref:`NEP18`). -2. *Promotion and dispatching* +2. Promotion and dispatching * Given the DTypes of all inputs, find the correct implementation. E.g. an implementation for ``float64``, ``int64`` or a user-defined DType. @@ -336,7 +351,7 @@ A UFunc call is split into the following steps: For example, adding a ``float32`` and a ``float64`` is implemented by first casting the ``float32`` to ``float64``. -3. *Parametric ``dtype`` resolution:* +3. Parametric ``dtype`` resolution: * In general, whenever an output DType is parametric the parameters have to be found (resolved). @@ -347,15 +362,16 @@ A UFunc call is split into the following steps: which fills in the default dtype instances (ensuring for example native byte order). -4. *Preparing the iteration:* +4. Preparing the iteration: * This step is largely handled by ``NpyIter`` internally (the iterator). * Allocate all outputs and temporary buffers necessary to perform casts. + This *requires* the dtypes as resolved in step 3. * Find the best iteration order, which includes information to efficiently implement broadcasting. For example, adding a single value to an array repeats the same value. -5. *Setup and fetch the C-level function:* +5. Setup and fetch the C-level function: * If necessary, allocate temporary working space. * Find the C-implemented, light weight, inner-loop function. @@ -368,33 +384,35 @@ A UFunc call is split into the following steps: the GIL may be released (to allow threading). * Clear floating point exception flags. -6. *Perform the actual calculation:* +6. Perform the actual calculation: * Run the DType specific inner-loop function. * The inner-loop may require access to additional data, such as dtypes or additional data set in the previous step. * The inner-loop function may be called an undefined number of times. -7. *Finalize:* +7. Finalize: - * Free any temporary working space allocated in 5. + * Free any temporary working space allocated in step 5. * Check for floating point exception flags. * Return the result. The ``ArrayMethod`` provides a concept to group steps 3 to 6 and partially 7. -However, implementers of a new ufunc or ``ArrayMethod`` do not need to -customize the behaviour in steps 4 or 6, aside from the inner-loop function. -For the ``ArrayMethod`` implementer, the central steps to have control over -are step 3 and step 5 to provide the custom inner-loop function. -Further customization is a potential future extension. +However, implementers of a new ufunc or ``ArrayMethod`` usually do not need to +customize the behaviour in steps 4 or 6 which NumPy can and does provide. +For the ``ArrayMethod`` implementer, the central steps to customize +are step 3 and step 5. These provide the custom inner-loop function and +potentially inner-loop specific setup. +Further customization is possible and anticipated as future extensions. -Step 2. is promotion and dispatching which will also be restructured -with new API which allows influencing the process where necessary. +Step 2. is promotion and dispatching and will be restructured +with new API to allow customization of the process where necessary. Step 1 is listed for completeness and is unaffected by this NEP. The following sketch provides an overview of step 2 to 6 with an emphasize -of how dtypes are handled: +of how dtypes are handled and which parts are customizable ("Registered") +and which are handled by NumPy: .. figure:: _static/nep43-sketch.svg :figclass: align-center @@ -410,21 +428,19 @@ We use the ``class`` syntax to describe the information required to create a new ``ArrayMethod`` object: .. code-block:: python - :dedent: 0 class ArrayMethod: name: str # Name, mainly useful for debugging # Casting safety information (almost always "safe", necessary to # unify casting and universal functions) - casting: Casting = "safe" + casting: Casting = "no" # More general flags: flags: int - @staticmethod - def resolve_descriptors( - Context: context, Tuple[DType]: given_descrs)-> Casting, Tuple[DType]: + def resolve_descriptors(self, + Tuple[DTypeMeta], Tuple[DType|None]: given_descrs) -> Casting, Tuple[DType]: """Returns the safety of the operation (casting safety) and the """ # A default implementation can be provided for non-parametric @@ -456,7 +472,6 @@ a new ``ArrayMethod`` object: With ``Context`` providing mostly static information about the function call: .. code-block:: python - :dedent: 0 class Context: # The ArrayMethod object itself: @@ -468,8 +483,6 @@ With ``Context`` providing mostly static information about the function call: int : nin = 1 # The number of output arguments: int : nout = 1 - # The DTypes this Method operates on/is defined for: - Tuple[DTypeMeta] : dtypes # The actual dtypes instances the inner-loop operates on: Tuple[DType] : descriptors @@ -616,7 +629,8 @@ definitions (see also :ref:`NEP 42 ` ``CastingImpl``): NPY_CASTING resolve_descriptors( - PyArrayMethod_Context *context, + PyArrayMethodObject *self, + PyArray_DTypeMeta *dtypes, PyArray_Descr *given_dtypes[nin+nout], PyArray_Descr *loop_dtypes[nin+nout]); @@ -647,25 +661,25 @@ definitions (see also :ref:`NEP 42 ` ``CastingImpl``): for example for implementing warnings (see Error and Warning Handling below). To simplify this NumPy will pass a single zero initialized ``npy_intp *`` when ``user_data`` is not set. - *NOTE that it would be possible to pass this as part of ``Context``.* + *Note that it would be possible to pass this as part of Context.* * The optional ``get_loop`` function will not be public initially, to avoid finalizing the API which requires design choices also with casting: - .. code-block:: + .. code-block:: C innerloop * get_loop( PyArrayMethod_Context *context, - /* (move_references is currently used internally for casting) */ int aligned, int move_references, npy_intp *strides, PyArray_StridedUnaryOp **out_loop, NpyAuxData **innerloop_data, NPY_ARRAYMETHOD_FLAGS *flags); - The ``NPY_ARRAYMETHOD_FLAGS`` can indicate whether the Python API is required - and floating point errors must be checked. + ``NPY_ARRAYMETHOD_FLAGS`` can indicate whether the Python API is required + and floating point errors must be checked. ``move_references`` is used + internally for NumPy casting at this time. * The inner-loop function:: @@ -737,8 +751,15 @@ This step is required to allocate output arrays and has to happen before casting can be prepared. While the returned casting-safety (``NPY_CASTING``) will almost always be -"safe" for universal functions, including it has two big advantages: - +"no" for universal functions, including it has two big advantages: + +* ``-1`` indicates that an error occurred. If a Python error is set, it will + be raised. If no Python error is set this will be considered an "impossible" + cast and a custom error will be set. (This distinction is important for the + ``np.can_cast()`` function, which should raise the first one and return + ``False`` in the second case, it is not noteworthy for typical ufuncs). + *This point is under consideration, we may use -1 to indicate + a general error, and use a different return value for an impossible cast.* * Returning the casting safety is central to NEP 42 for casting and allows the unmodified use of ``ArrayMethod`` there. * There may be a future desire to implement fast but unsafe implementations. @@ -746,7 +767,7 @@ While the returned casting-safety (``NPY_CASTING``) will almost always be perspective. Currently, this would use ``int64 + int64 -> int64`` and then cast to ``int32``. An implementation that skips the cast would have to signal that it effectively includes the "same-kind" cast and is - thus not considered "safe". + thus not considered "no". ``get_loop`` method @@ -774,7 +795,7 @@ Extending the inner-loop signature Extending the inner-loop signature is another central and necessary part of the NEP. -**Passing in the ``Context``:** +**Passing in the Context:** Passing in the ``Context`` potentially allows for the future extension of the signature by adding new fields to the context struct. @@ -802,8 +823,8 @@ exists in NumPy for this purpose, it seems a natural choice. To simplify some use-cases (see "Error Handling" below), we will pass a ``npy_intp *innerloop_data = 0`` instead when ``innerloop_data`` is not provided. -*Note: Since ``get_loop`` is expected to be private initially we can gain -experience with ``innerloop_data`` before exposing it as public API.* +*Note:* Since ``get_loop`` is expected to be private initially we can gain +experience with ``innerloop_data`` before exposing it as public API. **Return value:** @@ -934,7 +955,7 @@ This wrapped ``ArrayMethod`` will have two additional methods: have changed the byte-order), and further resolve the physical unit making the final signature:: - ``Unit[Float64]("m") + Unit[Float64]("m") -> Unit[Float64]("m")`` + Unit[Float64]("m") + Unit[Float64]("m") -> Unit[Float64]("m") the UFunc machinery will take care of casting the "km" input to "m". @@ -1019,8 +1040,8 @@ In the future we expect that ``ArrayMethod``\ s can also be defined for **Promotion:** If a matching ``ArrayMethod`` exists, dispatching is straight forward. -However, when it does not, require additional definitions to implement -promotion: +However, when it does not, additional definitions are required to implement +this "promotion": * By default any UFunc has a promotion which uses the common DType of all inputs and dispatches a second time. This is well-defined for most @@ -1065,7 +1086,7 @@ In this case, just as a ``Timedelta64 * int64`` and ``int64 * timedelta64`` ``ArrayMethod`` is necessary, a second promoter will have to be registered to handle the case where the integer is passed first. -**Dispatching rules for ``ArrayMethod`` and Promoters:** +**Dispatching rules for ArrayMethod and Promoters:** Promoter and ``ArrayMethod`` are discovered by finding the best match as defined by the DType class hierarchy. diff --git a/doc/neps/nep-0046-sponsorship-guidelines.rst b/doc/neps/nep-0046-sponsorship-guidelines.rst new file mode 100644 index 000000000000..b8b312aa52d9 --- /dev/null +++ b/doc/neps/nep-0046-sponsorship-guidelines.rst @@ -0,0 +1,256 @@ +.. _NEP46: + +===================================== +NEP 46 — NumPy Sponsorship Guidelines +===================================== + +:Author: Ralf Gommers +:Status: Accepted +:Type: Process +:Created: 2020-12-27 +:Resolution: https://mail.python.org/pipermail/numpy-discussion/2021-January/081424.html + + +Abstract +-------- + +This NEP provides guidelines on how the NumPy project will acknowledge +financial and in-kind support. + + +Motivation and Scope +-------------------- + +In the past few years, the NumPy project has gotten significant financial +support, as well as dedicated work time for maintainers to work on NumPy. There +is a need to acknowledge that support - it's the right thing to do, it's +helpful when looking for new funding, and funders and organizations expect or +require it, Furthermore, having a clear policy for how NumPy acknowledges +support is helpful when searching for new support. Finally, this policy may +help set reasonable expectations for potential funders. + +This NEP is aimed at both the NumPy community - who can use it as a guideline +when looking for support on behalf of the project and when acknowledging +existing support - and at past, current and prospective sponsors, who often +want or need to know what they get in return for their support other than a +healthier NumPy. + +The scope of this proposal includes: + +- direct financial support, employers providing paid time for NumPy maintainers + and regular contributors, and in-kind support such as free hardware resources or + services, +- where and how NumPy acknowledges support (e.g., logo placement on the website), +- the amount and duration of support which leads to acknowledgement, and +- who in the NumPy project is responsible for sponsorship related topics, and + how to contact them. + + +How NumPy will acknowledge support +---------------------------------- + +There will be two different ways to acknowledge financial and in-kind support: +one to recognize significant active support, and another one to recognize +support received in the past and smaller amounts of support. + +Entities who fall under "significant active supporter" we'll call Sponsor. +The minimum level of support given to NumPy to be considered a Sponsor are: + +- $30,000/yr for unrestricted financial contributions (e.g., donations) +- $60,000/yr for financial contributions for a particular purpose (e.g., grants) +- $100,000/yr for in-kind contributions (e.g., time for employees to contribute) + +We define support being active as: + +- for a one-off donation: it was received within the previous 12 months, +- for recurring or financial or in-kind contributions: they should be ongoing. + +After support moves from "active" to "inactive" status, the acknowledgement +will be left in its place for at least another 6 months. If appropriate, the +funding team can discuss opportunities for renewal with the sponsor. After +those 6 months, acknowledgement may be moved to the historical overview. The +exact timing of this move is at the discretion of the funding team, because +there may be reasons to keep it in the more prominent place for longer. + +The rationale for the above funding levels is that unrestricted financial +contributions are typically the most valuable for the project, and the hardest +to obtain. The opposite is true for in-kind contributions. The dollar value of +the levels also reflect that NumPy's needs have grown to the point where we +need multiple paid developers in order to effectively support our user base and +continue to move the project forward. Financial support at or above these +levels is needed to be able to make a significant difference. + +Sponsors will get acknowledged through: + +- a small logo displayed on the front page of the NumPy website +- prominent logo placement on https://numpy.org/about/ +- logos displayed in talks about NumPy by maintainers +- announcements of the sponsorship on the NumPy mailing list and the numpy-team + Twitter account + +In addition to Sponsors, we already have the concept of Institutional Partner +(defined in NumPy's +`governance document `__), +for entities who employ a NumPy maintainer and let them work on NumPy as part +of their official duties. The governance document doesn't currently define a +minimum amount of paid maintainer time needed to be considered for partnership. +Therefore we propose that level here, roughly in line with the sponsorship +levels: + +- 6 person-months/yr of paid work time for one or more NumPy maintainers or + regular contributors to any NumPy team or activity + +Institutional Partners get the same benefits as Sponsors, in addition to what +is specified in the NumPy governance document. + +Finally, a new page on the website (https://numpy.org/funding/, linked from the +About page) will be added to acknowledge all current and previous sponsors, +partners, and any other entities and individuals who provided $5,000 or more of +financial or in-kind support. This page will include relevant details of +support (dates, amounts, names, and purpose); no logos will be used on this +page. The rationale for the $5,000 minimum level is to keep the amount of work +maintaining the page reasonable; the level is the equivalent of, e.g., one GSoC +or a person-week's worth of engineering time in a Western country, which seems +like a reasonable lower limit. + + +Implementation +-------------- + +The following content changes need to be made: + +- Add a section with small logos towards the bottom of the `numpy.org + `__ website. +- Create a full list of historical and current support and deploy it to + https://numpy.org/funding. +- Update the NumPy governance document for changes to Institutional Partner + eligibility requirements and benefits. +- Update https://numpy.org/about with details on how to get in touch with the + NumPy project about sponsorship related matters (see next section). + + +NumPy Funding Team +~~~~~~~~~~~~~~~~~~ + +At the moment NumPy has only one official body, the Steering Council, and no +good way to get in touch with either that body or any person or group +responsible for funding and sponsorship related matters. The way this is +typically done now is to somehow find the personal email of a maintainer, and +email them in private. There is a need to organize this more transparently - a +potential sponsor isn't likely to inquire through the mailing list, nor is it +easy for a potential sponsor to know if they're reaching out to the right +person in private. + +https://numpy.org/about/ already says that NumPy has a "funding and grants" +team. However that is not the case. We propose to organize this team, name team +members on it, and add the names of those team members plus a dedicated email +address for the team to the About page. + + +Status before this proposal +--------------------------- + +Acknowledgement of support +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +At the time of writing (Dec 2020), the logos of the four largest financial +sponsors and two institutional partners are displayed on +https://numpy.org/about/. The `Nature paper about NumPy `__ +mentions some early funding. No comprehensive list of received funding and +in-kind support is published anywhere. + +Decisions on which logos to list on the website have been made mostly by the +website team. Decisions on which entities to recognize as Institutional Partner +have been made by the NumPy Steering Council. + + +NumPy governance, decision-making, and financial oversight +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +*This section is meant as context for the reader, to help put the rest of this +NEP in perspective, and perhaps answer questions the reader has when reading +this as a potential sponsor.* + +NumPy has a formal governance structure defined in +`this governance document `__). +Decisions are made by consensus among all active participants in a discussion +(typically on the mailing list), and if consensus cannot be reached then the +Steering Council takes the decision (also by consensus). + +NumPy is a sponsored project of NumFOCUS, a US-based 501(c)3 nonprofit. +NumFOCUS administers NumPy funds, and ensures they are spent in accordance with +its mission and nonprofit status. In practice, NumPy has a NumFOCUS +subcommittee (with its members named in the NumPy governance document) who can +authorize financial transactions. Those transactions, for example paying a +contractor for a particular activity or deliverable, are decided on by the +NumPy Steering Council. + + +Alternatives +------------ + +*Tiered sponsorship levels.* We considered using tiered sponsorship levels, and +rejected this alternative because it would be more complex, and not necessarily +communicate the right intent - the minimum levels are for us to determine how +to acknowledge support that we receive, not a commercial value proposition. +Entities typically will support NumPy because they rely on the project or want +to help advance it, and not to get brand awareness through logo placement. + +*Listing all donations*. Note that in the past we have received many smaller +donations, mostly from individuals through NumFOCUS. It would be great to list +all of those contributions, but given the way we receive information on those +donations right now, that would be quite labor-intensive. If we manage to move +to a more suitable platform, such as `Open Collective `__, +in the future, we should reconsider listing all individual donations. + + +Related Work +------------ + +Here we provide a few examples of how other projects handle sponsorship +guidelines and acknowledgements. + +*Scikit-learn* has a narrow banner with logos at the bottom of +https://scikit-learn.org, and a list of present funding and past sponsors at +https://scikit-learn.org/stable/about.html#funding. Plus a separate section +"Infrastructure support" at the bottom of that same About page. + +*Jupyter* has logos of sponsors and institutional partners in two sections on +https://jupyter.org/about. Some subprojects have separate approaches, for +example sponsors are listed (by using the `all-contributors +`__ bot) in the README for +`jupyterlab-git `__. For a recent +discussion on that, see `here `_. + +*NumFOCUS* has a large banner with sponsor logos on its front page at +https://numfocus.org, and a full page with sponsors at different sponsorship +levels listed at https://numfocus.org/sponsors. They also have a +`Corporate Sponsorship Prospectus `__, +which includes a lot of detail on both sponsorship levels and benefits, as well +as how that helps NumFOCUS-affiliated projects (including NumPy). + + +Discussion +---------- + +- `Mailing list thread discussing this NEP `__ +- `PR with review of the NEP draft `__ + + +References and Footnotes +------------------------ + +- `Inside NumPy: preparing for the next decade `__ presentation at SciPy'19 discussing the impact of the first NumPy grant. +- `Issue `__ and + `email `__ + where IBM offered a $5,000 bounty for VSX SIMD support +- `JupyterLab Corporate Engagement and Contribution Guide `__ + + +.. _jupyterlab-git acknowledgements discussion: https://github.com/jupyterlab/jupyterlab-git/pull/530 + + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0047-array-api-standard.rst b/doc/neps/nep-0047-array-api-standard.rst new file mode 100644 index 000000000000..19965c20d908 --- /dev/null +++ b/doc/neps/nep-0047-array-api-standard.rst @@ -0,0 +1,590 @@ +.. _NEP47: + +======================================== +NEP 47 — Adopting the array API standard +======================================== + +:Author: Ralf Gommers +:Author: Stephan Hoyer +:Author: Aaron Meurer +:Status: Draft +:Type: Standards Track +:Created: 2021-01-21 +:Resolution: + + +Abstract +-------- + +We propose to adopt the `Python array API standard`_, developed by the +`Consortium for Python Data API Standards`_. Implementing this as a separate +new namespace in NumPy will allow authors of libraries which depend on NumPy +as well as end users to write code that is portable between NumPy and all +other array/tensor libraries that adopt this standard. + +.. note:: + + We expect that this NEP will remain in a draft state for quite a while. + Given the large scope we don't expect to propose it for acceptance any + time soon; instead, we want to solicit feedback on both the high-level + design and implementation, and learn what needs describing better in this + NEP or changing in either the implementation or the array API standard + itself. + + +Motivation and Scope +-------------------- + +Python users have a wealth of choice for libraries and frameworks for +numerical computing, data science, machine learning, and deep learning. New +frameworks pushing forward the state of the art in these fields are appearing +every year. One unintended consequence of all this activity and creativity +has been fragmentation in multidimensional array (a.k.a. tensor) libraries - +which are the fundamental data structure for these fields. Choices include +NumPy, Tensorflow, PyTorch, Dask, JAX, CuPy, MXNet, and others. + +The APIs of each of these libraries are largely similar, but with enough +differences that it’s quite difficult to write code that works with multiple +(or all) of these libraries. The array API standard aims to address that +issue, by specifying an API for the most common ways arrays are constructed +and used. The proposed API is quite similar to NumPy's API, and deviates mainly +in places where (a) NumPy made design choices that are inherently not portable +to other implementations, and (b) where other libraries consistently deviated +from NumPy on purpose because NumPy's design turned out to have issues or +unnecessary complexity. + +For a longer discussion on the purpose of the array API standard we refer to +the `Purpose and Scope section of the array API standard `__ +and the two blog posts announcing the formation of the Consortium [1]_ and +the release of the first draft version of the standard for community review [2]_. + +The scope of this NEP includes: + +- Adopting the 2021 version of the array API standard +- Adding a separate namespace, tentatively named ``numpy.array_api`` +- Changes needed/desired outside of the new namespace, for example new dunder + methods on the ``ndarray`` object +- Implementation choices, and differences between functions in the new + namespace with those in the main ``numpy`` namespace +- A new array object conforming to the array API standard +- Maintenance effort and testing strategy +- Impact on NumPy's total exposed API surface and on other future and + under-discussion design choices +- Relation to existing and proposed NumPy array protocols + (``__array_ufunc__``, ``__array_function__``, ``__array_module__``). +- Required improvements to existing NumPy functionality + +Out of scope for this NEP are: + +- Changes in the array API standard itself. Those are likely to come up + during review of this NEP, but should be upstreamed as needed and this NEP + subsequently updated. + + +Usage and Impact +---------------- + +*This section will be fleshed out later, for now we refer to the use cases given +in* `the array API standard Use Cases section `__ + +In addition to those use cases, the new namespace contains functionality that +is widely used and supported by many array libraries. As such, it is a good +set of functions to teach to newcomers to NumPy and recommend as "best +practice". That contrasts with NumPy's main namespace, which contains many +functions and objects that have been superceded or we consider mistakes - but +that we can't remove because of backwards compatibility reasons. + +The usage of the ``numpy.array_api`` namespace by downstream libraries is +intended to enable them to consume multiple kinds of arrays, *without having +to have a hard dependency on all of those array libraries*: + +.. image:: _static/nep-0047-library-dependencies.png + +Adoption in downstream libraries +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The prototype implementation of the ``array_api`` namespace will be used with +SciPy, scikit-learn and other libraries of interest that depend on NumPy, in +order to get more experience with the design and find out if any important +parts are missing. + +The pattern to support multiple array libraries is intended to be something +like:: + + def somefunc(x, y): + # Retrieves standard namespace. Raises if x and y have different + # namespaces. See Appendix for possible get_namespace implementation + xp = get_namespace(x, y) + out = xp.mean(x, axis=0) + 2*xp.std(y, axis=0) + return out + +The ``get_namespace`` call is effectively the library author opting in to +using the standard API namespace, and thereby explicitly supporting +all conforming array libraries. + + +The ``asarray`` / ``asanyarray`` pattern +```````````````````````````````````````` + +Many existing libraries use the same ``asarray`` (or ``asanyarray``) pattern +as NumPy itself does; accepting any object that can be coerced into a ``np.ndarray``. +We consider this design pattern problematic - keeping in mind the Zen of +Python, *"explicit is better than implicit"*, as well as the pattern being +historically problematic in the SciPy ecosystem for ``ndarray`` subclasses +and with over-eager object creation. All other array/tensor libraries are +more strict, and that works out fine in practice. We would advise authors of +new libraries to avoid the ``asarray`` pattern. Instead they should either +accept just NumPy arrays or, if they want to support multiple kinds of +arrays, check if the incoming array object supports the array API standard +by checking for ``__array_namespace__`` as shown in the example above. + +Existing libraries can do such a check as well, and only call ``asarray`` if +the check fails. This is very similar to the ``__duckarray__`` idea in +:ref:`NEP30`. + + +.. _adoption-application-code: + +Adoption in application code +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The new namespace can be seen by end users as a cleaned up and slimmed down +version of NumPy's main namespace. Encouraging end users to use this +namespace like:: + + import numpy.array_api as xp + + x = xp.linspace(0, 2*xp.pi, num=100) + y = xp.cos(x) + +seems perfectly reasonable, and potentially beneficial - users get offered only +one function for each purpose (the one we consider best-practice), and they +then write code that is more easily portable to other libraries. + + +Backward compatibility +---------------------- + +No deprecations or removals of existing NumPy APIs or other backwards +incompatible changes are proposed. + + +High-level design +----------------- + +The array API standard consists of approximately 120 objects, all of which +have a direct NumPy equivalent. This figure shows what is included at a high level: + +.. image:: _static/nep-0047-scope-of-array-API.png + +The most important changes compared to what NumPy currently offers are: + +- A new array object which: + + - conforms to the casting rules and indexing behaviour specified by the + standard, + - does not have methods other than dunder methods, + - does not support the full range of NumPy indexing behaviour. Advanced + indexing with integers is not supported. Only boolean indexing + with a single (possibly multi-dimensional) boolean array is supported. + An indexing expression that selects a single element returns a 0-D array + rather than a scalar. + +- Functions in the ``array_api`` namespace: + + - do not accept ``array_like`` inputs, only NumPy arrays and Python scalars + - do not support ``__array_ufunc__`` and ``__array_function__``, + - use positional-only and keyword-only parameters in their signatures, + - have inline type annotations, + - may have minor changes to signatures and semantics of individual + functions compared to their equivalents already present in NumPy, + - only support dtype literals, not format strings or other ways of + specifying dtypes + +- DLPack_ support will be added to NumPy, +- New syntax for "device support" will be added, through a ``.device`` + attribute on the new array object, and ``device=`` keywords in array creation + functions in the ``array_api`` namespace, +- Casting rules that differ from those NumPy currently has. Output dtypes can + be derived from input dtypes (i.e. no value-based casting), and 0-D arrays + are treated like >=1-D arrays. +- Not all dtypes NumPy has are part of the standard. Only boolean, signed and + unsigned integers, and floating-point dtypes up to ``float64`` are supported. + Complex dtypes are expected to be added in the next version of the standard. + Extended precision, string, void, object and datetime dtypes, as well as + structured dtypes, are not included. + +Improvements to existing NumPy functionality that are needed include: + +- Add support for stacks of matrices to some functions in ``numpy.linalg`` + that are currently missing such support. +- Add the ``keepdims`` keyword to ``np.argmin`` and ``np.argmax``. +- Add a "never copy" mode to ``np.asarray``. + + +Functions in the ``array_api`` namespace +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's start with an example of a function implementation that shows the most +important differences with the equivalent function in the main namespace:: + + def max(x: array, /, *, + axis: Optional[Union[int, Tuple[int, ...]]] = None, + keepdims: bool = False + ) -> array: + """ + Array API compatible wrapper for :py:func:`np.max `. + """ + return np.max._implementation(x, axis=axis, keepdims=keepdims) + +This function does not accept ``array_like`` inputs, only ``ndarray``. There +are multiple reasons for this. Other array libraries all work like this. +Letting the user do coercion of lists, generators, or other foreign objects +separately results in a cleaner design with less unexpected behaviour. +It's higher-performance - less overhead from ``asarray`` calls. Static typing +is easier. Subclasses will work as expected. And the slight increase in verbosity +because users have to explicitly coerce to ``ndarray`` on rare occasions +seems like a small price to pay. + +This function does not support ``__array_ufunc__`` nor ``__array_function__``. +These protocols serve a similar purpose as the array API standard module itself, +but through a different mechanisms. Because only ``ndarray`` instances are accepted, +dispatching via one of these protocols isn't useful anymore. + +This function uses positional-only parameters in its signature. This makes code +more portable - writing ``max(x=x, ...)`` is no longer valid, hence if other +libraries call the first parameter ``input`` rather than ``x``, that is fine. +The rationale for keyword-only parameters (not shown in the above example) is +two-fold: clarity of end user code, and it being easier to extend the signature +in the future with keywords in the desired order. + +This function has inline type annotations. Inline annotations are far easier to +maintain than separate stub files. And because the types are simple, this will +not result in a large amount of clutter with type aliases or unions like in the +current stub files NumPy has. + + +DLPack support for zero-copy data interchange +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ability to convert one kind of array into another kind is valuable, and +indeed necessary when downstream libraries want to support multiple kinds of +arrays. This requires a well-specified data exchange protocol. NumPy already +supports two of these, namely the buffer protocol (i.e., PEP 3118), and +the ``__array_interface__`` (Python side) / ``__array_struct__`` (C side) +protocol. Both work similarly, letting the "producer" describe how the data +is laid out in memory so the "consumer" can construct its own kind of array +with a view on that data. + +DLPack works in a very similar way. The main reasons to prefer DLPack over +the options already present in NumPy are: + +1. DLPack is the only protocol with device support (e.g., GPUs using CUDA or + ROCm drivers, or OpenCL devices). NumPy is CPU-only, but other array + libraries are not. Having one protocol per device isn't tenable, hence + device support is a must. +2. Widespread support. DLPack has the widest adoption of all protocols, only + NumPy is missing support. And the experiences of other libraries with it + are positive. This contrasts with the protocols NumPy does support, which + are used very little - when other libraries want to interoperate with + NumPy, they typically use the (more limited, and NumPy-specific) + ``__array__`` protocol. + +Adding support for DLPack to NumPy entails: + +- Adding a ``ndarray.__dlpack__`` method +- Adding a ``from_dlpack`` function, which takes as input an object + supporting ``__dlpack__``, and returns an ``ndarray``. + +DLPack is currently a ~200 LoC header, and is meant to be included directly, so +no external dependency is needed. Implementation should be straightforward. + + +Syntax for device support +~~~~~~~~~~~~~~~~~~~~~~~~~ + +NumPy itself is CPU-only, so it clearly doesn't have a need for device support. +However, other libraries (e.g. TensorFlow, PyTorch, JAX, MXNet) support +multiple types of devices: CPU, GPU, TPU, and more exotic hardware. +To write portable code on systems with multiple devices, it's often necessary +to create new arrays on the same device as some other array, or check that +two arrays live on the same device. Hence syntax for that is needed. + +The array object will have a ``.device`` attribute which enables comparing +devices of different arrays (they only should compare equal if both arrays are +from the same library and it's the same hardware device). Furthermore, +``device=`` keywords in array creation functions are needed. For example:: + + def empty(shape: Union[int, Tuple[int, ...]], /, *, + dtype: Optional[dtype] = None, + device: Optional[device] = None) -> array: + """ + Array API compatible wrapper for :py:func:`np.empty `. + """ + return np.empty(shape, dtype=dtype, device=device) + +The implementation for NumPy may be as simple as setting the device attribute to +the string ``'cpu'`` and raising an exception if array creation functions +encounter any other value. + + +Dtypes and casting rules +~~~~~~~~~~~~~~~~~~~~~~~~ + +The supported dtypes in this namespace are boolean, 8/16/32/64-bit signed and +unsigned integer, and 32/64-bit floating-point dtypes. These will be added to +the namespace as dtype literals with the expected names (e.g., ``bool``, +``uint16``, ``float64``). + +The most obvious omissions are the complex dtypes. The rationale for the lack +of complex support in the first version of the array API standard is that several +libraries (PyTorch, MXNet) are still in the process of adding support for +complex dtypes. The next version of the standard is expected to include ``complex64`` +and ``complex128`` (see `this issue `__ +for more details). + +Specifying dtypes to functions, e.g. via the ``dtype=`` keyword, is expected +to only use the dtype literals. Format strings, Python builtin dtypes, or +string representations of the dtype literals are not accepted - this will +improve readability and portability of code at little cost. + +Casting rules are only defined between different dtypes of the same kind. The +rationale for this is that mixed-kind (e.g., integer to floating-point) +casting behavior differs between libraries. NumPy's mixed-kind casting +behavior doesn't need to be changed or restricted, it only needs to be +documented that if users use mixed-kind casting, their code may not be +portable. + +.. image:: _static/nep-0047-casting-rules-lattice.png + +*Type promotion diagram. Promotion between any two types is given by their +join on this lattice. Only the types of participating arrays matter, not +their values. Dashed lines indicate that behaviour for Python scalars is +undefined on overflow. Boolean, integer and floating-point dtypes are not +connected, indicating mixed-kind promotion is undefined.* + +The most important difference between the casting rules in NumPy and in the +array API standard is how scalars and 0-dimensional arrays are handled. In +the standard, array scalars do not exist and 0-dimensional arrays follow the +same casting rules as higher-dimensional arrays. + +See the `Type Promotion Rules section of the array API standard `__ +for more details. + +.. note:: + + It is not clear what the best way is to support the different casting rules + for 0-dimensional arrays and no value-based casting. One option may be to + implement this second set of casting rules, keep them private, mark the + array API functions with a private attribute that says they adhere to + these different rules, and let the casting machinery check whether for + that attribute. + + This needs discussion. + + +Indexing +~~~~~~~~ + +An indexing expression that would return a scalar with ``ndarray``, e.g. +``arr_2d[0, 0]``, will return a 0-D array with the new array object. There are +several reasons for that: array scalars are largely considered a design mistake +which no other array library copied; it works better for non-CPU libraries +(typically arrays can live on the device, scalars live on the host); and it's +simply a consistent design. To get a Python scalar out of a 0-D array, one can +simply use the builtin for the type, e.g. ``float(arr_0d)``. + +The other `indexing modes in the standard `__ +do work largely the same as they do for ``numpy.ndarray``. One noteworthy +difference is that clipping in slice indexing (e.g., ``a[:n]`` where ``n`` is +larger than the size of the first axis) is unspecified behaviour, because +that kind of check can be expensive on accelerators. + +The lack of advanced indexing, and boolean indexing being limited to a single +n-D boolean array, is due to those indexing modes not being suitable for all +types of arrays or JIT compilation. Their absence does not seem to be +problematic; if a user or library author wants to use them, they can do so +through zero-copy conversion to ``numpy.ndarray``. This will signal correctly +to whomever reads the code that it is then NumPy-specific rather than portable +to all conforming array types. + + + +The array object +~~~~~~~~~~~~~~~~ + +The array object in the standard does not have methods other than dunder +methods. The rationale for that is that not all array libraries have methods +on their array object (e.g., TensorFlow does not). It also provides only a +single way of doing something, rather than have functions and methods that +are effectively duplicate. + +Mixing operations that may produce views (e.g., indexing, ``nonzero``) +in combination with mutation (e.g., item or slice assignment) is +`explicitly documented in the standard to not be supported `__. +This cannot easily be prohibited in the array object itself; instead this will +be guidance to the user via documentation. + +The standard current does not prescribe a name for the array object itself. +We propose to simply name it ``ndarray``. This is the most obvious name, and +because of the separate namespace should not clash with ``numpy.ndarray``. + + +Implementation +-------------- + +.. note:: + + This section needs a lot more detail, which will gradually be added when + the implementation progresses. + +A prototype of the ``array_api`` namespace can be found in +https://github.com/data-apis/numpy/tree/array-api/numpy/_array_api. +The docstring in its ``__init__.py`` has notes on completeness of the +implementation. The code for the wrapper functions also contains ``# Note:`` +comments everywhere there is a difference with the NumPy API. +Two important parts that are not implemented yet are the new array object and +DLPack support. Functions may need changes to ensure the changed casting rules +are respected. + +The array object +~~~~~~~~~~~~~~~~ + +Regarding the array object implementation, we plan to start with a regular +Python class that wraps a ``numpy.ndarray`` instance. Attributes and methods +can forward to that wrapped instance, applying input validation and +implementing changed behaviour as needed. + +The casting rules are probably the most challenging part. The in-progress +dtype system refactor (NEPs 40-43) should make implementing the correct casting +behaviour easier - it is already moving away from value-based casting for +example. + + +The dtype objects +~~~~~~~~~~~~~~~~~ + +We must be able to compare dtypes for equality, and expressions like these must +be possible:: + + np.array_api.some_func(..., dtype=x.dtype) + +The above implies it would be nice to have ``np.array_api.float32 == +np.array_api.ndarray(...).dtype``. + +Dtypes should not be assumed to have a class hierarchy by users, however we are +free to implement it with a class hierarchy if that's convenient. We considered +the following options to implement dtype objects: + +1. Alias dtypes to those in the main namespace. E.g., ``np.array_api.float32 = + np.float32``. +2. Make the dtypes instances of ``np.dtype``. E.g., ``np.array_api.float32 = + np.dtype(np.float32)``. +3. Create new singleton classes with only the required methods/attributes + (currently just ``__eq__``). + +It seems like (2) would be easiest from the perspective of interacting with +functions outside the main namespace. And (3) would adhere best to the +standard. + +TBD: the standard does not yet have a good way to inspect properties of a +dtype, to ask questions like "is this an integer dtype?". Perhaps this is easy +enough to do for users, like so:: + + def _get_dtype(dt_or_arr): + return dt_or_arr.dtype if hasattr(dt_or_arr, 'dtype') else dt_or_arr + + def is_floating(dtype_or_array): + dtype = _get_dtype(dtype_or_array) + return dtype in (float32, float64) + + def is_integer(dtype_or_array): + dtype = _get_dtype(dtype_or_array) + return dtype in (uint8, uint16, uint32, uint64, int8, int16, int32, int64) + +However it could make sense to add to the standard. Note that NumPy itself +currently does not have a great for asking such questions, see +`gh-17325 `__. + + +Feedback from downstream library authors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +TODO - this can only be done after trying out some use cases + +Leo Fang (CuPy): *"My impression is for CuPy we could simply take this new array object and s/numpy/cupy"* + + +Related Work +------------ + +:ref:`NEP37` contains a similar mechanism to retrieve a NumPy-like namespace. +In fact, NEP 37 inspired the (slightly simpler) mechanism in the array API +standard. + +Other libraries have adopted large parts of NumPy's API, made changes where +necessary, and documented deviations. See for example +`the jax.numpy documentation `__ +and `Difference between CuPy and NumPy `__. +The array API standard was constructed with the help of such comparisons, only +between many array libraries rather than only between NumPy and one other library. + + +Alternatives +------------ + + + + +Appendix - a possible ``get_namespace`` implementation +------------------------------------------------------ + +The ``get_namespace`` function mentioned in the +:ref:`adoption-application-code` section can be implemented like:: + + def get_namespace(*xs): + # `xs` contains one or more arrays, or possibly Python scalars (accepting + # those is a matter of taste, but doesn't seem unreasonable). + namespaces = { + x.__array_namespace__() if hasattr(x, '__array_namespace__') else None for x in xs if not isinstance(x, (bool, int, float, complex)) + } + + if not namespaces: + # one could special-case np.ndarray above or use np.asarray here if + # older numpy versions need to be supported. + raise ValueError("Unrecognized array input") + + if len(namespaces) != 1: + raise ValueError(f"Multiple namespaces for array inputs: {namespaces}") + + xp, = namespaces + if xp is None: + raise ValueError("The input is not a supported array type") + + return xp + + +Discussion +---------- + +- `First discussion on the mailing list about the array API standard `__ + + +References and Footnotes +------------------------ + +.. _Python array API standard: https://data-apis.github.io/array-api/latest + +.. _Consortium for Python Data API Standards: https://data-apis.org/ + +.. _DLPack: https://github.com/dmlc/dlpack + +.. [1] https://data-apis.org/blog/announcing_the_consortium/ + +.. [2] https://data-apis.org/blog/array_api_standard_release/ + + +Copyright +--------- + +This document has been placed in the public domain. [1]_ diff --git a/doc/neps/nep-0048-spending-project-funds.rst b/doc/neps/nep-0048-spending-project-funds.rst new file mode 100644 index 000000000000..3571eef2d8e6 --- /dev/null +++ b/doc/neps/nep-0048-spending-project-funds.rst @@ -0,0 +1,458 @@ +.. _NEP48: + +===================================== +NEP 48 — Spending NumPy Project Funds +===================================== + +:Author: Ralf Gommers +:Author: Inessa Pawson +:Author: Stefan van der Walt +:Status: Draft +:Type: Informational +:Created: 2021-02-07 +:Resolution: + + +Abstract +-------- + +The NumPy project has historically never received significant **unrestricted** +funding. However, that is starting to change. This NEP aims to provide +guidance about spending NumPy project unrestricted funds by formulating a set +of principles about *what* to pay for and *who* to pay. It will also touch on +how decisions regarding spending funds get made, how funds get administered, +and transparency around these topics. + + +Motivation and Scope +-------------------- + +NumPy is a fiscally sponsored project of NumFOCUS, a 501(c)(3) nonprofit +organization headquartered in Austin, TX. Therefore, for all legal and +accounting matters the NumPy project has to follow the rules and regulations +for US nonprofits. All nonprofit donations are classified into two categories: +**unrestricted funds** which may be used for any legal purpose appropriate +to the organization and **restricted funds**, monies set aside for a +particular purpose (e.g., project, educational program, etc.). + +For the detailed timeline of NumPy funding refer to +:ref:`numpy-funding-history`. + +Since its inception and until 2020, the NumPy project has only spent on the order of +$10,000 USD of funds that were not restricted to a particular program. Project +income of this type has been relying on donations from individuals and, from +mid 2019, recurring monthly contributions from Tidelift. By the end of 2020, +the Tidelift contributions increased to $3,000/month, and there's also a +potential for an increase of donations and grants going directly to the +project. Having a clear set of principles around how to use these funds will +facilitate spending them fairly and effectively. Additionally, it will make it +easier to solicit donations and other contributions. + +A key assumption this NEP makes is that NumPy remains a largely +volunteer-driven project, and that the project funds are not enough to employ +maintainers full-time. If funding increases to the point where that assumption +is no longer true, this NEP should be updated. + +In scope for this NEP are: + +- Principles of spending project funds: what to pay for, and who to pay. +- Describing how NumPy's funds get administered. +- Describing how decisions to spend funds get proposed and made. + +Out of scope for this NEP are: + +- Making any decisions about spending project funds on a specific project or + activity. +- Principles for spending funds that are intended for NumPy development, but + don't fall in the category of NumPy unrestricted funds. This includes most of + the grant funding, which is usually earmarked for certain + activities/deliverables and goes to an Institutional Partner rather than + directly to the NumPy project, and companies or institutions funding specific + features. + *Rationale: As a project, we have no direct control over how this work gets + executed (at least formally, until issues or PRs show up). In some cases, we + may not even know the contributions were funded or done by an employee on + work time. (Whether that's the case or not should not change how we approach + a contribution). For grants though, we do expect the research/project leader + and funded team to align their work with the needs of NumPy and be + receptive to feedback from other NumPy maintainers and contributors.* + + +Principles of spending project funds +------------------------------------ + +NumPy will likely always be a project with many times more volunteer +contributors than funded people. Therefore having those funded people operate +in ways that attract more volunteers and enhance their participation experience +is critical. That key principle motivates many of the more detailed principles +given below for what to pay for and whom to pay. + +The approach for spending funds will be: + +- first figure out what we want to fund, +- then look for a great candidate, +- after that's settled, determine a fair compensation level. + +The next sections go into detail on each of these three points. + +.. _section-what-to-pay-for: + +What to pay for +``````````````` + +1. Pay for things that are important *and* otherwise won't get done. + *Rationale: there is way more to be done than there are funds to do all + those things. So count on interested volunteers or external sponsored work + to do many of those things.* +2. Plan for sustainability. Don't rely on money always being there. +3. Consider potential positive benefits for NumPy maintainers and contributors, + maintainers of other projects, end users, and other stakeholders like + packagers and educators. +4. Think broadly. There's more to a project than code: websites, documentation, + community building, governance - it's all important. +5. For proposed funded work, include paid time for others to review your work + if such review is expected to be significant effort - do not just increase + the load on volunteer maintainers. + *Rationale: we want the effect of spending funds to be positive for + everyone, not just for the people getting paid. This is also a matter of + fairness.* + +When considering development work, principle (1) implies that priority should +be giving to (a) the most boring/painful tasks that no one likes doing, and to +necessary structural changes to the code base that are too large to be done by +a volunteer in a reasonable amount of time. + +There are also many tasks, activities, and projects outside of +development work that are important and could enhance the project or community +- think of, for example, user surveys, translations, outreach, dedicated +mentoring of newcomers, community organizating, website improvements, and +administrative tasks. + +Time of people to perform tasks is also not the only thing that funds can be +used for: expenses for in-person developer meetings or sprints, hosted hardware +for benchmarking or development work, and CI or other software services could +all be good candidates to spend funds on. + +Whom to pay +``````````` + +1. All else being equal, give preference to existing maintainers/contributors. +2. When looking outside of the current team, consider this an opportunity to + make the project more diverse. +3. Pay attention to the following when considering paying someone: + + - the necessary technical or domain-specific skills to execute the tasks, + - communication and self-management skills, + - experience contributing to and working with open source projects. + +It will likely depend on the project/tasks whether there's already a clear best +candidate within the NumPy team, or whether we look for new people to get +involved. Before making any decisions, the decision makers (according to the +NumPy governance document - currently that's the Steering Council) should think +about whether an opportunity should be advertised to give a wider group of +people a chance to apply for it. + +Compensating fairly +``````````````````` + +.. note:: + + This section on compensating fairly will be considered *Draft* even if this + NEP as a whole is accepted. Once we have applied the approach outlined here + at least 2-3 times and we are happy with it, will we remove this note and + consider this section *Accepted*. + +Paying people fairly is a difficult topic, especially when it comes to +distributed teams. Therefore, we will only offer some guidance here. Final +decisions will always have to be considered and approved by the group of people +that bears this responsibility (according to the current NumPy governance +structure, this would be the NumPy Steering Council). + +Discussions on remote employee compensation tend to be dominated by two +narratives: "pay local market rates" and "same work -- same pay". + +We consider them both extreme: + +- "Same work -- same pay" is unfair to people living in locations with a higher + cost of living. For example, the average rent for a single family apartment + can differ by a large factor (from a few hundred dollars to thousands of + dollars per month). +- "Pay local market rates" bakes in existing inequalities between countries + and makes fixed-cost items like a development machine or a holiday trip + abroad relatively harder to afford in locations where market rates are lower. + +We seek to find a middle ground between these two extremes. + +Useful points of reference include companies like GitLab and +Buffer who are transparent about their remuneration policies ([3]_, [4]_), +Google Summer of Code stipends ([5]_), other open source projects that manage +their budget in a transparent manner (e.g., Babel and Webpack on Open +Collective ([6]_, [7]_)), and standard salary comparison sites. + +Since NumPy is a not-for-profit project, we also looked to the nonprofit sector +for guidelines on remuneration policies and compensation levels. Our findings +show that most smaller non-profits tend to pay a median salary/wage. We +recognize merit in this approach: applying candidates are likely to have a +genuine interest in open source, rather than to be motivated purely by +financial incentives. + +Considering all of the above, we will use the following guidelines for +determining compensation: + +1. Aim to compensate people appropriately, up to a level that's expected for + senior engineers or other professionals as applicable. +2. Establish a compensation cap of $125,000 USD that cannot be exceeded even + for the residents from the most expensive/competitive locations ([#f-pay]_). +3. For equivalent work and seniority, a pay differential between locations + should never be more than 2x. + For example, if we pay $110,000 USD to a senior-level developer from New + York, for equivalent work a senior-level developer from South-East Asia + should be paid at least $55,000 USD. To compare locations, we will use + `Numbeo Cost of Living calculator `__ + (or its equivalent). + +Some other considerations: + +- Often, compensated work is offered for a limited amount of hours or fixed + term. In those cases, consider compensation equivalent to a remuneration + package that comes with permanent employment (e.g., one month of work should + be compensated by at most 1/12th of a full-year salary + benefits). +- When comparing rates, an individual contractor should typically make 20% more + than someone who is employed since they have to take care of their benefits + and accounting on their own. +- Some people may be happy with one-off payments towards a particular + deliverable (e.g., "triage all open issues for label X for $x,xxx"). + This should be compensated at a lower rate compared to an individual + contractor. Or they may motivate lower amounts for another reason (e.g., "I + want to receive $x,xxx to hire a cleaner or pay for childcare, to free up + time for work on open source). +- When funding someone's time through their employer, that employer may want to + set the compensation level based on its internal rules (e.g., overhead rates). + Small deviations from the guidelines in this NEP may be needed in such cases, + however they should be within reason. +- It's entirely possible that another strategy rather than paying people for + their time on certain tasks may turn out to be more effective. Anything that + helps the project and community grow and improve is worth considering. +- Transparency helps. If everyone involved is comfortable sharing their + compensation levels with the rest of the team (or better make it public), + it's least likely to be way off the mark for fairness. + +We highly recommend that the individuals involved in decision-making about +hiring and compensation peruse the content of the References section of this +NEP. It offers a lot of helpful advice on this topic. + + +Defining fundable activities and projects +----------------------------------------- + +We'd like to have a broader set of fundable ideas that we will prioritize with +input from NumPy team members and the wider community. All ideas will be +documented on a single wiki page. Anyone may propose an idea. Only members of a +NumPy team may edit the wiki page. + +Each listed idea must meet the following requirements: + +1. It must be clearly scoped: its description must explain the importance to + the project, referencing the NumPy Roadmap if possible, the items to pay for + or activities and deliverables, and why it should be a funded activity (see + :ref:`section-what-to-pay-for`). +2. It must contain the following metadata: title, cost, time duration or effort + estimate, and (if known) names of the team member(s) to execute or coordinate. +3. It must have an assigned priority (low, medium, or high). This discussion + can originate at a NumPy community meeting or on the mailing list. However, + it must be finalized on the mailing list allowing everyone to weigh in. + +If a proposed idea has been assigned a high priority level, a decision on +allocating funding for it will be made on the private NumPy Steering Council +mailing list. *Rationale: these will often involve decisions about individuals, +which is typically hard to do in public. This is the current practice that +seems to be working well.* + +Sometimes, it may be practical to make a single funding decision ad-hoc (e.g., +"Here's a great opportunity plus the right person to execute it right now”). +However, this approach to decision-making should be used rarely. + + +Strategy for spending/saving funds +---------------------------------- + +There is an expectation from NumPy individual, corporate, and institutional +donors that the funds will be used for the benefit of the project and the +community. Therefore, we should spend available funds, thoughtfully, +strategically, and fairly, as they come in. For emergencies, we should keep a +$10,000 - $15,000 USD reserve which could cover, for example, a year of CI and +hosting services, 1-2 months of full-time maintenance work, or contracting a +consultant for a specific need. + + +How project funds get administered +---------------------------------- + +We will first summarize how administering of funds works today, and then +discuss how to make this process more efficient and transparent. + +Currently, the project funds are held by NumFOCUS in a dedicated account. +NumFOCUS has a small accounting team, which produces an account overview as a +set of spreadsheets on a monthly basis. These land in a shared drive, typically +with about a one month delay (e.g., the balance and transactions for February +are available at the end of March), where a few NumPy team members can access +them. Expense claims and invoices are submitted through the NumFOCUS website. +Those then show up in another spreadsheet, where a NumPy team member must +review and approve each of them before payments are made. Following NumPy +bylaws, the NumFOCUS finance subcommittee, consisting of five people, meets +every six months to review all the project related transactions. (In practice, +there have been so few transactions that we skipped some of these meetings.) + +The existing process is time-consuming and error-prone. More transparency and +automation are desirable. + + +Transparency about project funds and in decision making +``````````````````````````````````````````````````````` + +**To discuss: do we want full transparency by publishing our accounts, +transparency to everyone on a NumPy team, or some other level?** + +Ralf: I'd personally like it to be fully transparent, like through Open +Collective, so the whole community can see current balance, income and expenses +paid out at any moment in time. Moving to Open Collective is nontrivial, +however we can publish the data elsewhere for now if we'd want to. +*Note: Google Season of Docs this year requires having an Open Collective +account, so this is likely to happen soon enough.* + +Stefan/Inessa: at least a summary overview should be fully public, and all +transactions should be visible to the Steering Council. Full transparency of +all transactions is probably fine, but not necessary. + +*The options here may be determined by the accounting system and amount of +effort required.* + + +.. _numpy-funding-history: + +NumPy funding – history and current status +------------------------------------------ + +The NumPy project received its first major funding in 2017. For an overview of +the early history of NumPy (and SciPy), including some institutions sponsoring +time for their employees or contractors to work on NumPy, see [1]_ and [2]_. To +date, NumPy has received four grants: + +- Two grants, from the Alfred P. Sloan Foundation and the Gordon and Betty + Moore Foundation respectively, of about $1.3M combined to the Berkeley + Institute of Data Science. Work performed during the period 2017-2020; + PI Stéfan van der Walt. +- Two grants from the Chan Zuckerberg Foundation to NumFOCUS, for a combined + amount of $335k. Work performed during the period 2020-2021; PI's Ralf + Gommers (first grant) and Melissa Mendonça (second grant). + +From 2012 onwards NumPy has been a fiscally sponsored project of NumFOCUS. +Note that fiscal sponsorship doesn't mean NumPy gets funding, rather that it +can receive funds under the umbrella of a nonprofit. See `NumFOCUS Project +Support `__ for more details. + +Only since 2017 has the NumPy website displayed a "Donate" button, and since +2019 the NumPy repositories have had the GitHub Sponsors button. Before that, +it was possible to donate to NumPy on the NumFOCUS website. The sum total of +donations from individuals to NumPy for 2017-2020 was about $6,100. + +From May 2019 onwards, Tidelift has supported NumPy financially as part of +its "managed open source" business model. From May 2019 till July 2020 this was +$1,000/month, and it started steadily growing after that to about $3,000/month +(as of Feb 2021). + +Finally, there has been other incidental project income, for example, some book +royalties from Packt Publishing, GSoC mentoring fees from Google, and +merchandise sales revenue through the NumFOCUS web shop. All of these were +small (two or three figure) amounts. + +This brings the total amount of project income which did not already have a +spending target to about $35,000. Most of that is recent, from Tidelift. +Over the past 1.5 years we spent about $10,000 for work on the new NumPy +website and Sphinx theme. Those spending decisions were made by the NumPy +Steering Council and announced on the mailing list. + +That leaves about $25,000 in available funds at the time of writing, and +that amount is currently growing at a rate of about $3,000/month. + + +Related Work +------------ + +See references. We assume that other open source projects have also developed +guidelines on spending project funds. However, we were unable to find any +examples at the time of writing. + + +Alternatives +------------ + +*Alternative spending strategy*: not having cash reserves. The rationale +being that NumPy is important enough that in a real emergency some person or +entity will likely jump in to help out. This is not a responsible approach to +financial stewardship of the project though. Hence, we decided against it. + + +Discussion +---------- + + + +References and Footnotes +------------------------ + +.. [1] Pauli Virtanen et al., "SciPy 1.0: fundamental algorithms for scientific + computing in Python", https://www.nature.com/articles/s41592-019-0686-2, + 2020 + +.. [2] Charles Harris et al., "Array programming with NumPy", https://www.nature.com/articles/s41586-020-2649-2, 2020 + +.. [3] https://remote.com/blog/remote-compensation + +.. [4] https://about.gitlab.com/company/culture/all-remote/compensation/#how-do-you-decide-how-much-to-pay-people + +.. [5] https://developers.google.com/open-source/gsoc/help/student-stipends + +.. [6] Jurgen Appelo, "Compensation: what is fair?", https://blog.agilityscales.com/compensation-what-is-fair-38a65a822c29, 2016 + +.. [7] Project Include, "Compensating fairly", https://projectinclude.org/compensating_fairly + +.. [#f-pay] This cap is derived from comparing with compensation levels at + other open source projects (e.g., Babel, Webpack, Drupal - all in + the $100,000 -- $125,000 range) and Partner Institutions. + +- Nadia Eghbal, "Roads and Bridges: The Unseen Labor Behind Our Digital + Infrastructure", 2016 +- Nadia Eghbal, "Working in Public: The Making and Maintenance of Open + Source", 2020 +- https://github.com/nayafia/lemonade-stand +- Daniel Oberhaus, `"The Internet Was Built on the Free Labor of Open Source + Developers. Is That Sustainable?" + `_, 2019 +- David Heinemeier Hansson, `"The perils of mixing open source and money" `_, 2013 +- Danny Crichton, `"Open source sustainability" `_, 2018 +- Nadia Eghbal, "Rebuilding the Cathedral", https://www.youtube.com/watch?v=VS6IpvTWwkQ, 2017 +- Nadia Eghbal, "Where money meets open source", https://www.youtube.com/watch?v=bjAinwgvQqc&t=246s, 2017 +- Eileen Uchitelle, ""The unbearable vulnerability of open source", https://www.youtube.com/watch?v=VdwO3LQ56oM, 2017 (the inverted triangle, open source is a funnel) +- Dries Buytaert, "Balancing Makers and Takers to scale and sustain Open Source", https://dri.es/balancing-makers-and-takers-to-scale-and-sustain-open-source, 2019 +- Safia Abdalla, "Beyond Maintenance", https://increment.com/open-source/beyond-maintenance/, 2019 +- Xavier Damman, "Money and Open Source Communities", https://blog.opencollective.com/money-and-open-source-communities/, 2016 +- Aseem Sood, "Let's talk about money", https://blog.opencollective.com/lets-talk-about-money/, 2017 +- Alanna Irving, "Has your open source community raised money? Here's how to spend it.", https://blog.opencollective.com/has-your-open-source-community-raised-money-heres-how-to-spend-it/, 2017 +- Alanna Irving, "Funding open source, how Webpack reached $400k+/year", https://blog.opencollective.com/funding-open-source-how-webpack-reached-400k-year/, 2017 +- Alanna Irving, "Babel's rise to financial sustainability", https://blog.opencollective.com/babels-rise-to-financial-sustainability/, 2019 +- Devon Zuegel, "The city guide to open source", https://www.youtube.com/watch?v=80KTVu6GGSE, 2020 + blog: https://increment.com/open-source/the-city-guide-to-open-source/ + +GitHub Sponsors: + +- https://github.blog/2019-05-23-announcing-github-sponsors-a-new-way-to-contribute-to-open-source/ +- https://github.blog/2020-05-12-github-sponsors-is-out-of-beta-for-sponsored-organizations/ +- https://blog.opencollective.com/on-github-sponsors/, 2019 +- https://blog.opencollective.com/double-the-love/, 2020 +- https://blog.opencollective.com/github-sponsors-for-companies-open-source-collective-for-people/ + + +Copyright +--------- + +This document has been placed in the public domain. diff --git a/doc/neps/nep-0049.rst b/doc/neps/nep-0049.rst new file mode 100644 index 000000000000..743dd2ad6e90 --- /dev/null +++ b/doc/neps/nep-0049.rst @@ -0,0 +1,340 @@ +=================================== +NEP 49 — Data allocation strategies +=================================== + +:Author: Matti Picus +:Status: Draft +:Type: Standards Track +:Created: 2021-04-18 +:Resolution: http://numpy-discussion.10968.n7.nabble.com/NEP-49-Data-allocation-strategies-tt49185.html + + +Abstract +-------- + +The ``numpy.ndarray`` requires additional memory allocations +to hold ``numpy.ndarray.strides``, ``numpy.ndarray.shape`` and +``numpy.ndarray.data`` attributes. These attributes are specially allocated +after creating the python object in ``__new__`` method. + +This NEP proposes a mechanism to override the memory management strategy used +for ``ndarray->data`` with user-provided alternatives. This allocation holds +the data and can be very large. As accessing this data often becomes +a performance bottleneck, custom allocation strategies to guarantee data +alignment or pinning allocations to specialized memory hardware can enable +hardware-specific optimizations. The other allocations remain unchanged. + +Motivation and Scope +-------------------- + +Users may wish to override the internal data memory routines with ones of their +own. Two such use-cases are to ensure data alignment and to pin certain +allocations to certain NUMA cores. This desire for alignment was discussed +multiple times on the mailing list `in 2005`_, and in `issue 5312`_ in 2014, +which led to `PR 5457`_ and more mailing list discussions here_ `and here`_. In +a comment on the issue `from 2017`_, a user described how 64-byte alignment +improved performance by 40x. + +Also related is `issue 14177`_ around the use of ``madvise`` and huge pages on +Linux. + +Various tracing and profiling libraries like filprofiler_ or `electric fence`_ +override ``malloc``. + +The long CPython discussion of `BPO 18835`_ began with discussing the need for +``PyMem_Alloc32`` and ``PyMem_Alloc64``. The early conclusion was that the +cost (of wasted padding) vs. the benifit of aligned memory is best left to the +user, but then evolves into a discussion of various proposals to deal with +memory allocations, including `PEP 445`_ `memory interfaces`_ to +``PyTraceMalloc_Track`` which apparently was explictly added for NumPy. + +Allowing users to implement different strategies via the NumPy C-API will +enable exploration of this rich area of possible optimizations. The intention +is to create a flexible enough interface without burdening normative users. + +.. _`issue 5312`: https://github.com/numpy/numpy/issues/5312 +.. _`from 2017`: https://github.com/numpy/numpy/issues/5312#issuecomment-315234656 +.. _`in 2005`: https://numpy-discussion.scipy.narkive.com/MvmMkJcK/numpy-arrays-data-allocation-and-simd-alignement +.. _`here`: http://numpy-discussion.10968.n7.nabble.com/Aligned-configurable-memory-allocation-td39712.html +.. _`and here`: http://numpy-discussion.10968.n7.nabble.com/Numpy-s-policy-for-releasing-memory-td1533.html +.. _`issue 14177`: https://github.com/numpy/numpy/issues/14177 +.. _`filprofiler`: https://github.com/pythonspeed/filprofiler/blob/master/design/allocator-overrides.md +.. _`electric fence`: https://github.com/boundarydevices/efence +.. _`memory interfaces`: https://docs.python.org/3/c-api/memory.html#customize-memory-allocators +.. _`BPO 18835`: https://bugs.python.org/issue18835 +.. _`PEP 445`: https://www.python.org/dev/peps/pep-0445/ + +Usage and Impact +---------------- + +The new functions can only be accessed via the NumPy C-API. An example is +included later in this NEP. The added ``struct`` will increase the size of the +``ndarray`` object. It is a necessary price to pay for this approach. We +can be reasonably sure that the change in size will have a minimal impact on +end-user code because NumPy version 1.20 already changed the object size. + +The implementation preserves the use of ``PyTraceMalloc_Track`` to track +allocations already present in NumPy. + +Backward compatibility +---------------------- + +The design will not break backward compatibility. Projects that were assigning +to the ``ndarray->data`` pointer were already breaking the current memory +management strategy and should restore +``ndarray->data`` before calling ``Py_DECREF``. As mentioned above, the change +in size should not impact end-users. + +Detailed description +-------------------- + +High level design +================= + +Users who wish to change the NumPy data memory management routines will use +:c:func:`PyDataMem_SetHandler`, which uses a :c:type:`PyDataMem_Handler` +structure to hold pointers to functions used to manage the data memory. + +Since a call to ``PyDataMem_SetHandler`` will change the default functions, but +that function may be called during the lifetime of an ``ndarray`` object, each +``ndarray`` will carry with it the ``PyDataMem_Handler`` struct used at the +time of its instantiation, and these will be used to reallocate or free the +data memory of the instance. Internally NumPy may use ``memcpy`` or ``memset`` +on the pointer to the data memory. + +The name of the handler will be exposed on the python level via a +``numpy.core.multiarray.get_handler_name(arr)`` function. If called as +``numpy.core.multiarray.get_handler_name()`` it will return the name of the +global handler that will be used to allocate data for the next new `ndarrray`. + +NumPy C-API functions +===================== + +.. c:type:: PyDataMem_Handler + + A struct to hold function pointers used to manipulate memory + + .. code-block:: c + + typedef struct { + char name[128]; /* multiple of 64 to keep the struct aligned */ + PyDataMem_AllocFunc *alloc; + PyDataMem_ZeroedAllocFunc *zeroed_alloc; + PyDataMem_FreeFunc *free; + PyDataMem_ReallocFunc *realloc; + } PyDataMem_Handler; + + where the function's signatures are + + .. code-block:: c + + typedef void *(PyDataMem_AllocFunc)(size_t size); + typedef void *(PyDataMem_ZeroedAllocFunc)(size_t nelems, size_t elsize); + typedef void (PyDataMem_FreeFunc)(void *ptr, size_t size); + typedef void *(PyDataMem_ReallocFunc)(void *ptr, size_t size); + +.. c:function:: const PyDataMem_Handler * PyDataMem_SetHandler(PyDataMem_Handler *handler) + + Sets a new allocation policy. If the input value is ``NULL``, will reset + the policy to the default. Returns the previous policy, ``NULL`` if the + previous policy was the default. We wrap the user-provided functions + so they will still call the Python and NumPy memory management callback + hooks. All the function pointers must be filled in, ``NULL`` is not + accepted. + +.. c:function:: const char * PyDataMem_GetHandlerName(PyArrayObject *obj) + + Return the const char name of the ``PyDataMem_Handler`` used by the + ``PyArrayObject``. If ``NULL``, return the name of the current global policy + that will be used to allocate data for the next ``PyArrayObject``. + + +Sample code +=========== + +This code adds a 64-byte header to each ``data`` pointer and stores information +about the allocation in the header. Before calling ``free``, a check ensures +the ``sz`` argument is correct. + +.. code-block:: c + + #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION + #include + NPY_NO_EXPORT void * + + shift_alloc(size_t sz) { + char *real = (char *)malloc(sz + 64); + if (real == NULL) { + return NULL; + } + snprintf(real, 64, "originally allocated %ld", (unsigned long)sz); + return (void *)(real + 64); + } + + NPY_NO_EXPORT void * + shift_zero(size_t sz, size_t cnt) { + char *real = (char *)calloc(sz + 64, cnt); + if (real == NULL) { + return NULL; + } + snprintf(real, 64, "originally allocated %ld via zero", + (unsigned long)sz); + return (void *)(real + 64); + } + + NPY_NO_EXPORT void + shift_free(void * p, npy_uintp sz) { + if (p == NULL) { + return ; + } + char *real = (char *)p - 64; + if (strncmp(real, "originally allocated", 20) != 0) { + fprintf(stdout, "uh-oh, unmatched shift_free, " + "no appropriate prefix\\n"); + /* Make the C runtime crash by calling free on the wrong address */ + free((char *)p + 10); + /* free(real); */ + } + else { + int i = atoi(real +20); + if (i != sz) { + fprintf(stderr, "uh-oh, unmatched " + "shift_free(ptr, %d) but allocated %d\\n", sz, i); + /* Make the C runtime crash by calling free on the wrong address */ + /* free((char *)p + 10); */ + free(real); + } + else { + free(real); + } + } + } + + NPY_NO_EXPORT void * + shift_realloc(void * p, npy_uintp sz) { + if (p != NULL) { + char *real = (char *)p - 64; + if (strncmp(real, "originally allocated", 20) != 0) { + fprintf(stdout, "uh-oh, unmatched shift_realloc\\n"); + return realloc(p, sz); + } + return (void *)((char *)realloc(real, sz + 64) + 64); + } + else { + char *real = (char *)realloc(p, sz + 64); + if (real == NULL) { + return NULL; + } + snprintf(real, 64, "originally allocated " + "%ld via realloc", (unsigned long)sz); + return (void *)(real + 64); + } + } + + static PyDataMem_Handler new_handler = { + "secret_data_allocator", + shift_alloc, /* alloc */ + shift_zero, /* zeroed_alloc */ + shift_free, /* free */ + shift_realloc /* realloc */ + }; + + static PyObject* mem_policy_test_prefix(PyObject *self, PyObject *args) + { + + if (!PyArray_Check(args)) { + PyErr_SetString(PyExc_ValueError, + "must be called with a numpy scalar or ndarray"); + } + return PyUnicode_FromString( + PyDataMem_GetHandlerName((PyArrayObject*)args)); + }; + + static PyObject* mem_policy_set_new_policy(PyObject *self, PyObject *args) + { + + const PyDataMem_Handler *old = PyDataMem_SetHandler(&new_handler); + return PyUnicode_FromString(old->name); + + }; + + static PyObject* mem_policy_set_old_policy(PyObject *self, PyObject *args) + { + + const PyDataMem_Handler *old = PyDataMem_SetHandler(NULL); + return PyUnicode_FromString(old->name); + + }; + + static PyMethodDef methods[] = { + {"test_prefix", (PyCFunction)mem_policy_test_prefix, METH_O}, + {"set_new_policy", (PyCFunction)mem_policy_set_new_policy, METH_NOARGS}, + {"set_old_policy", (PyCFunction)mem_policy_set_old_policy, METH_NOARGS}, + { NULL } + }; + + static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "mem_policy", /* m_name */ + NULL, /* m_doc */ + -1, /* m_size */ + methods, /* m_methods */ + }; + + PyMODINIT_FUNC + PyInit_mem_policy(void) { + PyObject *mod = PyModule_Create(&moduledef); + import_array(); + return mod; + } + + +Related Work +------------ + +This NEP is being tracked by the pnumpy_ project and a `comment in the PR`_ +mentions use in orchestrating FPGA DMAs. + +Implementation +-------------- + +This NEP has been implemented in `PR 17582`_. + +Alternatives +------------ + +These were discussed in `issue 17467`_. `PR 5457`_ and `PR 5470`_ proposed a +global interface for specifying aligned allocations. + +``PyArray_malloc_aligned`` and friends were added to NumPy with the +`numpy.random` module API refactor. and are used there for performance. + +`PR 390`_ had two parts: expose ``PyDataMem_*`` via the NumPy C-API, and a hook +mechanism. The PR was merged with no example code for using these features. + +Discussion +---------- + +Not yet discussed on the mailing list. + + +References and Footnotes +------------------------ + +.. [1] Each NEP must either be explicitly labeled as placed in the public domain (see + this NEP as an example) or licensed under the `Open Publication License`_. + +.. _Open Publication License: https://www.opencontent.org/openpub/ + +.. _`PR 17582`: https://github.com/numpy/numpy/pull/17582 +.. _`PR 5457`: https://github.com/numpy/numpy/pull/5457 +.. _`PR 5470`: https://github.com/numpy/numpy/pull/5470 +.. _`PR 390`: https://github.com/numpy/numpy/pull/390 +.. _`issue 17467`: https://github.com/numpy/numpy/issues/17467 +.. _`comment in the PR`: https://github.com/numpy/numpy/pull/17582#issuecomment-809145547 +.. _pnumpy: https://quansight.github.io/pnumpy/stable/index.html + +Copyright +--------- + +This document has been placed in the public domain. [1]_ diff --git a/doc/neps/roadmap.rst b/doc/neps/roadmap.rst index 40fcbd325222..7e5d1a03b0a8 100644 --- a/doc/neps/roadmap.rst +++ b/doc/neps/roadmap.rst @@ -17,6 +17,12 @@ facilitate interoperability with all such packages, and the code that uses them, may include (among other things) interoperability protocols, better duck typing support and ndarray subclass handling. +The key goal is: *make it easy for code written for NumPy to also work with +other NumPy-like projects.* This will enable GPU support via, e.g, CuPy or JAX, +distributed array support via Dask, and writing special-purpose arrays (either +from scratch, or as a ``numpy.ndarray`` subclass) that work well with SciPy, +scikit-learn and other such packages. + The ``__array_ufunc__`` and ``__array_function__`` protocols are stable, but do not cover the whole API. New protocols for overriding other functionality in NumPy are needed. Work in this area aims to bring to completion one or more @@ -33,56 +39,59 @@ of the API, as discussed in `this section of NEP 37 `__. -Extensibility -------------- - -We aim to make it much easier to extend NumPy. The primary topic here is to -improve the dtype system. - -- Easier custom dtypes: - - - Simplify and/or wrap the current C-API - - More consistent support for dtype metadata - - Support for writing a dtype in Python - -- New string dtype(s): - - - Encoded strings with fixed-width storage (utf8, latin1, ...) and/or - - Variable length strings (could share implementation with dtype=object, - but are explicitly type-checked) - - One of these should probably be the default for text data. The current - behavior on Python 3 is neither efficient nor user friendly. - - Performance ----------- -Improvements to NumPy's performance are important to many users. The primary -topic at the moment is better use of SIMD instructions, also on platforms other -than x86 - see :ref:`NEP38`. +Improvements to NumPy's performance are important to many users. We have +focused this effort on Universal SIMD (see :ref:`NEP38`) intrinsics which +provide nice improvements across various hardware platforms via an abstraction +layer. The infrastructure is in place, and we welcome follow-on PRs to add +SIMD support across all relevant NumPy functions. Other performance improvement ideas include: -- Reducing ufunc and ``__array_function__`` overhead. -- Optimizations in individual functions. - A better story around parallel execution. +- Optimizations in individual functions. +- Reducing ufunc and ``__array_function__`` overhead. Furthermore we would like to improve the benchmarking system, in terms of coverage, easy of use, and publication of the results (now `here `__) as part of the docs or website. -Website and documentation +Documentation and website ------------------------- The NumPy `documentation `__ is of varying quality. The API documentation is in good shape; tutorials and high-level documentation on many topics are missing or outdated. See :ref:`NEP44` for -planned improvements. +planned improvements. Adding more tutorials is underway in the +`numpy-tutorials repo `__. Our website (https://numpy.org) was completely redesigned recently. We aim to -further improve it by adding translations, better Hugo-Sphinx integration via a -new Sphinx theme, and more (see `this tracking issue `__). +further improve it by adding translations, more case studies and other +high-level content, and more (see `this tracking issue `__). + + +Extensibility +------------- + +We aim to make it much easier to extend NumPy. The primary topic here is to +improve the dtype system - see :ref:`NEP41` and related NEPs linked from it. +Concrete goals for the dtype system rewrite are: + +- Easier custom dtypes: + + - Simplify and/or wrap the current C-API + - More consistent support for dtype metadata + - Support for writing a dtype in Python + +- Allow adding (a) new string dtype(s). This could be encoded strings with + fixed-width storage (e.g., ``utf8`` or ``latin1``), and/or a variable length + string dtype. The latter could share an implementation with ``dtype=object``, + but be explicitly type-checked. + One of these should probably be the default for text data. The current + string dtype support is neither efficient nor user friendly. User experience @@ -90,18 +99,17 @@ User experience Type annotations ```````````````` -We aim to add type annotations for all NumPy functionality, so users can use +NumPy 1.20 adds type annotations for most NumPy functionality, so users can use tools like `mypy`_ to type check their code and IDEs can improve their support -for NumPy. The existing type stubs in the `numpy-stubs`_ repository are being -improved and will be moved into the NumPy code base. +for NumPy. Improving those type annotations, for example to support annotating +array shapes and dtypes, is ongoing. Platform support ```````````````` We aim to increase our support for different hardware architectures. This includes adding CI coverage when CI services are available, providing wheels on -PyPI for ARM64 (``aarch64``) and POWER8/9 (``ppc64le``), providing better -build and install documentation, and resolving build issues on other platforms -like AIX. +PyPI for POWER8/9 (``ppc64le``), providing better build and install +documentation, and resolving build issues on other platforms like AIX. Maintenance @@ -126,4 +134,3 @@ Maintenance .. _`mypy`: https://mypy.readthedocs.io -.. _`numpy-stubs`: https://github.com/numpy/numpy-stubs diff --git a/doc/release/upcoming_changes/README.rst b/doc/release/upcoming_changes/README.rst index ff5ca514c3b2..436535ecddbc 100644 --- a/doc/release/upcoming_changes/README.rst +++ b/doc/release/upcoming_changes/README.rst @@ -50,7 +50,7 @@ and double-backticks for code. If you are unsure what pull request type to use, don't hesitate to ask in your PR. -You can install ``towncrier`` and run ``towncrier --draft --version 1.18`` +You can install ``towncrier`` and run ``towncrier build --draft --version 1.18`` if you want to get a preview of how your change will look in the final release notes. diff --git a/doc/source/_static/favicon/apple-touch-icon.png b/doc/source/_static/favicon/apple-touch-icon.png new file mode 100644 index 000000000000..e6cd574260aa Binary files /dev/null and b/doc/source/_static/favicon/apple-touch-icon.png differ diff --git a/doc/source/_static/favicon/favicon-16x16.png b/doc/source/_static/favicon/favicon-16x16.png new file mode 100644 index 000000000000..95beb08342d6 Binary files /dev/null and b/doc/source/_static/favicon/favicon-16x16.png differ diff --git a/doc/source/_static/favicon/favicon-32x32.png b/doc/source/_static/favicon/favicon-32x32.png new file mode 100644 index 000000000000..cc06622fa04b Binary files /dev/null and b/doc/source/_static/favicon/favicon-32x32.png differ diff --git a/doc/source/_static/favicon/favicon.ico b/doc/source/_static/favicon/favicon.ico new file mode 100644 index 000000000000..4ed63bf67ed4 Binary files /dev/null and b/doc/source/_static/favicon/favicon.ico differ diff --git a/doc/source/_static/numpy.css b/doc/source/_static/numpy.css index 22d08cc0dca6..53b610bf13d9 100644 --- a/doc/source/_static/numpy.css +++ b/doc/source/_static/numpy.css @@ -18,14 +18,8 @@ pre, code { } h1 { - font-style: "Lato", sans-serif; + font-family: "Lato", sans-serif; color: #013243; /* warm black */ - font-weight: 700; - letter-spacing: -.04em; - text-align: right; - margin-top: 3rem; - margin-bottom: 4rem; - font-size: 3rem; } diff --git a/doc/source/_static/numpylogo.svg b/doc/source/_static/numpylogo.svg index 5f0dac7007a1..a566851b8699 100644 --- a/doc/source/_static/numpylogo.svg +++ b/doc/source/_static/numpylogo.svg @@ -1,23 +1,23 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/source/_static/scipy-mathjax b/doc/source/_static/scipy-mathjax new file mode 160000 index 000000000000..3d21c58225c0 --- /dev/null +++ b/doc/source/_static/scipy-mathjax @@ -0,0 +1 @@ +Subproject commit 3d21c58225c09243d5a088b1557654d280925e02 diff --git a/doc/source/_templates/autosummary/module.rst b/doc/source/_templates/autosummary/module.rst new file mode 100644 index 000000000000..e1f428d6598e --- /dev/null +++ b/doc/source/_templates/autosummary/module.rst @@ -0,0 +1,40 @@ +{% extends "!autosummary/module.rst" %} + +{# This file is almost the same as the default, but adds :toctree: to the autosummary directives. + The original can be found at `sphinx/ext/autosummary/templates/autosummary/module.rst`. #} + +{% block attributes %} +{% if attributes %} + .. rubric:: Module Attributes + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} +{% endif %} +{% endblock %} + +{% block functions %} +{% if functions %} + .. rubric:: Functions + + .. autosummary:: + :toctree: + {% for item in functions %} + {{ item }} + {%- endfor %} +{% endif %} +{% endblock %} + +{% block classes %} +{% if classes %} + .. rubric:: Classes + + .. autosummary:: + :toctree: + {% for item in classes %} + {{ item }} + {%- endfor %} +{% endif %} +{% endblock %} diff --git a/doc/source/_templates/indexcontent.html b/doc/source/_templates/indexcontent.html index 6dd6bf9b0851..184a3ca64e2c 100644 --- a/doc/source/_templates/indexcontent.html +++ b/doc/source/_templates/indexcontent.html @@ -7,30 +7,29 @@

{{ docstitle|e }}

Welcome! This is the documentation for NumPy {{ release|e }} - {% if last_updated %}, last updated {{ last_updated|e }}{% endif %}. + {%- if last_updated %}, last updated {{ last_updated|e }}{% endif %}.

For users:

What is NumPy?
Who uses it and why

+

NumPy: the absolute beginner's guide

Installation
-

Quickstart
- Aimed at domain experts or people migrating to NumPy

-

Absolute beginner's guide
- Start here for an overview of NumPy features and syntax

-

Tutorials
+

NumPy quickstart
+ Aimed at domain experts or people migrating to NumPy

+

NumPy for MATLAB users
+

NumPy Tutorials
Learn about concepts and submodules

-

How-tos
+

NumPy How Tos
How to do common tasks with NumPy

-

NumPy API reference
- Automatically generated reference documentation

-

Explanations
- In depth explanation of concepts, best practices and techniques

-

F2Py guide
- Documentation for the f2py module (Fortran extensions for Python)

+

NumPy fundamentals
+

Building from source
+

Using NumPy C-API
+

F2PY Users Guide and Reference Manual
+ Documentation for the f2py module (Fortran extensions for Python)

Glossary
- List of the most important terms

+ List of the most important terms

@@ -41,10 +40,8 @@

{{ docstitle|e }}

Contributing to NumPy

Under-the-hood docs
Specialized, in-depth documentation

-

Building and extending the documentation
- How to contribute to this documentation (user and API)

-

The numpydoc docstring guide
- How to write docstrings in the numpydoc format

+

A guide to NumPy documentation
+

Building the NumPy API and reference docs

Benchmarking
benchmarking NumPy

NumPy Enhancement Proposals
diff --git a/doc/source/conf.py b/doc/source/conf.py index 381a01612401..5ba7f70b8ff5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -4,7 +4,7 @@ import sys # Minimum version, enforced by sphinx -needs_sphinx = '2.2.0' +needs_sphinx = '3.2.0' # This is a nasty hack to use platform-agnostic names for types in the @@ -83,22 +83,21 @@ class PyTypeObject(ctypes.Structure): 'matplotlib.sphinxext.plot_directive', 'IPython.sphinxext.ipython_console_highlighting', 'IPython.sphinxext.ipython_directive', - 'sphinx.ext.imgmath', + 'sphinx.ext.mathjax', ] -imgmath_image_format = 'svg' - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' -master_doc = 'contents' +# Will change to `root_doc` in Sphinx 4 +master_doc = 'index' # General substitutions. project = 'NumPy' -copyright = '2008-2020, The SciPy community' +copyright = '2008-2021, The NumPy community' # The default replacements for |version| and |release|, also used in various # other places throughout the built documents. @@ -138,14 +137,19 @@ class PyTypeObject(ctypes.Structure): # output. They are ignored by default. #show_authors = False -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - def setup(app): # add a config value for `ifconfig` directives app.add_config_value('python_version_major', str(sys.version_info.major), 'env') app.add_lexer('NumPyC', NumPyLexer) +# While these objects do have type `module`, the names are aliases for modules +# elsewhere. Sphinx does not support referring to modules by an aliases name, +# so we make the alias look like a "real" module for it. +# If we deemed it desirable, we could in future make these real modules, which +# would make `from numpy.char import split` work. +sys.modules['numpy.char'] = numpy.char +sys.modules['numpy.testing.dec'] = numpy.testing.dec + # ----------------------------------------------------------------------------- # HTML output # ----------------------------------------------------------------------------- @@ -154,7 +158,10 @@ def setup(app): html_logo = '_static/numpylogo.svg' +html_favicon = '_static/favicon/favicon.ico' + html_theme_options = { + "logo_link": "index", "github_url": "https://github.com/numpy/numpy", "twitter_url": "https://twitter.com/numpy_team", } @@ -179,6 +186,8 @@ def setup(app): pngmath_use_preview = True pngmath_dvipng_args = ['-gamma', '1.5', '-D', '96', '-bg', 'Transparent'] +mathjax_path = "scipy-mathjax/MathJax.js?config=scipy-mathjax" + plot_html_show_formats = False plot_html_show_source_link = False @@ -280,11 +289,12 @@ def setup(app): 'neps': ('https://numpy.org/neps', None), 'python': ('https://docs.python.org/dev', None), 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'matplotlib': ('https://matplotlib.org', None), + 'matplotlib': ('https://matplotlib.org/stable', None), 'imageio': ('https://imageio.readthedocs.io/en/stable', None), 'skimage': ('https://scikit-image.org/docs/stable', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable', None), 'scipy-lecture-notes': ('https://scipy-lectures.org', None), + 'pytest': ('https://docs.pytest.org/en/stable', None), } @@ -437,7 +447,7 @@ def linkcode_resolve(domain, info): linespec = "" if 'dev' in numpy.__version__: - return "https://github.com/numpy/numpy/blob/master/numpy/%s%s" % ( + return "https://github.com/numpy/numpy/blob/main/numpy/%s%s" % ( fn, linespec) else: return "https://github.com/numpy/numpy/blob/v%s/numpy/%s%s" % ( diff --git a/doc/source/dev/development_advanced_debugging.rst b/doc/source/dev/development_advanced_debugging.rst new file mode 100644 index 000000000000..fa4014fdbc45 --- /dev/null +++ b/doc/source/dev/development_advanced_debugging.rst @@ -0,0 +1,190 @@ +======================== +Advanced debugging tools +======================== + +If you reached here, you want to dive into, or use, more advanced tooling. +This is usually not necessary for first time contributers and most +day-to-day developement. +These are used more rarely, for example close to a new NumPy release, +or when a large or particular complex change was made. + +Since not all of these tools are used on a regular bases and only available +on some systems, please expect differences, issues, or quirks; +we will be happy to help if you get stuck and appreciate any improvements +or suggestions to these workflows. + + +Finding C errors with additional tooling +######################################## + +Most development will not require more than a typical debugging toolchain +as shown in :ref:`Debugging `. +But for example memory leaks can be particularly subtle or difficult to +narrow down. + +We do not expect any of these tools to be run by most contributors. +However, you can ensure that we can track down such issues more easily easier: + +* Tests should cover all code paths, incluing error paths. +* Try to write short and simple tests. If you have a very complicated test + consider creating an additional simpler test as well. + This can be helpful, because often it is only easy to find which test + triggers an issue and not which line of the test. +* Never use ``np.empty`` if data is read/used. ``valgrind`` will notice this + and report an error. When you do not care about values, you can generate + random values instead. + +This will help us catch any oversights before your change is released +and means you do not have to worry about making reference counting errors, +which can be intimidating. + + +Python debug build for finding memory leaks +=========================================== + +Debug builds of Python are easily available for example on ``debian`` systems, +and can be used on all platforms. +Running a test or terminal is usually as easy as:: + + python3.8d runtests.py + # or + python3.8d runtests.py --ipython + +and were already mentioned in :ref:`Debugging `. + +A Python debug build will help: + +- Find bugs which may otherwise cause random behaviour. + One example is when an object is still used after it has been deleted. + +- Python debug builds allows to check correct reference counting. + This works using the additional commands:: + + sys.gettotalrefcount() + sys.getallocatedblocks() + + +Use together with ``pytest`` +---------------------------- + +Running the test suite only with a debug python build will not find many +errors on its own. An additional advantage of a debug build of Python is that +it allows detecting memory leaks. + +A tool to make this easier is `pytest-leaks`_, which can be installed using ``pip``. +Unfortunately, ``pytest`` itself may leak memory, but good results can usually +(currently) be achieved by removing:: + + @pytest.fixture(autouse=True) + def add_np(doctest_namespace): + doctest_namespace['np'] = numpy + + @pytest.fixture(autouse=True) + def env_setup(monkeypatch): + monkeypatch.setenv('PYTHONHASHSEED', '0') + +from ``numpy/conftest.py`` (This may change with new ``pytest-leaks`` versions +or ``pytest`` updates). + +This allows to run the test suite, or part of it, conveniently:: + + python3.8d runtests.py -t numpy/core/tests/test_multiarray.py -- -R2:3 -s + +where ``-R2:3`` is the ``pytest-leaks`` command (see its documentation), the +``-s`` causes output to print and may be necessary (in some versions captured +output was detected as a leak). + +Note that some tests are known (or even designed) to leak references, we try +to mark them, but expect some false positives. + +.. _pytest-leaks: https://github.com/abalkin/pytest-leaks + +``valgrind`` +============ + +Valgrind is a powerful tool to find certain memory access problems and should +be run on complicated C code. +Basic use of ``valgrind`` usually requires no more than:: + + PYTHONMALLOC=malloc python runtests.py + +where ``PYTHONMALLOC=malloc`` is necessary to avoid false positives from python +itself. +Depending on the system and valgrind version, you may see more false positives. +``valgrind`` supports "suppressions" to ignore some of these, and Python does +have a supression file (and even a compile time option) which may help if you +find it necessary. + +Valgrind helps: + +- Find use of uninitialized variables/memory. + +- Detect memory access violations (reading or writing outside of allocated + memory). + +- Find *many* memory leaks. Note that for *most* leaks the python + debug build approach (and ``pytest-leaks``) is much more sensitive. + The reason is that ``valgrind`` can only detect if memory is definitely + lost. If:: + + dtype = np.dtype(np.int64) + arr.astype(dtype=dtype) + + Has incorrect reference counting for ``dtype``, this is a bug, but valgrind + cannot see it because ``np.dtype(np.int64)`` always returns the same object. + However, not all dtypes are singletons, so this might leak memory for + different input. + In rare cases NumPy uses ``malloc`` and not the Python memory allocators + which are invisible to the Python debug build. + ``malloc`` should normally be avoided, but there are some exceptions + (e.g. the ``PyArray_Dims`` structure is public API and cannot use the + Python allocators.) + +Even though using valgrind for memory leak detection is slow and less sensitive +it can be a convenient: you can run most programs with valgrind without +modification. + +Things to be aware of: + +- Valgrind does not support the numpy ``longdouble``, this means that tests + will fail or be flagged errors that are completely fine. + +- Expect some errors before and after running your NumPy code. + +- Caches can mean that errors (specifically memory leaks) may not be detected + or are only detect at a later, unrelated time. + +A big advantage of valgrind is that it has no requirements aside from valgrind +itself (although you probably want to use debug builds for better tracebacks). + + +Use together with ``pytest`` +---------------------------- +You can run the test suite with valgrind which may be sufficient +when you are only interested in a few tests:: + + PYTHOMMALLOC=malloc valgrind python runtests.py \ + -t numpy/core/tests/test_multiarray.py -- --continue-on-collection-errors + +Note the ``--continue-on-collection-errors``, which is currently necessary due to +missing ``longdouble`` support causing failures (this will usually not be +necessary if you do not run the full test suite). + +If you wish to detect memory leaks you will also require ``--show-leak-kinds=definite`` +and possibly more valgrind options. Just as for ``pytest-leaks`` certain +tests are known to leak cause errors in valgrind and may or may not be marked +as such. + +We have developed `pytest-valgrind`_ which: + +- Reports errors for each test individually + +- Narrows down memory leaks to individual tests (by default valgrind + only checks for memory leaks after a program stops, which is very + cumbersome). + +Please refer to its ``README`` for more information (it includes an example +command for NumPy). + +.. _pytest-valgrind: https://github.com/seberg/pytest-valgrind + diff --git a/doc/source/dev/development_environment.rst b/doc/source/dev/development_environment.rst index cb027c6620f8..665198c69de6 100644 --- a/doc/source/dev/development_environment.rst +++ b/doc/source/dev/development_environment.rst @@ -57,7 +57,8 @@ When using pytest as a target (the default), you can Using ``runtests.py`` is the recommended approach to running tests. There are also a number of alternatives to it, for example in-place -build or installing to a virtualenv. See the FAQ below for details. +build or installing to a virtualenv or a conda environment. See the FAQ below +for details. .. note:: @@ -130,17 +131,27 @@ to see this output, you can run the ``build_src`` stage verbosely:: $ python build build_src -v -Using virtualenvs ------------------ +Using virtual environments +-------------------------- A frequently asked question is "How do I set up a development version of NumPy in parallel to a released version that I use to do my job/research?". One simple way to achieve this is to install the released version in -site-packages, by using a binary installer or pip for example, and set -up the development version in a virtualenv. First install +site-packages, by using pip or conda for example, and set +up the development version in a virtual environment. + +If you use conda, we recommend creating a separate virtual environment for +numpy development using the ``environment.yml`` file in the root of the repo +(this will create the environment and install all development dependencies at +once):: + + $ conda env create -f environment.yml # `mamba` works too for this command + $ conda activate numpy-dev + +If you installed Python some other way than conda, first install `virtualenv`_ (optionally use `virtualenvwrapper`_), then create your -virtualenv (named numpy-dev here) with:: +virtualenv (named ``numpy-dev`` here) with:: $ virtualenv numpy-dev @@ -188,6 +199,35 @@ For more extensive information, see :ref:`testing-guidelines` *Note: do not run the tests from the root directory of your numpy git repo without ``runtests.py``, that will result in strange test errors.* +Running Linting +--------------- +Lint checks can be performed on newly added lines of Python code. + +Install all dependent packages using pip:: + + $ python -m pip install -r linter_requirements.txt + +To run lint checks before committing new code, run:: + + $ python runtests.py --lint uncommitted + +To check all changes in newly added Python code of current branch with target branch, run:: + + $ python runtests.py --lint main + +If there are no errors, the script exits with no message. In case of errors:: + + $ python runtests.py --lint main + ./numpy/core/tests/test_scalarmath.py:34:5: E303 too many blank lines (3) + 1 E303 too many blank lines (3) + +It is advisable to run lint checks before pushing commits to a remote branch +since the linter runs as part of the CI pipeline. + +For more details on Style Guidelines: + + - `Python Style Guide`_ + - `C Style Guide`_ Rebuilding & cleaning the workspace ----------------------------------- @@ -207,6 +247,8 @@ repo, use one of:: $ git reset --hard +.. _debugging: + Debugging --------- @@ -262,6 +304,8 @@ typically packaged as ``python-dbg``) is highly recommended. .. _virtualenvwrapper: http://www.doughellmann.com/projects/virtualenvwrapper/ .. _Waf: https://code.google.com/p/waf/ .. _`match test names using python operators`: https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests +.. _`Python Style Guide`: https://www.python.org/dev/peps/pep-0008/ +.. _`C Style Guide`: https://numpy.org/neps/nep-0045-c_style_guide.html Understanding the code & getting started ---------------------------------------- diff --git a/doc/source/dev/development_gitpod.rst b/doc/source/dev/development_gitpod.rst new file mode 100644 index 000000000000..92cca81fca69 --- /dev/null +++ b/doc/source/dev/development_gitpod.rst @@ -0,0 +1,271 @@ +.. _development-gitpod: + + +Using Gitpod for NumPy development +======================================================= + +This section of the documentation will guide you through: + +* using GitPod for your NumPy development environment +* creating a personal fork of the NumPy repository on GitHub +* a quick tour of Gitpod and VSCode +* working on the NumPy documentation in Gitpod + +Gitpod +------- + +`Gitpod`_ is an open-source platform for automated and ready-to-code +development environments. It enables developers to describe their dev +environment as code and start instant and fresh development environments for +each new task directly from your browser. This reduces the need to install local +development environments and deal with incompatible dependencies. + +Gitpod GitHub integration +-------------------------- + +To be able to use Gitpod, you will need to have the Gitpod app installed on your +GitHub account, so if +you do not have an account yet, you will need to create one first. + +Head over to the `Gitpod`_ website and click on the **Continue with GitHub** +button. You will be redirected to the GitHub authentication page. +You will then be asked to install the `Gitpod GitHub app `_. + +Make sure to select **All repositories** access option to avoid issues with +permissions later on. Click on the green **Install** button + +.. image:: ./gitpod-imgs/installing-gitpod-io.png + :alt: Gitpod repository access and installation screenshot + +This will install the necessary hooks for the integration. + +Forking the NumPy repository +----------------------------- + +The best way to work on NumPy as a contributor is by making a fork of the +repository first. + +#. Browse to the `NumPy repository on GitHub`_ and `create your own fork`_. +#. Browse to your fork. Your fork will have a URL like + https://github.com/melissawm/NumPy, except with your GitHub username in place of ``melissawm``. + +Starting Gitpod +---------------- +Once you have authenticated to Gitpod through GitHub, you can install the +`Gitpod browser extension `_ +which will add a **Gitpod** button next to the **Code** button in the +repository: + +.. image:: ./gitpod-imgs/NumPy-github.png + :alt: NumPy repository with Gitpod button screenshot + +#. If you install the extension - you can click the **Gitpod** button to start + a new workspace. + +#. Alternatively, if you do not want to install the browser extension, you can + visit https://gitpod.io/#https://github.com/USERNAME/NumPy replacing + ``USERNAME`` with your GitHub username. + +#. In both cases, this will open a new tab on your web browser and start + building your development environment. Please note this can take a few + minutes. + +#. Once the build is complete, you will be directed to your workspace, + including the VSCode editor and all the dependencies you need to work on + NumPy. The first time you start your workspace, you will notice that there + might be some actions running. This will ensure that you have a development + version of NumPy installed and that the docs are being pre-built for you. + +#. When your workspace is ready, you can :ref:`test the build` by + entering:: + + $ python runtests.py -v + +``runtests.py`` is another script in the NumPy root directory. It runs a suite +of tests that make sure NumPy is working as it should, and ``-v`` activates the +``--verbose`` option to show all the test output. + +Quick workspace tour +--------------------- +Gitpod uses VSCode as the editor. If you have not used this editor before, you +can check the Getting started `VSCode docs`_ to familiarize yourself with it. + +Your workspace will look similar to the image below: + +.. image:: ./gitpod-imgs/gitpod-workspace.png + :alt: Gitpod workspace screenshot + +.. note:: By default, VSCode initializes with a light theme. You can change to + a dark theme by with the keyboard shortcut :kbd:`Cmd-K Cmd-T` in Mac or + :kbd:`Ctrl-K Ctrl-T` in Linux and Windows. + +We have marked some important sections in the editor: + +#. Your current Python interpreter - by default, this is ``numpy-dev`` and + should be displayed in the status bar and on your terminal. You do not need + to activate the conda environment as this will always be activated for you. +#. Your current branch is always displayed in the status bar. You can also use + this button to change or create branches. +#. GitHub Pull Requests extension - you can use this to work with Pull Requests + from your workspace. +#. Marketplace extensions - we have added some essential extensions to the NumPy + Gitpod. Still, you can also install other extensions or syntax highlighting + themes for your user, and these will be preserved for you. +#. Your workspace directory - by default, it is ``/workspace/numpy``. **Do not + change this** as this is the only directory preserved in Gitpod. + +We have also pre-installed a few tools and VSCode extensions to help with the +development experience: + +* `GitHub CLI `_ +* `VSCode rst extension `_ +* `VSCode Live server extension `_ +* `VSCode Gitlens extension `_ +* `VSCode autodocstrings extension `_ +* `VSCode Git Graph extension `_ + +Development workflow with Gitpod +--------------------------------- +The :ref:`development-workflow` section of this documentation contains +information regarding the NumPy development workflow. Make sure to check this +before working on your contributions. + +When using Gitpod, git is pre configured for you: + +#. You do not need to configure your git username, and email as this should be + done for you as you authenticated through GitHub. You can check the git + configuration with the command ``git config --list`` in your terminal. +#. As you started your workspace from your own NumPy fork, you will by default + have both ``upstream`` and ``origin`` added as remotes. You can verify this by + typing ``git remote`` on your terminal or by clicking on the **branch name** + on the status bar (see image below). + + .. image:: ./gitpod-imgs/NumPy-gitpod-branches.png + :alt: Gitpod workspace branches plugin screenshot + +Rendering the NumPy documentation +---------------------------------- +You can find the detailed documentation on how rendering the documentation with +Sphinx works in the :ref:`howto-build-docs` section. + +The documentation is pre-built during your workspace initialization. So once +this task is completed, you have two main options to render the documentation +in Gitpod. + +Option 1: Using Liveserve +*************************** + +#. View the documentation in ``NumPy/doc/build/html``. You can start with + ``index.html`` and browse, or you can jump straight to the file you're + interested in. +#. To see the rendered version of a page, you can right-click on the ``.html`` + file and click on **Open with Live Serve**. Alternatively, you can open the + file in the editor and click on the **Go live** button on the status bar. + + .. image:: ./gitpod-imgs/vscode-statusbar.png + :alt: Gitpod workspace VSCode start live serve screenshot + +#. A simple browser will open to the right-hand side of the editor. We recommend + closing it and click on the **Open in browser** button in the pop-up. +#. To stop the server click on the **Port: 5500** button on the status bar. + +Option 2: Using the rst extension +*********************************** + +A quick and easy way to see live changes in a ``.rst`` file as you work on it +uses the rst extension with docutils. + +.. note:: This will generate a simple live preview of the document without the + ``html`` theme, and some backlinks might not be added correctly. But it is an + easy and lightweight way to get instant feedback on your work. + +#. Open any of the source documentation files located in ``doc/source`` in the + editor. +#. Open VSCode Command Palette with :kbd:`Cmd-Shift-P` in Mac or + :kbd:`Ctrl-Shift-P` in Linux and Windows. Start typing "restructured" + and choose either "Open preview" or "Open preview to the Side". + + .. image:: ./gitpod-imgs/vscode-rst.png + :alt: Gitpod workspace VSCode open rst screenshot + +#. As you work on the document, you will see a live rendering of it on the editor. + + .. image:: ./gitpod-imgs/rst-rendering.png + :alt: Gitpod workspace VSCode rst rendering screenshot + +If you want to see the final output with the ``html`` theme you will need to +rebuild the docs with ``make html`` and use Live Serve as described in option 1. + +FAQ's and troubleshooting +------------------------- + +How long is my Gitpod workspace kept for? +***************************************** + +Your stopped workspace will be kept for 14 days and deleted afterwards if you do +not use them. + +Can I come back to a previous workspace? +***************************************** + +Yes, let's say you stepped away for a while and you want to carry on working on +your NumPy contributions. You need to visit https://gitpod.io/workspaces and +click on the workspace you want to spin up again. All your changes will be there +as you last left them. + +Can I install additional VSCode extensions? +******************************************* + +Absolutely! Any extensions you installed will be installed in your own workspace +and preserved. + +I registered on Gitpod but I still cannot see a ``Gitpod`` button in my repositories. +************************************************************************************* + +Head to https://gitpod.io/integrations and make sure you are logged in. +Hover over GitHub and click on the three buttons that appear on the right. +Click on edit permissions and make sure you have ``user:email``, +``read:user``, and ``public_repo`` checked. Click on **Update Permissions** +and confirm the changes in the GitHub application page. + +.. image:: ./gitpod-imgs/gitpod-edit-permissions-gh.png + :alt: Gitpod integrations - edit GH permissions screenshot + +How long does my workspace stay active if I'm not using it? +*********************************************************** + +If you keep your workspace open in a browser tab but don't interact with it, +it will shut down after 30 minutes. If you close the browser tab, it will +shut down after 3 minutes. + +My terminal is blank - there is no cursor and it's completely unresponsive +************************************************************************** + +Unfortunately this is a known-issue on Gitpod's side. You can sort this +issue in two ways: + +#. Create a new Gitpod workspace altogether. +#. Head to your `Gitpod dashboard `_ and locate + the running workspace. Hover on it and click on the **three dots menu** + and then click on **Stop**. When the workspace is completely stopped you + can click on its name to restart it again. + +.. image:: ./gitpod-imgs/gitpod-dashboard-stop.png + :alt: Gitpod dashboard and workspace menu screenshot + +I authenticated through GitHub but I still cannot commit to the repository through Gitpod. +****************************************************************************************** + +Head to https://gitpod.io/integrations and make sure you are logged in. +Hover over GitHub and click on the three buttons that appear on the right. +Click on edit permissions and make sure you have ``public_repo`` checked. +Click on **Update Permissions** and confirm the changes in the +GitHub application page. + +.. image:: ./gitpod-imgs/gitpod-edit-permissions-repo.png + :alt: Gitpod integrations - edit GH repository permissions screenshot + +.. _Gitpod: https://www.gitpod.io/ +.. _NumPy repository on GitHub: https://github.com/NumPy/NumPy +.. _create your own fork: https://help.github.com/en/articles/fork-a-repo +.. _VSCode docs: https://code.visualstudio.com/docs/getstarted/tips-and-tricks diff --git a/doc/source/dev/development_workflow.rst b/doc/source/dev/development_workflow.rst index 34535b2f54b6..8c56f6fb2cbc 100644 --- a/doc/source/dev/development_workflow.rst +++ b/doc/source/dev/development_workflow.rst @@ -49,10 +49,10 @@ First, fetch new commits from the ``upstream`` repository: git fetch upstream -Then, create a new branch based on the master branch of the upstream +Then, create a new branch based on the main branch of the upstream repository:: - git checkout -b my-new-feature upstream/master + git checkout -b my-new-feature upstream/main .. _editing-workflow: @@ -147,7 +147,7 @@ In more detail It may be the case that while you were working on your edits, new commits have been added to ``upstream`` that affect your work. In this case, follow the -:ref:`rebasing-on-master` section of this document to apply those changes to +:ref:`rebasing-on-main` section of this document to apply those changes to your branch. .. _writing-the-commit-message: @@ -226,10 +226,10 @@ mailing list may also be useful. -.. _rebasing-on-master: +.. _rebasing-on-main: -Rebasing on master -================== +Rebasing on main +================ This updates your feature branch with changes from the upstream `NumPy github`_ repo. If you do not absolutely need to do this, try to avoid doing @@ -244,8 +244,8 @@ Next, you need to update the feature branch:: git checkout my-new-feature # make a backup in case you mess up git branch tmp my-new-feature - # rebase on upstream master branch - git rebase upstream/master + # rebase on upstream main branch + git rebase upstream/main If you have made changes to files that have changed also upstream, this may generate merge conflicts that you need to resolve. See @@ -258,7 +258,7 @@ Finally, remove the backup branch upon a successful rebase:: .. note:: - Rebasing on master is preferred over merging upstream back to your + Rebasing on main is preferred over merging upstream back to your branch. Using ``git merge`` and ``git pull`` is discouraged when working on feature branches. @@ -325,7 +325,7 @@ Suppose that the commit history looks like this:: 29001ed Add pre-nep for a couple of structured_array_extensions. ... -and ``6ad92e5`` is the last commit in the ``master`` branch. Suppose we +and ``6ad92e5`` is the last commit in the ``main`` branch. Suppose we want to make the following changes: * Rewrite the commit message for ``13d7934`` to something more sensible. @@ -392,7 +392,7 @@ Deleting a branch on github_ :: - git checkout master + git checkout main # delete branch locally git branch -D my-unwanted-branch # delete branch on github @@ -451,25 +451,25 @@ Backporting =========== Backporting is the process of copying new feature/fixes committed in -`numpy/master`_ back to stable release branches. To do this you make a branch +`numpy/main`_ back to stable release branches. To do this you make a branch off the branch you are backporting to, cherry pick the commits you want from -``numpy/master``, and then submit a pull request for the branch containing the +``numpy/main``, and then submit a pull request for the branch containing the backport. 1. First, you need to make the branch you will work on. This needs to be - based on the older version of NumPy (not master):: + based on the older version of NumPy (not main):: # Make a new branch based on numpy/maintenance/1.8.x, # backport-3324 is our new name for the branch. git checkout -b backport-3324 upstream/maintenance/1.8.x -2. Now you need to apply the changes from master to this branch using +2. Now you need to apply the changes from main to this branch using `git cherry-pick`_:: # Update remote git fetch upstream # Check the commit log for commits to cherry pick - git log upstream/master + git log upstream/main # This pull request included commits aa7a047 to c098283 (inclusive) # so you use the .. syntax (for a range of commits), the ^ makes the # range inclusive. @@ -480,7 +480,7 @@ backport. 3. You might run into some conflicts cherry picking here. These are resolved the same way as merge/rebase conflicts. Except here you can - use `git blame`_ to see the difference between master and the + use `git blame`_ to see the difference between main and the backported branch to make sure nothing gets screwed up. 4. Push the new branch to your Github repository:: @@ -488,8 +488,8 @@ backport. git push -u origin backport-3324 5. Finally make a pull request using Github. Make sure it is against the - maintenance branch and not master, Github will usually suggest you - make the pull request against master. + maintenance branch and not main, Github will usually suggest you + make the pull request against main. .. _pushing-to-main: @@ -499,7 +499,7 @@ Pushing changes to the main repo *Requires commit rights to the main NumPy repo.* When you have a set of "ready" changes in a feature branch ready for -NumPy's ``master`` or ``maintenance`` branches, you can push +NumPy's ``main`` or ``maintenance`` branches, you can push them to ``upstream`` as follows: 1. First, merge or rebase on the target branch. @@ -507,23 +507,23 @@ them to ``upstream`` as follows: a) Only a few, unrelated commits then prefer rebasing:: git fetch upstream - git rebase upstream/master + git rebase upstream/main - See :ref:`rebasing-on-master`. + See :ref:`rebasing-on-main`. b) If all of the commits are related, create a merge commit:: git fetch upstream - git merge --no-ff upstream/master + git merge --no-ff upstream/main 2. Check that what you are going to push looks sensible:: - git log -p upstream/master.. + git log -p upstream/main.. git log --oneline --graph 3. Push to upstream:: - git push upstream my-feature-branch:master + git push upstream my-feature-branch:main .. note:: diff --git a/doc/source/dev/gitpod-imgs/NumPy-github.png b/doc/source/dev/gitpod-imgs/NumPy-github.png new file mode 100644 index 000000000000..010b0fc5ea33 Binary files /dev/null and b/doc/source/dev/gitpod-imgs/NumPy-github.png differ diff --git a/doc/source/dev/gitpod-imgs/NumPy-gitpod-branches.png b/doc/source/dev/gitpod-imgs/NumPy-gitpod-branches.png new file mode 100644 index 000000000000..3ee6c5f2022f Binary files /dev/null and b/doc/source/dev/gitpod-imgs/NumPy-gitpod-branches.png differ diff --git a/doc/source/dev/gitpod-imgs/gitpod-dashboard-stop.png b/doc/source/dev/gitpod-imgs/gitpod-dashboard-stop.png new file mode 100644 index 000000000000..40f137745941 Binary files /dev/null and b/doc/source/dev/gitpod-imgs/gitpod-dashboard-stop.png differ diff --git a/doc/source/dev/gitpod-imgs/gitpod-edit-permissions-gh.png b/doc/source/dev/gitpod-imgs/gitpod-edit-permissions-gh.png new file mode 100644 index 000000000000..8955e907a96d Binary files /dev/null and b/doc/source/dev/gitpod-imgs/gitpod-edit-permissions-gh.png differ diff --git a/doc/source/dev/gitpod-imgs/gitpod-edit-permissions-repo.png b/doc/source/dev/gitpod-imgs/gitpod-edit-permissions-repo.png new file mode 100644 index 000000000000..8bfaff81cfb6 Binary files /dev/null and b/doc/source/dev/gitpod-imgs/gitpod-edit-permissions-repo.png differ diff --git a/doc/source/dev/gitpod-imgs/gitpod-workspace.png b/doc/source/dev/gitpod-imgs/gitpod-workspace.png new file mode 100644 index 000000000000..a65c9bd7e152 Binary files /dev/null and b/doc/source/dev/gitpod-imgs/gitpod-workspace.png differ diff --git a/doc/source/dev/gitpod-imgs/installing-gitpod-io.png b/doc/source/dev/gitpod-imgs/installing-gitpod-io.png new file mode 100644 index 000000000000..97319a7293ce Binary files /dev/null and b/doc/source/dev/gitpod-imgs/installing-gitpod-io.png differ diff --git a/doc/source/dev/gitpod-imgs/rst-rendering.png b/doc/source/dev/gitpod-imgs/rst-rendering.png new file mode 100644 index 000000000000..41cc305f3a33 Binary files /dev/null and b/doc/source/dev/gitpod-imgs/rst-rendering.png differ diff --git a/doc/source/dev/gitpod-imgs/vscode-rst.png b/doc/source/dev/gitpod-imgs/vscode-rst.png new file mode 100644 index 000000000000..5b574c115a2b Binary files /dev/null and b/doc/source/dev/gitpod-imgs/vscode-rst.png differ diff --git a/doc/source/dev/gitpod-imgs/vscode-statusbar.png b/doc/source/dev/gitpod-imgs/vscode-statusbar.png new file mode 100644 index 000000000000..3febbcee0ee5 Binary files /dev/null and b/doc/source/dev/gitpod-imgs/vscode-statusbar.png differ diff --git a/doc/source/dev/gitwash/development_setup.rst b/doc/source/dev/gitwash/development_setup.rst index a7e9c28b9611..2be7125da032 100644 --- a/doc/source/dev/gitwash/development_setup.rst +++ b/doc/source/dev/gitwash/development_setup.rst @@ -100,8 +100,8 @@ Make the local copy #. Set up your repository so ``git pull`` pulls from ``upstream`` by default: :: - git config branch.master.remote upstream - git config branch.master.merge refs/heads/master + git config branch.main.remote upstream + git config branch.main.merge refs/heads/main ****************************************************************************** Look it over @@ -109,17 +109,17 @@ Look it over #. The branches shown by ``git branch -a`` will include - - the ``master`` branch you just cloned on your own machine - - the ``master`` branch from your fork on GitHub, which git named + - the ``main`` branch you just cloned on your own machine + - the ``main`` branch from your fork on GitHub, which git named ``origin`` by default - - the ``master`` branch on the the main NumPy repo, which you named + - the ``main`` branch on the the main NumPy repo, which you named ``upstream``. :: - master - remotes/origin/master - remotes/upstream/master + main + remotes/origin/main + remotes/upstream/main If ``upstream`` isn't there, it will be added after you access the NumPy repo with a command like ``git fetch`` or ``git pull``. @@ -139,8 +139,8 @@ Look it over user.name=Your Name remote.origin.url=git@github.com:your-github-id/numpy.git remote.origin.fetch=+refs/heads/*:refs/remotes/origin/* - branch.master.remote=upstream - branch.master.merge=refs/heads/master + branch.main.remote=upstream + branch.main.merge=refs/heads/main remote.upstream.url=https://github.com/numpy/numpy.git remote.upstream.fetch=+refs/heads/*:refs/remotes/upstream/* @@ -165,7 +165,7 @@ slightly. Instead of :: run :: - git clone git@github.com:numpy/numpy.git + git clone git@github.com:your-user-name/numpy.git and instead of showing an ``https`` URL, ``git remote -v`` will show :: diff --git a/doc/source/dev/gitwash/dot2_dot3.rst b/doc/source/dev/gitwash/dot2_dot3.rst index 7759e2e60d68..30852b5ad387 100644 --- a/doc/source/dev/gitwash/dot2_dot3.rst +++ b/doc/source/dev/gitwash/dot2_dot3.rst @@ -7,22 +7,22 @@ Thanks to Yarik Halchenko for this explanation. Imagine a series of commits A, B, C, D... Imagine that there are two -branches, *topic* and *master*. You branched *topic* off *master* when -*master* was at commit 'E'. The graph of the commits looks like this:: +branches, *topic* and *main*. You branched *topic* off *main* when +*main* was at commit 'E'. The graph of the commits looks like this:: A---B---C topic / - D---E---F---G master + D---E---F---G main Then:: - git diff master..topic + git diff main..topic will output the difference from G to C (i.e. with effects of F and G), while:: - git diff master...topic + git diff main...topic would output just differences in the topic branch (i.e. only A, B, and C). diff --git a/doc/source/dev/gitwash/git_links.inc b/doc/source/dev/gitwash/git_links.inc index 8032dca41e97..8126cf9ac6cf 100644 --- a/doc/source/dev/gitwash/git_links.inc +++ b/doc/source/dev/gitwash/git_links.inc @@ -47,7 +47,7 @@ .. _ipython git workflow: https://mail.python.org/pipermail/ipython-dev/2010-October/005632.html .. _git parable: http://tom.preston-werner.com/2009/05/19/the-git-parable.html .. _git foundation: http://matthew-brett.github.com/pydagogue/foundation.html -.. _numpy/master: https://github.com/numpy/numpy +.. _numpy/main: https://github.com/numpy/numpy .. _git cherry-pick: https://www.kernel.org/pub/software/scm/git/docs/git-cherry-pick.html .. _git blame: https://www.kernel.org/pub/software/scm/git/docs/git-blame.html .. _this blog post: https://github.com/blog/612-introducing-github-compare-view diff --git a/doc/source/dev/governance/governance.rst b/doc/source/dev/governance/governance.rst index 9209f26b7bd5..8c9cc0825eb0 100644 --- a/doc/source/dev/governance/governance.rst +++ b/doc/source/dev/governance/governance.rst @@ -381,7 +381,7 @@ A list of current Institutional Partners is maintained at the page Document history ================ -https://github.com/numpy/numpy/commits/master/doc/source/dev/governance/governance.rst +https://github.com/numpy/numpy/commits/main/doc/source/dev/governance/governance.rst Acknowledgements ================ diff --git a/doc/source/dev/governance/people.rst b/doc/source/dev/governance/people.rst index 18366402e2be..c48b801ab670 100644 --- a/doc/source/dev/governance/people.rst +++ b/doc/source/dev/governance/people.rst @@ -7,59 +7,43 @@ Steering council ---------------- * Sebastian Berg - -* Jaime Fernández del Río - * Ralf Gommers - -* Allan Haldane - * Charles Harris - * Stephan Hoyer - +* Melissa Weber Mendonça +* Inessa Pawson * Matti Picus - -* Nathaniel Smith - -* Julian Taylor - -* Pauli Virtanen - * Stéfan van der Walt - * Eric Wieser - Emeritus members ---------------- * Travis Oliphant -- project founder / emeritus leader (2005-2012) - * Alex Griffing (2015-2017) - * Marten van Kerkwijk (2017-2019) +* Allan Haldane (2015-2021) +* Nathaniel Smith (2012-2021) +* Julian Taylor (2013-2021) +* Pauli Virtanen (2008-2021) +* Jaime Fernández del Río (2014-2021) NumFOCUS Subcommittee --------------------- -* Chuck Harris - +* Charles Harris * Ralf Gommers - -* Jaime Fernández del Río - +* Melissa Weber Mendonça * Sebastian Berg - * External member: Thomas Caswell Institutional Partners ---------------------- -* UC Berkeley (Stéfan van der Walt, Sebastian Berg, Warren Weckesser, Ross Barnowski) +* UC Berkeley (Stéfan van der Walt, Sebastian Berg, Ross Barnowski) -* Quansight (Ralf Gommers, Hameer Abbasi, Melissa Weber Mendonça, Mars Lee, Matti Picus) +* Quansight (Ralf Gommers, Melissa Weber Mendonça, Mars Lee, Matti Picus, Pearu Peterson) diff --git a/doc/source/dev/howto-docs.rst b/doc/source/dev/howto-docs.rst index 17194bd58a99..3687d7550ebe 100644 --- a/doc/source/dev/howto-docs.rst +++ b/doc/source/dev/howto-docs.rst @@ -7,9 +7,9 @@ How to contribute to the NumPy documentation This guide will help you decide what to contribute and how to submit it to the official NumPy documentation. -****************************************************************************** +*************************** Documentation team meetings -****************************************************************************** +*************************** The NumPy community has set a firm goal of improving its documentation. We hold regular documentation meetings on Zoom (dates are announced on the @@ -21,20 +21,23 @@ Minutes are taken `on hackmd.io `__ and stored in the `NumPy Archive repository `__. -************************* +************* What's needed -************************* -NumPy docs have the details covered. API reference -documentation is generated directly from -`docstrings `_ in the code -when the documentation is :ref:`built`. +************* -What we lack are docs with broader scope -- tutorials, how-tos, and explanations. -Reporting defects is another way to contribute. We discuss both. +The :ref:`NumPy Documentation ` has the details covered. +API reference documentation is generated directly from +`docstrings `_ in the code when the +documentation is :ref:`built`. Although we have mostly +complete reference documentation for each function and class exposed to users, +there is a lack of usage examples for some of them. -************************* +What we lack are docs with broader scope -- tutorials, how-tos, and +explanations. Reporting defects is another way to contribute. We discuss both. + +****************** Contributing fixes -************************* +****************** We're eager to hear about and fix doc defects. But to attack the biggest problems we end up having to defer or overlook some bug reports. Here are the @@ -48,17 +51,17 @@ a `pull request (PR) `__ with the fix, if you know how to do that; otherwise please `open an issue `__. -**Typos and misspellings** fall on a lower rung; we welcome hearing about them but -may not be able to fix them promptly. These too can be handled as pull +**Typos and misspellings** fall on a lower rung; we welcome hearing about them +but may not be able to fix them promptly. These too can be handled as pull requests or issues. Obvious **wording** mistakes (like leaving out a "not") fall into the typo category, but other rewordings -- even for grammar -- require a judgment call, which raises the bar. Test the waters by first presenting the fix as an issue. -****************************************************************************** +********************** Contributing new pages -****************************************************************************** +********************** Your frustrations using our documents are our best guide to what needs fixing. @@ -74,31 +77,39 @@ If you're looking for subjects, our formal roadmap for documentation is a *NumPy Enhancement Proposal (NEP)*, `NEP 44 - Restructuring the NumPy Documentation `__. It identifies areas where our docs need help and lists several -additions we'd like to see, including Jupyter notebooks. - -You can find larger planned and in-progress ideas `at -our GitHub project `__. +additions we'd like to see, including :ref:`Jupyter notebooks `. .. _tutorials_howtos_explanations: +Documentation framework +======================= -Formula writing -============================================================================== There are formulas for writing useful documents, and four formulas cover nearly everything. There are four formulas because there are four categories of document -- ``tutorial``, ``how-to guide``, ``explanation``, and ``reference``. The insight that docs divide up this way belongs to -Daniele Procida, who goes on -`in this short article `__ to explain -the differences and reveal the formulas. When you begin a document or -propose one, have in mind which of these types it will be. +Daniele Procida and his `Diátaxis Framework `__. When you +begin a document or propose one, have in mind which of these types it will be. +.. _numpy_tutorials: -.. _contributing: +NumPy tutorials +=============== +In addition to the documentation that is part of the NumPy source tree, you can +submit content in Jupyter Notebook format to the +`NumPy Tutorials `__ page. This +set of tutorials and educational materials is meant to provide high-quality +resources by the NumPy project, both for self-learning and for teaching classes +with. These resources are developed in a separate GitHub repository, +`numpy-tutorials `__, where you can +check out existing notebooks, open issues to suggest new topics or submit your +own tutorials as pull requests. + +.. _contributing: More on contributing -============================================================================== +==================== Don't worry if English is not your first language, or if you can only come up with a rough draft. Open source is a community effort. Do your best -- we'll @@ -126,9 +137,9 @@ rST, see the `Quick reStructuredText Guide `__ -************************************************************ +*********************** Contributing indirectly -************************************************************ +*********************** If you run across outside material that would be a useful addition to the NumPy docs, let us know by `opening an issue `__. @@ -138,9 +149,9 @@ if you write a tutorial on your blog, create a YouTube video, or answer question on Stack Overflow and other sites. -************************************************************ +********************* Documentation reading -************************************************************ +********************* - The leading organization of technical writers, `Write the Docs `__, diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 4641a7e2fff6..aaf9fe4a091c 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -4,21 +4,6 @@ Contributing to NumPy ##################### -.. TODO: this is hidden because there's a bug in the pydata theme that won't render TOC items under headers - -.. toctree:: - :hidden: - - Git Basics - development_environment - development_workflow - ../benchmarking - NumPy C style guide - releasing - governance/index - howto-docs - - Not a coder? Not a problem! NumPy is multi-faceted, and we can use a lot of help. These are all activities we'd like to get help with (they're all important, so we list them in alphabetical order): @@ -77,8 +62,8 @@ Here's the short summary, complete TOC links are below: * Pull the latest changes from upstream:: - git checkout master - git pull upstream master + git checkout main + git pull upstream main * Create a branch for the feature you want to work on. Since the branch name will appear in the merge message, use a sensible name @@ -172,13 +157,13 @@ Here's the short summary, complete TOC links are below: For a more detailed discussion, read on and follow the links at the bottom of this page. -Divergence between ``upstream/master`` and your feature branch --------------------------------------------------------------- +Divergence between ``upstream/main`` and your feature branch +------------------------------------------------------------ If GitHub indicates that the branch of your Pull Request can no longer be merged automatically, you have to incorporate changes that have been made since you started into your branch. Our recommended way to do this is to -:ref:`rebase on master`. +:ref:`rebase on main `. .. _guidelines: @@ -301,7 +286,9 @@ The rest of the story Git Basics development_environment + development_gitpod development_workflow + development_advanced_debugging reviewer_guidelines ../benchmarking NumPy C style guide diff --git a/doc/source/dev/releasing.rst b/doc/source/dev/releasing.rst index 61fa1951426c..75081aec1675 100644 --- a/doc/source/dev/releasing.rst +++ b/doc/source/dev/releasing.rst @@ -1,5 +1,5 @@ =================== -Releasing a Version +Releasing a version =================== ------------------------ diff --git a/doc/source/dev/reviewer_guidelines.rst b/doc/source/dev/reviewer_guidelines.rst index 0b225b9b6b14..ffac85f7720a 100644 --- a/doc/source/dev/reviewer_guidelines.rst +++ b/doc/source/dev/reviewer_guidelines.rst @@ -37,9 +37,8 @@ Communication Guidelines style or grammar, consider merging the current PR when all important concerns are addressed. Then, either push a commit directly (if you are a maintainer) or open a follow-up PR yourself. -- If you need help writing replies in reviews, check out some `Standard replies - for reviewing - `_. +- If you need help writing replies in reviews, check out some + :ref:`standard replies for reviewing`. Reviewer Checklist ================== @@ -74,8 +73,8 @@ For maintainers - Make sure all automated CI tests pass before merging a PR, and that the :ref:`documentation builds ` without any errors. -- In case of merge conflicts, ask the PR submitter to :ref:`rebase on master - `. +- In case of merge conflicts, ask the PR submitter to :ref:`rebase on main + `. - For PRs that add new features or are in some way complex, wait at least a day or two before merging it. That way, others get a chance to comment before the code goes in. Consider adding it to the release notes. @@ -116,4 +115,67 @@ the PR page. Assuming you have your :ref:`development environment` set up, you can now build the code and test it. +.. _saved-replies: + +Standard replies for reviewing +============================== + +It may be helpful to store some of these in GitHub's `saved +replies `_ for reviewing: + +**Usage question** + .. code-block:: md + + You are asking a usage question. The issue tracker is for bugs and new features. + I'm going to close this issue, feel free to ask for help via our [help channels](https://numpy.org/gethelp/). + +**You’re welcome to update the docs** + .. code-block:: md + + Please feel free to offer a pull request updating the documentation if you feel it could be improved. + +**Self-contained example for bug** + .. code-block:: md + + Please provide a [self-contained example code](https://stackoverflow.com/help/mcve), including imports and data (if possible), so that other contributors can just run it and reproduce your issue. + Ideally your example code should be minimal. + +**Software versions** + .. code-block:: md + + To help diagnose your issue, please paste the output of: + ``` + python -c 'import numpy; print(numpy.version.version)' + ``` + Thanks. + +**Code blocks** + .. code-block:: md + + Readability can be greatly improved if you [format](https://help.github.com/articles/creating-and-highlighting-code-blocks/) your code snippets and complete error messages appropriately. + You can edit your issue descriptions and comments at any time to improve readability. + This helps maintainers a lot. Thanks! + +**Linking to code** + .. code-block:: md + + For clarity's sake, you can link to code like [this](https://help.github.com/articles/creating-a-permanent-link-to-a-code-snippet/). + +**Better description and title** + .. code-block:: md + + Please make the title of the PR more descriptive. + The title will become the commit message when this is merged. + You should state what issue (or PR) it fixes/resolves in the description using the syntax described [here](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword). + +**Regression test needed** + .. code-block:: md + + Please add a [non-regression test](https://en.wikipedia.org/wiki/Non-regression_testing) that would fail at main but pass in this PR. + +**Don’t change unrelated** + .. code-block:: md + + Please do not change unrelated lines. It makes your contribution harder to review and may introduce merge conflicts to other pull requests. + .. include:: gitwash/git_links.inc diff --git a/doc/source/docs/howto_build_docs.rst b/doc/source/docs/howto_build_docs.rst index 29912a5af8f1..38ea1338c2a1 100644 --- a/doc/source/docs/howto_build_docs.rst +++ b/doc/source/docs/howto_build_docs.rst @@ -48,6 +48,7 @@ with:: Now you are ready to generate the docs, so write:: + cd doc make html in the ``doc/`` directory. If all goes well, this will generate a @@ -55,6 +56,14 @@ in the ``doc/`` directory. If all goes well, this will generate a a message about ``installed numpy != current repo git version``, you must either override the check by setting ``GITVER`` or re-install NumPy. +If you have built numpy into a virtual environment and get an error +that says ``numpy not found, cannot build documentation without...``, +you need to override the makefile ``PYTHON`` variable at the command +line, so instead of writing ``make html`` write:: + + make PYTHON=python html + + Note that building the documentation on Windows is currently not actively supported, though it should be possible. (See Sphinx_ documentation for more information.) @@ -81,7 +90,7 @@ pdf format is also built with ``make dist``. See `HOWTO RELEASE`_ for details on how to update https://numpy.org/doc. .. _Matplotlib: https://matplotlib.org/ -.. _HOWTO RELEASE: https://github.com/numpy/numpy/blob/master/doc/HOWTO_RELEASE.rst.txt +.. _HOWTO RELEASE: https://github.com/numpy/numpy/blob/main/doc/HOWTO_RELEASE.rst.txt Sphinx extensions ----------------- diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst index 17071c8f175f..aa2dc13dff1f 100644 --- a/doc/source/glossary.rst +++ b/doc/source/glossary.rst @@ -166,9 +166,10 @@ Glossary array scalar - For uniformity in handling operands, NumPy treats - a :doc:`scalar ` as an array of zero - dimension. + An :doc:`array scalar ` is an instance of the types/classes float32, float64, + etc.. For uniformity in handling operands, NumPy treats a scalar as + an array of zero dimension. In contrast, a 0-dimensional array is an :doc:`ndarray ` instance + containing precisely one value. axis @@ -502,7 +503,7 @@ Glossary Some NumPy routines always return views, some always return copies, some may return one or the other, and for some the choice can be specified. - Responsiblity for managing views and copies falls to the programmer. + Responsibility for managing views and copies falls to the programmer. :func:`numpy.shares_memory` will check whether ``b`` is a view of ``a``, but an exact answer isn't always feasible, as the documentation page explains. diff --git a/doc/source/contents.rst b/doc/source/index.rst similarity index 96% rename from doc/source/contents.rst rename to doc/source/index.rst index 5d4e1209772a..21dec00fe5a9 100644 --- a/doc/source/contents.rst +++ b/doc/source/index.rst @@ -6,6 +6,7 @@ NumPy Documentation .. toctree:: :maxdepth: 1 + :hidden: User Guide API reference diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 3a4ed21689e5..92c271f6b964 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -330,6 +330,8 @@ NumPy provides several hooks that classes can customize: returned by :func:`__array__`. This practice will return ``TypeError``. +.. _matrix-objects: + Matrix objects ============== diff --git a/doc/source/reference/arrays.datetime.rst b/doc/source/reference/arrays.datetime.rst index c5947620efe7..e3b8d270dbc4 100644 --- a/doc/source/reference/arrays.datetime.rst +++ b/doc/source/reference/arrays.datetime.rst @@ -13,16 +13,15 @@ support datetime functionality. The data type is called "datetime64", so named because "datetime" is already taken by the datetime library included in Python. -.. note:: The datetime API is *experimental* in 1.7.0, and may undergo changes - in future versions of NumPy. Basic Datetimes =============== -The most basic way to create datetimes is from strings in -ISO 8601 date or datetime format. The unit for internal storage -is automatically selected from the form of the string, and can -be either a :ref:`date unit ` or a +The most basic way to create datetimes is from strings in ISO 8601 date +or datetime format. It is also possible to create datetimes from an integer by +offset relative to the Unix epoch (00:00:00 UTC on 1 January 1970). +The unit for internal storage is automatically selected from the +form of the string, and can be either a :ref:`date unit ` or a :ref:`time unit `. The date units are years ('Y'), months ('M'), weeks ('W'), and days ('D'), while the time units are hours ('h'), minutes ('m'), seconds ('s'), milliseconds ('ms'), and @@ -36,6 +35,11 @@ letters, for a "Not A Time" value. >>> np.datetime64('2005-02-25') numpy.datetime64('2005-02-25') + + From an integer and a date unit, 1 year since the UNIX epoch: + + >>> np.datetime64(1, 'Y') + numpy.datetime64('1971') Using months for the unit: diff --git a/doc/source/reference/arrays.ndarray.rst b/doc/source/reference/arrays.ndarray.rst index 191367058dba..f2204752df83 100644 --- a/doc/source/reference/arrays.ndarray.rst +++ b/doc/source/reference/arrays.ndarray.rst @@ -567,10 +567,8 @@ Matrix Multiplication: .. note:: Matrix operators ``@`` and ``@=`` were introduced in Python 3.5 - following PEP465. NumPy 1.10.0 has a preliminary implementation of ``@`` - for testing purposes. Further documentation can be found in the - :func:`matmul` documentation. - + following :pep:`465`, and the ``@`` operator has been introduced in NumPy + 1.10.0. Further information can be found in the :func:`matmul` documentation. Special methods =============== diff --git a/doc/source/reference/arrays.scalars.rst b/doc/source/reference/arrays.scalars.rst index 4b5da2e13a5c..6c9bcb504531 100644 --- a/doc/source/reference/arrays.scalars.rst +++ b/doc/source/reference/arrays.scalars.rst @@ -94,112 +94,180 @@ Python Boolean scalar. .. tip:: The default data type in NumPy is :class:`float_`. .. autoclass:: numpy.generic - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.number - :exclude-members: + :members: __init__ + :exclude-members: __init__ Integer types ~~~~~~~~~~~~~ .. autoclass:: numpy.integer - :exclude-members: + :members: __init__ + :exclude-members: __init__ + +.. note:: + + The numpy integer types mirror the behavior of C integers, and can therefore + be subject to :ref:`overflow-errors`. Signed integer types ++++++++++++++++++++ .. autoclass:: numpy.signedinteger - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.byte - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.short - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.intc - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.int_ - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.longlong - :exclude-members: + :members: __init__ + :exclude-members: __init__ Unsigned integer types ++++++++++++++++++++++ .. autoclass:: numpy.unsignedinteger - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.ubyte - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.ushort - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.uintc - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.uint - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.ulonglong - :exclude-members: + :members: __init__ + :exclude-members: __init__ Inexact types ~~~~~~~~~~~~~ .. autoclass:: numpy.inexact - :exclude-members: + :members: __init__ + :exclude-members: __init__ + +.. note:: + + Inexact scalars are printed using the fewest decimal digits needed to + distinguish their value from other values of the same datatype, + by judicious rounding. See the ``unique`` parameter of + `format_float_positional` and `format_float_scientific`. + + This means that variables with equal binary values but whose datatypes are of + different precisions may display differently:: + + >>> f16 = np.float16("0.1") + >>> f32 = np.float32(f16) + >>> f64 = np.float64(f32) + >>> f16 == f32 == f64 + True + >>> f16, f32, f64 + (0.1, 0.099975586, 0.0999755859375) + + Note that none of these floats hold the exact value :math:`\frac{1}{10}`; + ``f16`` prints as ``0.1`` because it is as close to that value as possible, + whereas the other types do not as they have more precision and therefore have + closer values. + + Conversely, floating-point scalars of different precisions which approximate + the same decimal value may compare unequal despite printing identically: + + >>> f16 = np.float16("0.1") + >>> f32 = np.float32("0.1") + >>> f64 = np.float64("0.1") + >>> f16 == f32 == f64 + False + >>> f16, f32, f64 + (0.1, 0.1, 0.1) Floating-point types ++++++++++++++++++++ .. autoclass:: numpy.floating - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.half - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.single - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.double - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.longdouble - :exclude-members: + :members: __init__ + :exclude-members: __init__ Complex floating-point types ++++++++++++++++++++++++++++ .. autoclass:: numpy.complexfloating - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.csingle - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.cdouble - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.clongdouble - :exclude-members: + :members: __init__ + :exclude-members: __init__ Other types ~~~~~~~~~~~ .. autoclass:: numpy.bool_ - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.datetime64 - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.timedelta64 - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.object_ - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. note:: @@ -222,16 +290,20 @@ arrays. (In the character codes ``#`` is an integer denoting how many elements the data type consists of.) .. autoclass:: numpy.flexible - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.bytes_ - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.str_ - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. autoclass:: numpy.void - :exclude-members: + :members: __init__ + :exclude-members: __init__ .. warning:: @@ -246,6 +318,8 @@ elements the data type consists of.) convention more consistent with other Python modules such as the :mod:`struct` module. +.. _sized-aliases: + Sized aliases ~~~~~~~~~~~~~ diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 3aa541b79e64..1673f1d6b1f7 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -22,8 +22,8 @@ Array structure and data access These macros access the :c:type:`PyArrayObject` structure members and are defined in ``ndarraytypes.h``. The input argument, *arr*, can be any -:c:type:`PyObject *` that is directly interpretable as a -:c:type:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` +:c:expr:`PyObject *` that is directly interpretable as a +:c:expr:`PyArrayObject *` (any instance of the :c:data:`PyArray_Type` and its sub-types). .. c:function:: int PyArray_NDIM(PyArrayObject *arr) @@ -825,7 +825,7 @@ General check of Python Type Evaluates true if *op* is an instance of (a subclass of) :c:data:`PyArray_Type` and has 0 dimensions. -.. c:function:: PyArray_IsScalar(op, cls) +.. c:macro:: PyArray_IsScalar(op, cls) Evaluates true if *op* is an instance of ``Py{cls}ArrType_Type``. @@ -864,8 +864,8 @@ Data-type checking For the typenum macros, the argument is an integer representing an enumerated array data type. For the array type checking macros the -argument must be a :c:type:`PyObject *` that can be directly interpreted as a -:c:type:`PyArrayObject *`. +argument must be a :c:expr:`PyObject *` that can be directly interpreted as a +:c:expr:`PyArrayObject *`. .. c:function:: int PyTypeNum_ISUNSIGNED(int num) @@ -1022,7 +1022,7 @@ argument must be a :c:type:`PyObject *` that can be directly interpret .. c:function:: int PyArray_EquivByteorders(int b1, int b2) - True if byteorder characters ( :c:data:`NPY_LITTLE`, + True if byteorder characters *b1* and *b2* ( :c:data:`NPY_LITTLE`, :c:data:`NPY_BIG`, :c:data:`NPY_NATIVE`, :c:data:`NPY_IGNORE` ) are either equal or equivalent as to their specification of a native byte order. Thus, on a little-endian machine :c:data:`NPY_LITTLE` @@ -2781,14 +2781,14 @@ Data-type descriptors Data-type objects must be reference counted so be aware of the action on the data-type reference of different C-API calls. The standard rule is that when a data-type object is returned it is a - new reference. Functions that take :c:type:`PyArray_Descr *` objects and + new reference. Functions that take :c:expr:`PyArray_Descr *` objects and return arrays steal references to the data-type their inputs unless otherwise noted. Therefore, you must own a reference to any data-type object used as input to such a function. .. c:function:: int PyArray_DescrCheck(PyObject* obj) - Evaluates as true if *obj* is a data-type object ( :c:type:`PyArray_Descr *` ). + Evaluates as true if *obj* is a data-type object ( :c:expr:`PyArray_Descr *` ). .. c:function:: PyArray_Descr* PyArray_DescrNew(PyArray_Descr* obj) @@ -3485,10 +3485,6 @@ Miscellaneous Macros Evaluates as True if arrays *a1* and *a2* have the same shape. -.. c:var:: a - -.. c:var:: b - .. c:macro:: PyArray_MAX(a,b) Returns the maximum of *a* and *b*. If (*a*) or (*b*) are @@ -3547,22 +3543,22 @@ Miscellaneous Macros Enumerated Types ^^^^^^^^^^^^^^^^ -.. c:type:: NPY_SORTKIND +.. c:enum:: NPY_SORTKIND A special variable-type which can take on different values to indicate the sorting algorithm being used. - .. c:var:: NPY_QUICKSORT + .. c:enumerator:: NPY_QUICKSORT - .. c:var:: NPY_HEAPSORT + .. c:enumerator:: NPY_HEAPSORT - .. c:var:: NPY_MERGESORT + .. c:enumerator:: NPY_MERGESORT - .. c:var:: NPY_STABLESORT + .. c:enumerator:: NPY_STABLESORT Used as an alias of :c:data:`NPY_MERGESORT` and vica versa. - .. c:var:: NPY_NSORTS + .. c:enumerator:: NPY_NSORTS Defined to be the number of sorts. It is fixed at three by the need for backwards compatibility, and consequently :c:data:`NPY_MERGESORT` and @@ -3570,90 +3566,90 @@ Enumerated Types of several stable sorting algorithms depending on the data type. -.. c:type:: NPY_SCALARKIND +.. c:enum:: NPY_SCALARKIND A special variable type indicating the number of "kinds" of scalars distinguished in determining scalar-coercion rules. This variable can take on the values: - .. c:var:: NPY_NOSCALAR + .. c:enumerator:: NPY_NOSCALAR - .. c:var:: NPY_BOOL_SCALAR + .. c:enumerator:: NPY_BOOL_SCALAR - .. c:var:: NPY_INTPOS_SCALAR + .. c:enumerator:: NPY_INTPOS_SCALAR - .. c:var:: NPY_INTNEG_SCALAR + .. c:enumerator:: NPY_INTNEG_SCALAR - .. c:var:: NPY_FLOAT_SCALAR + .. c:enumerator:: NPY_FLOAT_SCALAR - .. c:var:: NPY_COMPLEX_SCALAR + .. c:enumerator:: NPY_COMPLEX_SCALAR - .. c:var:: NPY_OBJECT_SCALAR + .. c:enumerator:: NPY_OBJECT_SCALAR - .. c:var:: NPY_NSCALARKINDS + .. c:enumerator:: NPY_NSCALARKINDS Defined to be the number of scalar kinds (not including :c:data:`NPY_NOSCALAR`). -.. c:type:: NPY_ORDER +.. c:enum:: NPY_ORDER An enumeration type indicating the element order that an array should be interpreted in. When a brand new array is created, generally only **NPY_CORDER** and **NPY_FORTRANORDER** are used, whereas when one or more inputs are provided, the order can be based on them. - .. c:var:: NPY_ANYORDER + .. c:enumerator:: NPY_ANYORDER Fortran order if all the inputs are Fortran, C otherwise. - .. c:var:: NPY_CORDER + .. c:enumerator:: NPY_CORDER C order. - .. c:var:: NPY_FORTRANORDER + .. c:enumerator:: NPY_FORTRANORDER Fortran order. - .. c:var:: NPY_KEEPORDER + .. c:enumerator:: NPY_KEEPORDER An order as close to the order of the inputs as possible, even if the input is in neither C nor Fortran order. -.. c:type:: NPY_CLIPMODE +.. c:enum:: NPY_CLIPMODE A variable type indicating the kind of clipping that should be applied in certain functions. - .. c:var:: NPY_RAISE + .. c:enumerator:: NPY_RAISE The default for most operations, raises an exception if an index is out of bounds. - .. c:var:: NPY_CLIP + .. c:enumerator:: NPY_CLIP Clips an index to the valid range if it is out of bounds. - .. c:var:: NPY_WRAP + .. c:enumerator:: NPY_WRAP Wraps an index to the valid range if it is out of bounds. -.. c:type:: NPY_SEARCHSIDE +.. c:enum:: NPY_SEARCHSIDE A variable type indicating whether the index returned should be that of the first suitable location (if :c:data:`NPY_SEARCHLEFT`) or of the last (if :c:data:`NPY_SEARCHRIGHT`). - .. c:var:: NPY_SEARCHLEFT + .. c:enumerator:: NPY_SEARCHLEFT - .. c:var:: NPY_SEARCHRIGHT + .. c:enumerator:: NPY_SEARCHRIGHT -.. c:type:: NPY_SELECTKIND +.. c:enum:: NPY_SELECTKIND A variable type indicating the selection algorithm being used. - .. c:var:: NPY_INTROSELECT + .. c:enumerator:: NPY_INTROSELECT -.. c:type:: NPY_CASTING +.. c:enum:: NPY_CASTING .. versionadded:: 1.6 @@ -3661,25 +3657,25 @@ Enumerated Types be. This is used by the iterator added in NumPy 1.6, and is intended to be used more broadly in a future version. - .. c:var:: NPY_NO_CASTING + .. c:enumerator:: NPY_NO_CASTING Only allow identical types. - .. c:var:: NPY_EQUIV_CASTING + .. c:enumerator:: NPY_EQUIV_CASTING Allow identical and casts involving byte swapping. - .. c:var:: NPY_SAFE_CASTING + .. c:enumerator:: NPY_SAFE_CASTING Only allow casts which will not cause values to be rounded, truncated, or otherwise changed. - .. c:var:: NPY_SAME_KIND_CASTING + .. c:enumerator:: NPY_SAME_KIND_CASTING Allow any safe casts, and casts between types of the same kind. For example, float64 -> float32 is permitted with this rule. - .. c:var:: NPY_UNSAFE_CASTING + .. c:enumerator:: NPY_UNSAFE_CASTING Allow any cast, no matter what kind of data loss may occur. diff --git a/doc/source/reference/c-api/coremath.rst b/doc/source/reference/c-api/coremath.rst index 338c584a1024..cec83b150d21 100644 --- a/doc/source/reference/c-api/coremath.rst +++ b/doc/source/reference/c-api/coremath.rst @@ -46,30 +46,30 @@ Floating point classification corresponding single and extension precision macro are available with the suffix F and L. -.. c:function:: int npy_isnan(x) +.. c:macro:: npy_isnan(x) This is a macro, and is equivalent to C99 isnan: works for single, double and extended precision, and return a non 0 value is x is a NaN. -.. c:function:: int npy_isfinite(x) +.. c:macro:: npy_isfinite(x) This is a macro, and is equivalent to C99 isfinite: works for single, double and extended precision, and return a non 0 value is x is neither a NaN nor an infinity. -.. c:function:: int npy_isinf(x) +.. c:macro:: npy_isinf(x) This is a macro, and is equivalent to C99 isinf: works for single, double and extended precision, and return a non 0 value is x is infinite (positive and negative). -.. c:function:: int npy_signbit(x) +.. c:macro:: npy_signbit(x) This is a macro, and is equivalent to C99 signbit: works for single, double and extended precision, and return a non 0 value is x has the signbit set (that is the number is negative). -.. c:function:: double npy_copysign(double x, double y) +.. c:macro:: npy_copysign(x, y) This is a function equivalent to C99 copysign: return x with the same sign as y. Works for any value, including inf and nan. Single and extended diff --git a/doc/source/reference/c-api/dtype.rst b/doc/source/reference/c-api/dtype.rst index a1a53cdb600b..382e45dc0aa7 100644 --- a/doc/source/reference/c-api/dtype.rst +++ b/doc/source/reference/c-api/dtype.rst @@ -25,157 +25,157 @@ select the precision desired. Enumerated Types ---------------- -.. c:var:: NPY_TYPES +.. c:enumerator:: NPY_TYPES There is a list of enumerated types defined providing the basic 24 data types plus some useful generic names. Whenever the code requires a type number, one of these enumerated types is requested. The types are all called ``NPY_{NAME}``: -.. c:var:: NPY_BOOL +.. c:enumerator:: NPY_BOOL The enumeration value for the boolean type, stored as one byte. It may only be set to the values 0 and 1. -.. c:var:: NPY_BYTE -.. c:var:: NPY_INT8 +.. c:enumerator:: NPY_BYTE +.. c:enumerator:: NPY_INT8 The enumeration value for an 8-bit/1-byte signed integer. -.. c:var:: NPY_SHORT -.. c:var:: NPY_INT16 +.. c:enumerator:: NPY_SHORT +.. c:enumerator:: NPY_INT16 The enumeration value for a 16-bit/2-byte signed integer. -.. c:var:: NPY_INT -.. c:var:: NPY_INT32 +.. c:enumerator:: NPY_INT +.. c:enumerator:: NPY_INT32 The enumeration value for a 32-bit/4-byte signed integer. -.. c:var:: NPY_LONG +.. c:enumerator:: NPY_LONG Equivalent to either NPY_INT or NPY_LONGLONG, depending on the platform. -.. c:var:: NPY_LONGLONG -.. c:var:: NPY_INT64 +.. c:enumerator:: NPY_LONGLONG +.. c:enumerator:: NPY_INT64 The enumeration value for a 64-bit/8-byte signed integer. -.. c:var:: NPY_UBYTE -.. c:var:: NPY_UINT8 +.. c:enumerator:: NPY_UBYTE +.. c:enumerator:: NPY_UINT8 The enumeration value for an 8-bit/1-byte unsigned integer. -.. c:var:: NPY_USHORT -.. c:var:: NPY_UINT16 +.. c:enumerator:: NPY_USHORT +.. c:enumerator:: NPY_UINT16 The enumeration value for a 16-bit/2-byte unsigned integer. -.. c:var:: NPY_UINT -.. c:var:: NPY_UINT32 +.. c:enumerator:: NPY_UINT +.. c:enumerator:: NPY_UINT32 The enumeration value for a 32-bit/4-byte unsigned integer. -.. c:var:: NPY_ULONG +.. c:enumerator:: NPY_ULONG Equivalent to either NPY_UINT or NPY_ULONGLONG, depending on the platform. -.. c:var:: NPY_ULONGLONG -.. c:var:: NPY_UINT64 +.. c:enumerator:: NPY_ULONGLONG +.. c:enumerator:: NPY_UINT64 The enumeration value for a 64-bit/8-byte unsigned integer. -.. c:var:: NPY_HALF -.. c:var:: NPY_FLOAT16 +.. c:enumerator:: NPY_HALF +.. c:enumerator:: NPY_FLOAT16 The enumeration value for a 16-bit/2-byte IEEE 754-2008 compatible floating point type. -.. c:var:: NPY_FLOAT -.. c:var:: NPY_FLOAT32 +.. c:enumerator:: NPY_FLOAT +.. c:enumerator:: NPY_FLOAT32 The enumeration value for a 32-bit/4-byte IEEE 754 compatible floating point type. -.. c:var:: NPY_DOUBLE -.. c:var:: NPY_FLOAT64 +.. c:enumerator:: NPY_DOUBLE +.. c:enumerator:: NPY_FLOAT64 The enumeration value for a 64-bit/8-byte IEEE 754 compatible floating point type. -.. c:var:: NPY_LONGDOUBLE +.. c:enumerator:: NPY_LONGDOUBLE The enumeration value for a platform-specific floating point type which is at least as large as NPY_DOUBLE, but larger on many platforms. -.. c:var:: NPY_CFLOAT -.. c:var:: NPY_COMPLEX64 +.. c:enumerator:: NPY_CFLOAT +.. c:enumerator:: NPY_COMPLEX64 The enumeration value for a 64-bit/8-byte complex type made up of two NPY_FLOAT values. -.. c:var:: NPY_CDOUBLE -.. c:var:: NPY_COMPLEX128 +.. c:enumerator:: NPY_CDOUBLE +.. c:enumerator:: NPY_COMPLEX128 The enumeration value for a 128-bit/16-byte complex type made up of two NPY_DOUBLE values. -.. c:var:: NPY_CLONGDOUBLE +.. c:enumerator:: NPY_CLONGDOUBLE The enumeration value for a platform-specific complex floating point type which is made up of two NPY_LONGDOUBLE values. -.. c:var:: NPY_DATETIME +.. c:enumerator:: NPY_DATETIME The enumeration value for a data type which holds dates or datetimes with a precision based on selectable date or time units. -.. c:var:: NPY_TIMEDELTA +.. c:enumerator:: NPY_TIMEDELTA The enumeration value for a data type which holds lengths of times in integers of selectable date or time units. -.. c:var:: NPY_STRING +.. c:enumerator:: NPY_STRING The enumeration value for ASCII strings of a selectable size. The strings have a fixed maximum size within a given array. -.. c:var:: NPY_UNICODE +.. c:enumerator:: NPY_UNICODE The enumeration value for UCS4 strings of a selectable size. The strings have a fixed maximum size within a given array. -.. c:var:: NPY_OBJECT +.. c:enumerator:: NPY_OBJECT The enumeration value for references to arbitrary Python objects. -.. c:var:: NPY_VOID +.. c:enumerator:: NPY_VOID Primarily used to hold struct dtypes, but can contain arbitrary binary data. Some useful aliases of the above types are -.. c:var:: NPY_INTP +.. c:enumerator:: NPY_INTP The enumeration value for a signed integer type which is the same size as a (void \*) pointer. This is the type used by all arrays of indices. -.. c:var:: NPY_UINTP +.. c:enumerator:: NPY_UINTP The enumeration value for an unsigned integer type which is the same size as a (void \*) pointer. -.. c:var:: NPY_MASK +.. c:enumerator:: NPY_MASK The enumeration value of the type used for masks, such as with the :c:data:`NPY_ITER_ARRAYMASK` iterator flag. This is equivalent to :c:data:`NPY_UINT8`. -.. c:var:: NPY_DEFAULT_TYPE +.. c:enumerator:: NPY_DEFAULT_TYPE The default type to use when no dtype is explicitly specified, for example when calling np.zero(shape). This is equivalent to @@ -297,9 +297,13 @@ Boolean Unsigned versions of the integers can be defined by pre-pending a 'u' to the front of the integer name. -.. c:type:: npy_(u)byte +.. c:type:: npy_byte - (unsigned) char + char + +.. c:type:: npy_ubyte + + unsigned char .. c:type:: npy_short @@ -309,14 +313,14 @@ to the front of the integer name. unsigned short -.. c:type:: npy_uint - - unsigned int - .. c:type:: npy_int int +.. c:type:: npy_uint + + unsigned int + .. c:type:: npy_int16 16-bit integer @@ -341,13 +345,21 @@ to the front of the integer name. 64-bit unsigned integer -.. c:type:: npy_(u)long +.. c:type:: npy_long - (unsigned) long int + long int -.. c:type:: npy_(u)longlong +.. c:type:: npy_ulong - (unsigned long long int) + unsigned long int + +.. c:type:: npy_longlong + + long long int + +.. c:type:: npy_ulonglong + + unsigned long long int .. c:type:: npy_intp @@ -367,18 +379,30 @@ to the front of the integer name. 16-bit float -.. c:type:: npy_(c)float +.. c:type:: npy_float 32-bit float -.. c:type:: npy_(c)double +.. c:type:: npy_cfloat + + 32-bit complex float + +.. c:type:: npy_double 64-bit double -.. c:type:: npy_(c)longdouble +.. c:type:: npy_cdouble + + 64-bit complex double + +.. c:type:: npy_longdouble long double +.. c:type:: npy_clongdouble + + long complex double + complex types are structures with **.real** and **.imag** members (in that order). diff --git a/doc/source/reference/c-api/iterator.rst b/doc/source/reference/c-api/iterator.rst index ae96bb3fb056..add96e3b4698 100644 --- a/doc/source/reference/c-api/iterator.rst +++ b/doc/source/reference/c-api/iterator.rst @@ -370,7 +370,7 @@ Construction and Destruction arrays or structured arrays containing an object type) may be accepted and used in the iterator. If this flag is enabled, the caller must be sure to check whether - :c:func:`NpyIter_IterationNeedsAPI(iter)` is true, in which case + :c:expr:`NpyIter_IterationNeedsAPI(iter)` is true, in which case it may not release the GIL during iteration. .. c:macro:: NPY_ITER_ZEROSIZE_OK @@ -738,7 +738,7 @@ Construction and Destruction the iterator. Any cached functions or pointers from the iterator must be retrieved again! - After calling this function, :c:func:`NpyIter_HasMultiIndex(iter)` will + After calling this function, :c:expr:`NpyIter_HasMultiIndex(iter)` will return false. Returns ``NPY_SUCCEED`` or ``NPY_FAIL``. diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 763f985a6577..ab82fda8754c 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -7,7 +7,7 @@ Python Types and C-Structures Several new types are defined in the C-code. Most of these are accessible from Python, but a few are not exposed due to their limited -use. Every new Python type has an associated :c:type:`PyObject *` with an +use. Every new Python type has an associated :c:expr:`PyObject *` with an internal structure that includes a pointer to a "method table" that defines how the new object behaves in Python. When you receive a Python object into C code, you always get a pointer to a @@ -61,7 +61,7 @@ hierarchy of actual Python types. PyArray_Type and PyArrayObject ------------------------------ -.. c:var:: PyArray_Type +.. c:var:: PyTypeObject PyArray_Type The Python type of the ndarray is :c:data:`PyArray_Type`. In C, every ndarray is a pointer to a :c:type:`PyArrayObject` structure. The ob_type @@ -201,7 +201,7 @@ PyArray_Type and PyArrayObject PyArrayDescr_Type and PyArray_Descr ----------------------------------- -.. c:var:: PyArrayDescr_Type +.. c:var:: PyTypeObject PyArrayDescr_Type The :c:data:`PyArrayDescr_Type` is the built-in type of the data-type-descriptor objects used to describe how the bytes comprising @@ -636,7 +636,7 @@ PyArrayDescr_Type and PyArray_Descr Either ``NULL`` or a dictionary containing low-level casting functions for user- defined data-types. Each function is - wrapped in a :c:type:`PyCapsule *` and keyed by + wrapped in a :c:expr:`PyCapsule *` and keyed by the data-type number. .. c:member:: NPY_SCALARKIND scalarkind(PyArrayObject* arr) @@ -754,7 +754,7 @@ The :c:data:`PyArray_Type` can also be sub-typed. PyUFunc_Type and PyUFuncObject ------------------------------ -.. c:var:: PyUFunc_Type +.. c:var:: PyTypeObject PyUFunc_Type The ufunc object is implemented by creation of the :c:data:`PyUFunc_Type`. It is a very simple type that implements only @@ -1006,7 +1006,7 @@ PyUFunc_Type and PyUFuncObject PyArrayIter_Type and PyArrayIterObject -------------------------------------- -.. c:var:: PyArrayIter_Type +.. c:var:: PyTypeObject PyArrayIter_Type This is an iterator object that makes it easy to loop over an N-dimensional array. It is the object returned from the flat @@ -1110,13 +1110,13 @@ the internal structure of the iterator object, and merely interact with it through the use of the macros :c:func:`PyArray_ITER_NEXT` (it), :c:func:`PyArray_ITER_GOTO` (it, dest), or :c:func:`PyArray_ITER_GOTO1D` (it, index). All of these macros require the argument *it* to be a -:c:type:`PyArrayIterObject *`. +:c:expr:`PyArrayIterObject *`. PyArrayMultiIter_Type and PyArrayMultiIterObject ------------------------------------------------ -.. c:var:: PyArrayMultiIter_Type +.. c:var:: PyTypeObject PyArrayMultiIter_Type This type provides an iterator that encapsulates the concept of broadcasting. It allows :math:`N` arrays to be broadcast together @@ -1178,7 +1178,7 @@ PyArrayMultiIter_Type and PyArrayMultiIterObject PyArrayNeighborhoodIter_Type and PyArrayNeighborhoodIterObject -------------------------------------------------------------- -.. c:var:: PyArrayNeighborhoodIter_Type +.. c:var:: PyTypeObject PyArrayNeighborhoodIter_Type This is an iterator object that makes it easy to loop over an N-dimensional neighborhood. @@ -1217,7 +1217,7 @@ PyArrayNeighborhoodIter_Type and PyArrayNeighborhoodIterObject PyArrayFlags_Type and PyArrayFlagsObject ---------------------------------------- -.. c:var:: PyArrayFlags_Type +.. c:var:: PyTypeObject PyArrayFlags_Type When the flags attribute is retrieved from Python, a special builtin object of this type is constructed. This special type makes @@ -1466,7 +1466,7 @@ for completeness and assistance in understanding the code. to define a 1-d loop for a ufunc for every defined signature of a user-defined data-type. -.. c:var:: PyArrayMapIter_Type +.. c:var:: PyTypeObject PyArrayMapIter_Type Advanced indexing is handled with this Python type. It is simply a loose wrapper around the C-structure containing the variables diff --git a/doc/source/reference/c-api/ufunc.rst b/doc/source/reference/c-api/ufunc.rst index 9eb70c3fbe34..95dc47839e4b 100644 --- a/doc/source/reference/c-api/ufunc.rst +++ b/doc/source/reference/c-api/ufunc.rst @@ -283,20 +283,6 @@ Functions signature is an array of data-type numbers indicating the inputs followed by the outputs assumed by the 1-d loop. -.. c:function:: int PyUFunc_GenericFunction( \ - PyUFuncObject* self, PyObject* args, PyObject* kwds, PyArrayObject** mps) - - .. deprecated:: NumPy 1.19 - - Unless NumPy is made aware of an issue with this, this function - is scheduled for rapid removal without replacement. - - Instead of this function ``PyObject_Call(ufunc, args, kwds)`` should be - used. The above function differs from this because it ignores support - for non-array, or array subclasses as inputs. - To ensure identical behaviour, it may be necessary to convert all inputs - using ``PyArray_FromAny(obj, NULL, 0, 0, NPY_ARRAY_ENSUREARRAY, NULL)``. - .. c:function:: int PyUFunc_checkfperr(int errmask, PyObject* errobj) A simple interface to the IEEE error-flag checking support. The diff --git a/doc/source/reference/global_state.rst b/doc/source/reference/global_state.rst index b59467210254..f184812354d3 100644 --- a/doc/source/reference/global_state.rst +++ b/doc/source/reference/global_state.rst @@ -84,17 +84,3 @@ contiguous in memory. Most users will have no reason to change these; for details see the :ref:`memory layout ` documentation. -Using the new casting implementation ------------------------------------- - -Within NumPy 1.20 it is possible to enable the new experimental casting -implementation for testing purposes. To do this set:: - - NPY_USE_NEW_CASTINGIMPL=1 - -Setting the flag is only useful to aid with NumPy developement to ensure the -new version is bug free and should be avoided for production code. -It is a helpful test for projects that either create custom datatypes or -use for example complicated structured dtypes. The flag is expected to be -removed in 1.21 with the new version being always in use. - diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 6eb74cd770a0..f12d923dfdb4 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -1,5 +1,7 @@ .. _reference: +.. module:: numpy + ############### NumPy Reference ############### @@ -7,9 +9,6 @@ NumPy Reference :Release: |version| :Date: |today| - -.. module:: numpy - This reference manual details functions, modules, and objects included in NumPy, describing what they are and what they do. For learning how to use NumPy, see the :ref:`complete documentation `. diff --git a/doc/source/reference/random/bit_generators/index.rst b/doc/source/reference/random/bit_generators/index.rst index 315657172fa4..c5c3498068bd 100644 --- a/doc/source/reference/random/bit_generators/index.rst +++ b/doc/source/reference/random/bit_generators/index.rst @@ -15,10 +15,13 @@ Supported BitGenerators The included BitGenerators are: -* PCG-64 - The default. A fast generator that supports many parallel streams - and can be advanced by an arbitrary amount. See the documentation for - :meth:`~.PCG64.advance`. PCG-64 has a period of :math:`2^{128}`. See the `PCG - author's page`_ for more details about this class of PRNG. +* PCG-64 - The default. A fast generator that can be advanced by an arbitrary + amount. See the documentation for :meth:`~.PCG64.advance`. PCG-64 has + a period of :math:`2^{128}`. See the `PCG author's page`_ for more details + about this class of PRNG. +* PCG-64 DXSM - An upgraded version of PCG-64 with better statistical + properties in parallel contexts. See :ref:`upgrading-pcg64` for more + information on these improvements. * MT19937 - The standard Python BitGenerator. Adds a `MT19937.jumped` function that returns a new generator with state as-if :math:`2^{128}` draws have been made. @@ -43,6 +46,7 @@ The included BitGenerators are: MT19937 PCG64 + PCG64DXSM Philox SFC64 @@ -105,6 +109,44 @@ If you need to generate a good seed "offline", then ``SeedSequence().entropy`` or using ``secrets.randbits(128)`` from the standard library are both convenient ways. +If you need to run several stochastic simulations in parallel, best practice +is to construct a random generator instance for each simulation. +To make sure that the random streams have distinct initial states, you can use +the `spawn` method of `~SeedSequence`. For instance, here we construct a list +of 12 instances: + +.. code-block:: python + + from numpy.random import PCG64, SeedSequence + + # High quality initial entropy + entropy = 0x87351080e25cb0fad77a44a3be03b491 + base_seq = SeedSequence(entropy) + child_seqs = base_seq.spawn(12) # a list of 12 SeedSequences + generators = [PCG64(seq) for seq in child_seqs] + +.. end_block + + +An alternative way is to use the fact that a `~SeedSequence` can be initialized +by a tuple of elements. Here we use a base entropy value and an integer +``worker_id`` + +.. code-block:: python + + from numpy.random import PCG64, SeedSequence + + # High quality initial entropy + entropy = 0x87351080e25cb0fad77a44a3be03b491 + sequences = [SeedSequence((entropy, worker_id)) for worker_id in range(12)] + generators = [PCG64(seq) for seq in sequences] + +.. end_block + +Note that the sequences produced by the latter method will be distinct from +those constructed via `~SeedSequence.spawn`. + + .. autosummary:: :toctree: generated/ diff --git a/doc/source/reference/random/bit_generators/mt19937.rst b/doc/source/reference/random/bit_generators/mt19937.rst index 71875db4e5dd..d05ea7c6f09a 100644 --- a/doc/source/reference/random/bit_generators/mt19937.rst +++ b/doc/source/reference/random/bit_generators/mt19937.rst @@ -4,7 +4,8 @@ Mersenne Twister (MT19937) .. currentmodule:: numpy.random .. autoclass:: MT19937 - :exclude-members: + :members: __init__ + :exclude-members: __init__ State ===== diff --git a/doc/source/reference/random/bit_generators/pcg64.rst b/doc/source/reference/random/bit_generators/pcg64.rst index edac4620baa3..889965f77bc2 100644 --- a/doc/source/reference/random/bit_generators/pcg64.rst +++ b/doc/source/reference/random/bit_generators/pcg64.rst @@ -4,7 +4,8 @@ Permuted Congruential Generator (64-bit, PCG64) .. currentmodule:: numpy.random .. autoclass:: PCG64 - :exclude-members: + :members: __init__ + :exclude-members: __init__ State ===== diff --git a/doc/source/reference/random/bit_generators/pcg64dxsm.rst b/doc/source/reference/random/bit_generators/pcg64dxsm.rst new file mode 100644 index 000000000000..e37efa5d39da --- /dev/null +++ b/doc/source/reference/random/bit_generators/pcg64dxsm.rst @@ -0,0 +1,32 @@ +Permuted Congruential Generator (64-bit, PCG64 DXSM) +---------------------------------------------------- + +.. currentmodule:: numpy.random + +.. autoclass:: PCG64DXSM + :members: __init__ + :exclude-members: __init__ + +State +===== + +.. autosummary:: + :toctree: generated/ + + ~PCG64DXSM.state + +Parallel generation +=================== +.. autosummary:: + :toctree: generated/ + + ~PCG64DXSM.advance + ~PCG64DXSM.jumped + +Extending +========= +.. autosummary:: + :toctree: generated/ + + ~PCG64DXSM.cffi + ~PCG64DXSM.ctypes diff --git a/doc/source/reference/random/bit_generators/philox.rst b/doc/source/reference/random/bit_generators/philox.rst index 8eba2d3511e4..3c2fa4cc5aa1 100644 --- a/doc/source/reference/random/bit_generators/philox.rst +++ b/doc/source/reference/random/bit_generators/philox.rst @@ -4,7 +4,8 @@ Philox Counter-based RNG .. currentmodule:: numpy.random .. autoclass:: Philox - :exclude-members: + :members: __init__ + :exclude-members: __init__ State ===== diff --git a/doc/source/reference/random/bit_generators/sfc64.rst b/doc/source/reference/random/bit_generators/sfc64.rst index d34124a33355..8cb255bc154d 100644 --- a/doc/source/reference/random/bit_generators/sfc64.rst +++ b/doc/source/reference/random/bit_generators/sfc64.rst @@ -4,7 +4,8 @@ SFC64 Small Fast Chaotic PRNG .. currentmodule:: numpy.random .. autoclass:: SFC64 - :exclude-members: + :members: __init__ + :exclude-members: __init__ State ===== diff --git a/doc/source/reference/random/c-api.rst b/doc/source/reference/random/c-api.rst index a79da7a492f7..de403ce98673 100644 --- a/doc/source/reference/random/c-api.rst +++ b/doc/source/reference/random/c-api.rst @@ -3,6 +3,8 @@ C API for random .. currentmodule:: numpy.random +.. versionadded:: 1.19.0 + Access to various distributions below is available via Cython or C-wrapper libraries like CFFI. All the functions accept a :c:type:`bitgen_t` as their first argument. To access these from Cython or C, you must link with the @@ -40,9 +42,9 @@ The functions are named with the following conventions: - The functions without "standard" in their name require additional parameters to describe the distributions. -- ``zig`` in the name are based on a ziggurat lookup algorithm is used instead - of calculating the ``log``, which is significantly faster. The non-ziggurat - variants are used in corner cases and for legacy compatibility. +- Functions with ``inv`` in their name are based on the slower inverse method + instead of a ziggurat lookup algorithm, which is significantly faster. The + non-ziggurat variants are used in corner cases and for legacy compatibility. .. c:function:: double random_standard_uniform(bitgen_t *bitgen_state) @@ -53,6 +55,8 @@ The functions are named with the following conventions: .. c:function:: void random_standard_exponential_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) +.. c:function:: void random_standard_exponential_inv_fill(bitgen_t *bitgen_state, npy_intp cnt, double *out) + .. c:function:: double random_standard_normal(bitgen_t* bitgen_state) .. c:function:: void random_standard_normal_fill(bitgen_t *bitgen_state, npy_intp count, double *out) @@ -69,6 +73,8 @@ The functions are named with the following conventions: .. c:function:: void random_standard_exponential_fill_f(bitgen_t *bitgen_state, npy_intp cnt, float *out) +.. c:function:: void random_standard_exponential_inv_fill_f(bitgen_t *bitgen_state, npy_intp cnt, float *out) + .. c:function:: float random_standard_normal_f(bitgen_t* bitgen_state) .. c:function:: float random_standard_gamma_f(bitgen_t *bitgen_state, float shape) diff --git a/doc/source/reference/random/generator.rst b/doc/source/reference/random/generator.rst index 8706e1de2c80..7934be98a6d4 100644 --- a/doc/source/reference/random/generator.rst +++ b/doc/source/reference/random/generator.rst @@ -15,7 +15,8 @@ can be changed by passing an instantized BitGenerator to ``Generator``. .. autofunction:: default_rng .. autoclass:: Generator - :exclude-members: + :members: __init__ + :exclude-members: __init__ Accessing the BitGenerator ========================== @@ -70,13 +71,13 @@ By default, `Generator.permuted` returns a copy. To operate in-place with `Generator.permuted`, pass the same array as the first argument *and* as the value of the ``out`` parameter. For example, - >>> rg = np.random.default_rng() + >>> rng = np.random.default_rng() >>> x = np.arange(0, 15).reshape(3, 5) >>> x array([[ 0, 1, 2, 3, 4], [ 5, 6, 7, 8, 9], [10, 11, 12, 13, 14]]) - >>> y = rg.permuted(x, axis=1, out=x) + >>> y = rng.permuted(x, axis=1, out=x) >>> x array([[ 1, 0, 2, 4, 3], # random [ 6, 7, 8, 9, 5], @@ -96,13 +97,13 @@ which dimension of the input array to use as the sequence. In the case of a two-dimensional array, ``axis=0`` will, in effect, rearrange the rows of the array, and ``axis=1`` will rearrange the columns. For example - >>> rg = np.random.default_rng() + >>> rng = np.random.default_rng() >>> x = np.arange(0, 15).reshape(3, 5) >>> x array([[ 0, 1, 2, 3, 4], [ 5, 6, 7, 8, 9], [10, 11, 12, 13, 14]]) - >>> rg.permutation(x, axis=1) + >>> rng.permutation(x, axis=1) array([[ 1, 3, 2, 0, 4], # random [ 6, 8, 7, 5, 9], [11, 13, 12, 10, 14]]) @@ -115,7 +116,7 @@ how `numpy.sort` treats it. Each slice along the given axis is shuffled independently of the others. Compare the following example of the use of `Generator.permuted` to the above example of `Generator.permutation`: - >>> rg.permuted(x, axis=1) + >>> rng.permuted(x, axis=1) array([[ 1, 0, 2, 4, 3], # random [ 5, 7, 6, 9, 8], [10, 14, 12, 13, 11]]) @@ -130,9 +131,9 @@ Shuffling non-NumPy sequences a sequence that is not a NumPy array, it shuffles that sequence in-place. For example, - >>> rg = np.random.default_rng() + >>> rng = np.random.default_rng() >>> a = ['A', 'B', 'C', 'D', 'E'] - >>> rg.shuffle(a) # shuffle the list in-place + >>> rng.shuffle(a) # shuffle the list in-place >>> a ['B', 'D', 'A', 'E', 'C'] # random diff --git a/doc/source/reference/random/index.rst b/doc/source/reference/random/index.rst index 13ce7c40cf26..96cd47017cca 100644 --- a/doc/source/reference/random/index.rst +++ b/doc/source/reference/random/index.rst @@ -25,7 +25,7 @@ nep-0019-rng-policy.html>`_ for context on the updated random Numpy number routines. The legacy `RandomState` random number routines are still available, but limited to a single BitGenerator. See :ref:`new-or-different` for a complete list of improvements and differences from the legacy -``Randomstate``. +``RandomState``. For convenience and backward compatibility, a single `RandomState` instance's methods are imported into the numpy.random namespace, see @@ -84,10 +84,10 @@ different .. code-block:: python try: - rg_integers = rg.integers + rng_integers = rng.integers except AttributeError: - rg_integers = rg.randint - a = rg_integers(1000) + rng_integers = rng.randint + a = rng_integers(1000) Seeds can be passed to any of the BitGenerators. The provided value is mixed via `SeedSequence` to spread a possible sequence of seeds across a wider @@ -97,8 +97,8 @@ is wrapped with a `Generator`. .. code-block:: python from numpy.random import Generator, PCG64 - rg = Generator(PCG64(12345)) - rg.standard_normal() + rng = Generator(PCG64(12345)) + rng.standard_normal() Here we use `default_rng` to create an instance of `Generator` to generate a random float: @@ -146,10 +146,10 @@ As a convenience NumPy provides the `default_rng` function to hide these details: >>> from numpy.random import default_rng ->>> rg = default_rng(12345) ->>> print(rg) +>>> rng = default_rng(12345) +>>> print(rng) Generator(PCG64) ->>> print(rg.random()) +>>> print(rng.random()) 0.22733602246716966 One can also instantiate `Generator` directly with a `BitGenerator` instance. @@ -158,16 +158,16 @@ To use the default `PCG64` bit generator, one can instantiate it directly and pass it to `Generator`: >>> from numpy.random import Generator, PCG64 ->>> rg = Generator(PCG64(12345)) ->>> print(rg) +>>> rng = Generator(PCG64(12345)) +>>> print(rng) Generator(PCG64) Similarly to use the older `MT19937` bit generator (not recommended), one can instantiate it directly and pass it to `Generator`: >>> from numpy.random import Generator, MT19937 ->>> rg = Generator(MT19937(12345)) ->>> print(rg) +>>> rng = Generator(MT19937(12345)) +>>> print(rng) Generator(MT19937) What's New or Different @@ -222,6 +222,9 @@ one of three ways: * :ref:`independent-streams` * :ref:`parallel-jumped` +Users with a very large amount of parallelism will want to consult +:ref:`upgrading-pcg64`. + Concepts -------- .. toctree:: @@ -230,6 +233,7 @@ Concepts generator Legacy Generator (RandomState) BitGenerators, SeedSequences + Upgrading PCG64 with PCG64DXSM Features -------- diff --git a/doc/source/reference/random/legacy.rst b/doc/source/reference/random/legacy.rst index 6cf4775b81ba..42437dbb6173 100644 --- a/doc/source/reference/random/legacy.rst +++ b/doc/source/reference/random/legacy.rst @@ -48,7 +48,8 @@ using the state of the `RandomState`: .. autoclass:: RandomState - :exclude-members: + :members: __init__ + :exclude-members: __init__ Seeding and State ================= diff --git a/doc/source/reference/random/new-or-different.rst b/doc/source/reference/random/new-or-different.rst index 6cab0f729e64..a815439267fc 100644 --- a/doc/source/reference/random/new-or-different.rst +++ b/doc/source/reference/random/new-or-different.rst @@ -58,18 +58,18 @@ And in more detail: from numpy.random import Generator, PCG64 import numpy.random - rg = Generator(PCG64()) - %timeit -n 1 rg.standard_normal(100000) + rng = Generator(PCG64()) + %timeit -n 1 rng.standard_normal(100000) %timeit -n 1 numpy.random.standard_normal(100000) .. ipython:: python - %timeit -n 1 rg.standard_exponential(100000) + %timeit -n 1 rng.standard_exponential(100000) %timeit -n 1 numpy.random.standard_exponential(100000) .. ipython:: python - %timeit -n 1 rg.standard_gamma(3.0, 100000) + %timeit -n 1 rng.standard_gamma(3.0, 100000) %timeit -n 1 numpy.random.standard_gamma(3.0, 100000) @@ -94,9 +94,9 @@ And in more detail: .. ipython:: python - rg = Generator(PCG64(0)) - rg.random(3, dtype='d') - rg.random(3, dtype='f') + rng = Generator(PCG64(0)) + rng.random(3, dtype='d') + rng.random(3, dtype='f') * Optional ``out`` argument that allows existing arrays to be filled for select distributions @@ -112,7 +112,7 @@ And in more detail: .. ipython:: python existing = np.zeros(4) - rg.random(out=existing[:2]) + rng.random(out=existing[:2]) print(existing) * Optional ``axis`` argument for methods like `~.Generator.choice`, @@ -121,9 +121,9 @@ And in more detail: .. ipython:: python - rg = Generator(PCG64(123456789)) + rng = Generator(PCG64(123456789)) a = np.arange(12).reshape((3, 4)) a - rg.choice(a, axis=1, size=5) - rg.shuffle(a, axis=1) # Shuffle in-place + rng.choice(a, axis=1, size=5) + rng.shuffle(a, axis=1) # Shuffle in-place a diff --git a/doc/source/reference/random/parallel.rst b/doc/source/reference/random/parallel.rst index 72158401447e..7f0207bdebb2 100644 --- a/doc/source/reference/random/parallel.rst +++ b/doc/source/reference/random/parallel.rst @@ -88,10 +88,11 @@ territory ([2]_). estimate the naive upper bound on a napkin and take comfort knowing that the probability is actually lower. -.. [2] In this calculation, we can ignore the amount of numbers drawn from each - stream. Each of the PRNGs we provide has some extra protection built in +.. [2] In this calculation, we can mostly ignore the amount of numbers drawn from each + stream. See :ref:`upgrading-pcg64` for the technical details about + `PCG64`. The other PRNGs we provide have some extra protection built in that avoids overlaps if the `~SeedSequence` pools differ in the - slightest bit. `PCG64` has :math:`2^{127}` separate cycles + slightest bit. `PCG64DXSM` has :math:`2^{127}` separate cycles determined by the seed in addition to the position in the :math:`2^{128}` long period for each cycle, so one has to both get on or near the same cycle *and* seed a nearby position in the cycle. @@ -150,12 +151,14 @@ BitGenerator, the size of the jump and the bits in the default unsigned random are listed below. +-----------------+-------------------------+-------------------------+-------------------------+ -| BitGenerator | Period | Jump Size | Bits | +| BitGenerator | Period | Jump Size | Bits per Draw | +=================+=========================+=========================+=========================+ -| MT19937 | :math:`2^{19937}` | :math:`2^{128}` | 32 | +| MT19937 | :math:`2^{19937}-1` | :math:`2^{128}` | 32 | +-----------------+-------------------------+-------------------------+-------------------------+ | PCG64 | :math:`2^{128}` | :math:`~2^{127}` ([3]_) | 64 | +-----------------+-------------------------+-------------------------+-------------------------+ +| PCG64DXSM | :math:`2^{128}` | :math:`~2^{127}` ([3]_) | 64 | ++-----------------+-------------------------+-------------------------+-------------------------+ | Philox | :math:`2^{256}` | :math:`2^{128}` | 64 | +-----------------+-------------------------+-------------------------+-------------------------+ diff --git a/doc/source/reference/random/performance.py b/doc/source/reference/random/performance.py index 28a42eb0dce2..794142836652 100644 --- a/doc/source/reference/random/performance.py +++ b/doc/source/reference/random/performance.py @@ -1,14 +1,13 @@ -from collections import OrderedDict from timeit import repeat import pandas as pd import numpy as np -from numpy.random import MT19937, PCG64, Philox, SFC64 +from numpy.random import MT19937, PCG64, PCG64DXSM, Philox, SFC64 -PRNGS = [MT19937, PCG64, Philox, SFC64] +PRNGS = [MT19937, PCG64, PCG64DXSM, Philox, SFC64] -funcs = OrderedDict() +funcs = {} integers = 'integers(0, 2**{bits},size=1000000, dtype="uint{bits}")' funcs['32-bit Unsigned Ints'] = integers.format(bits=32) funcs['64-bit Unsigned Ints'] = integers.format(bits=64) @@ -26,10 +25,10 @@ """ test = "rg.{func}" -table = OrderedDict() +table = {} for prng in PRNGS: print(prng) - col = OrderedDict() + col = {} for key in funcs: t = repeat(test.format(func=funcs[key]), setup.format(prng=prng().__class__.__name__), @@ -38,7 +37,7 @@ col = pd.Series(col) table[prng().__class__.__name__] = col -npfuncs = OrderedDict() +npfuncs = {} npfuncs.update(funcs) npfuncs['32-bit Unsigned Ints'] = 'randint(2**32,dtype="uint32",size=1000000)' npfuncs['64-bit Unsigned Ints'] = 'randint(2**64,dtype="uint64",size=1000000)' @@ -54,7 +53,7 @@ col[key] = 1000 * min(t) table['RandomState'] = pd.Series(col) -columns = ['MT19937','PCG64','Philox','SFC64', 'RandomState'] +columns = ['MT19937', 'PCG64', 'PCG64DXSM', 'Philox', 'SFC64', 'RandomState'] table = pd.DataFrame(table) order = np.log(table).mean().sort_values().index table = table.T diff --git a/doc/source/reference/random/performance.rst b/doc/source/reference/random/performance.rst index 74dad4cc37d3..85855be59363 100644 --- a/doc/source/reference/random/performance.rst +++ b/doc/source/reference/random/performance.rst @@ -5,9 +5,12 @@ Performance Recommendation ************** -The recommended generator for general use is `PCG64`. It is -statistically high quality, full-featured, and fast on most platforms, but -somewhat slow when compiled for 32-bit processes. + +The recommended generator for general use is `PCG64` or its upgraded variant +`PCG64DXSM` for heavily-parallel use cases. They are statistically high quality, +full-featured, and fast on most platforms, but somewhat slow when compiled for +32-bit processes. See :ref:`upgrading-pcg64` for details on when heavy +parallelism would indicate using `PCG64DXSM`. `Philox` is fairly slow, but its statistical properties have very high quality, and it is easy to get assuredly-independent stream by using @@ -39,49 +42,48 @@ Integer performance has a similar ordering. The pattern is similar for other, more complex generators. The normal performance of the legacy `RandomState` generator is much -lower than the other since it uses the Box-Muller transformation rather -than the Ziggurat generator. The performance gap for Exponentials is also +lower than the other since it uses the Box-Muller transform rather +than the Ziggurat method. The performance gap for Exponentials is also large due to the cost of computing the log function to invert the CDF. -The column labeled MT19973 is used the same 32-bit generator as -`RandomState` but produces random values using -`Generator`. +The column labeled MT19973 uses the same 32-bit generator as +`RandomState` but produces random variates using `Generator`. .. csv-table:: - :header: ,MT19937,PCG64,Philox,SFC64,RandomState - :widths: 14,14,14,14,14,14 - - 32-bit Unsigned Ints,3.2,2.7,4.9,2.7,3.2 - 64-bit Unsigned Ints,5.6,3.7,6.3,2.9,5.7 - Uniforms,7.3,4.1,8.1,3.1,7.3 - Normals,13.1,10.2,13.5,7.8,34.6 - Exponentials,7.9,5.4,8.5,4.1,40.3 - Gammas,34.8,28.0,34.7,25.1,58.1 - Binomials,25.0,21.4,26.1,19.5,25.2 - Laplaces,45.1,40.7,45.5,38.1,45.6 - Poissons,67.6,52.4,69.2,46.4,78.1 + :header: ,MT19937,PCG64,PCG64DXSM,Philox,SFC64,RandomState + :widths: 14,14,14,14,14,14,14 + + 32-bit Unsigned Ints,3.3,1.9,2.0,3.3,1.8,3.1 + 64-bit Unsigned Ints,5.6,3.2,2.9,4.9,2.5,5.5 + Uniforms,5.9,3.1,2.9,5.0,2.6,6.0 + Normals,13.9,10.8,10.5,12.0,8.3,56.8 + Exponentials,9.1,6.0,5.8,8.1,5.4,63.9 + Gammas,37.2,30.8,28.9,34.0,27.5,77.0 + Binomials,21.3,17.4,17.6,19.3,15.6,21.4 + Laplaces,73.2,72.3,76.1,73.0,72.3,82.5 + Poissons,111.7,103.4,100.5,109.4,90.7,115.2 The next table presents the performance in percentage relative to values generated by the legacy generator, ``RandomState(MT19937())``. The overall performance was computed using a geometric mean. .. csv-table:: - :header: ,MT19937,PCG64,Philox,SFC64 - :widths: 14,14,14,14,14 - - 32-bit Unsigned Ints,101,121,67,121 - 64-bit Unsigned Ints,102,156,91,199 - Uniforms,100,179,90,235 - Normals,263,338,257,443 - Exponentials,507,752,474,985 - Gammas,167,207,167,231 - Binomials,101,118,96,129 - Laplaces,101,112,100,120 - Poissons,116,149,113,168 - Overall,144,192,132,225 + :header: ,MT19937,PCG64,PCG64DXSM,Philox,SFC64 + :widths: 14,14,14,14,14,14 + + 32-bit Unsigned Ints,96,162,160,96,175 + 64-bit Unsigned Ints,97,171,188,113,218 + Uniforms,102,192,206,121,233 + Normals,409,526,541,471,684 + Exponentials,701,1071,1101,784,1179 + Gammas,207,250,266,227,281 + Binomials,100,123,122,111,138 + Laplaces,113,114,108,113,114 + Poissons,103,111,115,105,127 + Overall,159,219,225,174,251 .. note:: - All timings were taken using Linux on an i5-3570 processor. + All timings were taken using Linux on an AMD Ryzen 9 3900X processor. Performance on different Operating Systems ****************************************** @@ -98,33 +100,33 @@ across tables. 64-bit Linux ~~~~~~~~~~~~ -=================== ========= ======= ======== ======= -Distribution MT19937 PCG64 Philox SFC64 -=================== ========= ======= ======== ======= -32-bit Unsigned Int 100 119.8 67.7 120.2 -64-bit Unsigned Int 100 152.9 90.8 213.3 -Uniforms 100 179.0 87.0 232.0 -Normals 100 128.5 99.2 167.8 -Exponentials 100 148.3 93.0 189.3 -**Overall** 100 144.3 86.8 180.0 -=================== ========= ======= ======== ======= +===================== ========= ======= =========== ======== ======= +Distribution MT19937 PCG64 PCG64DXSM Philox SFC64 +===================== ========= ======= =========== ======== ======= +32-bit Unsigned Ints 100 168 166 100 182 +64-bit Unsigned Ints 100 176 193 116 224 +Uniforms 100 188 202 118 228 +Normals 100 128 132 115 167 +Exponentials 100 152 157 111 168 +Overall 100 161 168 112 192 +===================== ========= ======= =========== ======== ======= 64-bit Windows ~~~~~~~~~~~~~~ -The relative performance on 64-bit Linux and 64-bit Windows is broadly similar. - +The relative performance on 64-bit Linux and 64-bit Windows is broadly similar +with the notable exception of the Philox generator. -=================== ========= ======= ======== ======= -Distribution MT19937 PCG64 Philox SFC64 -=================== ========= ======= ======== ======= -32-bit Unsigned Int 100 129.1 35.0 135.0 -64-bit Unsigned Int 100 146.9 35.7 176.5 -Uniforms 100 165.0 37.0 192.0 -Normals 100 128.5 48.5 158.0 -Exponentials 100 151.6 39.0 172.8 -**Overall** 100 143.6 38.7 165.7 -=================== ========= ======= ======== ======= +===================== ========= ======= =========== ======== ======= +Distribution MT19937 PCG64 PCG64DXSM Philox SFC64 +===================== ========= ======= =========== ======== ======= +32-bit Unsigned Ints 100 155 131 29 150 +64-bit Unsigned Ints 100 157 143 25 154 +Uniforms 100 151 144 24 155 +Normals 100 129 128 37 150 +Exponentials 100 150 145 28 159 +**Overall** 100 148 138 28 154 +===================== ========= ======= =========== ======== ======= 32-bit Windows @@ -134,20 +136,20 @@ The performance of 64-bit generators on 32-bit Windows is much lower than on 64- operating systems due to register width. MT19937, the generator that has been in NumPy since 2005, operates on 32-bit integers. -=================== ========= ======= ======== ======= -Distribution MT19937 PCG64 Philox SFC64 -=================== ========= ======= ======== ======= -32-bit Unsigned Int 100 30.5 21.1 77.9 -64-bit Unsigned Int 100 26.3 19.2 97.0 -Uniforms 100 28.0 23.0 106.0 -Normals 100 40.1 31.3 112.6 -Exponentials 100 33.7 26.3 109.8 -**Overall** 100 31.4 23.8 99.8 -=================== ========= ======= ======== ======= +===================== ========= ======= =========== ======== ======= +Distribution MT19937 PCG64 PCG64DXSM Philox SFC64 +===================== ========= ======= =========== ======== ======= +32-bit Unsigned Ints 100 24 34 14 57 +64-bit Unsigned Ints 100 21 32 14 74 +Uniforms 100 21 34 16 73 +Normals 100 36 57 28 101 +Exponentials 100 28 44 20 88 +**Overall** 100 25 39 18 77 +===================== ========= ======= =========== ======== ======= .. note:: - Linux timings used Ubuntu 18.04 and GCC 7.4. Windows timings were made on + Linux timings used Ubuntu 20.04 and GCC 9.3.0. Windows timings were made on Windows 10 using Microsoft C/C++ Optimizing Compiler Version 19 (Visual - Studio 2015). All timings were produced on an i5-3570 processor. + Studio 2019). All timings were produced on an AMD Ryzen 9 3900X processor. diff --git a/doc/source/reference/random/upgrading-pcg64.rst b/doc/source/reference/random/upgrading-pcg64.rst new file mode 100644 index 000000000000..9e540ace9331 --- /dev/null +++ b/doc/source/reference/random/upgrading-pcg64.rst @@ -0,0 +1,152 @@ +.. _upgrading-pcg64: + +.. currentmodule:: numpy.random + +Upgrading ``PCG64`` with ``PCG64DXSM`` +-------------------------------------- + +Uses of the `PCG64` `BitGenerator` in a massively-parallel context have been +shown to have statistical weaknesses that were not apparent at the first +release in numpy 1.17. Most users will never observe this weakness and are +safe to continue to use `PCG64`. We have introduced a new `PCG64DXSM` +`BitGenerator` that will eventually become the new default `BitGenerator` +implementation used by `default_rng` in future releases. `PCG64DXSM` solves +the statistical weakness while preserving the performance and the features of +`PCG64`. + +Does this affect me? +==================== + +If you + + 1. only use a single `Generator` instance, + 2. only use `RandomState` or the functions in `numpy.random`, + 3. only use the `PCG64.jumped` method to generate parallel streams, + 4. explicitly use a `BitGenerator` other than `PCG64`, + +then this weakness does not affect you at all. Carry on. + +If you use moderate numbers of parallel streams created with `default_rng` or +`SeedSequence.spawn`, in the 1000s, then the chance of observing this weakness +is negligibly small. You can continue to use `PCG64` comfortably. + +If you use very large numbers of parallel streams, in the millions, and draw +large amounts of numbers from each, then the chance of observing this weakness +can become non-negligible, if still small. An example of such a use case would +be a very large distributed reinforcement learning problem with millions of +long Monte Carlo playouts each generating billions of random number draws. Such +use cases should consider using `PCG64DXSM` explicitly or another +modern `BitGenerator` like `SFC64` or `Philox`, but it is unlikely that any +old results you may have calculated are invalid. In any case, the weakness is +a kind of `Birthday Paradox `_ +collision. That is, a single pair of parallel streams out of the millions, +considered together, might fail a stringent set of statistical tests of +randomness. The remaining millions of streams would all be perfectly fine, and +the effect of the bad pair in the whole calculation is very likely to be +swamped by the remaining streams in most applications. + +.. _upgrading-pcg64-details: + +Technical Details +================= + +Like many PRNG algorithms, `PCG64` is constructed from a transition function, +which advances a 128-bit state, and an output function, that mixes the 128-bit +state into a 64-bit integer to be output. One of the guiding design principles +of the PCG family of PRNGs is to balance the computational cost (and +pseudorandomness strength) between the transition function and the output +function. The transition function is a 128-bit linear congruential generator +(LCG), which consists of multiplying the 128-bit state with a fixed +multiplication constant and then adding a user-chosen increment, in 128-bit +modular arithmetic. LCGs are well-analyzed PRNGs with known weaknesses, though +128-bit LCGs are large enough to pass stringent statistical tests on their own, +with only the trivial output function. The output function of `PCG64` is +intended to patch up some of those known weaknesses by doing "just enough" +scrambling of the bits to assist in the statistical properties without adding +too much computational cost. + +One of these known weaknesses is that advancing the state of the LCG by steps +numbering a power of two (``bg.advance(2**N)``) will leave the lower ``N`` bits +identical to the state that was just left. For a single stream drawn from +sequentially, this is of little consequence. The remaining :math:`128-N` bits provide +plenty of pseudorandomness that will be mixed in for any practical ``N`` that can +be observed in a single stream, which is why one does not need to worry about +this if you only use a single stream in your application. Similarly, the +`PCG64.jumped` method uses a carefully chosen number of steps to avoid creating +these collisions. However, once you start creating "randomly-initialized" +parallel streams, either using OS entropy by calling `default_rng` repeatedly +or using `SeedSequence.spawn`, then we need to consider how many lower bits +need to "collide" in order to create a bad pair of streams, and then evaluate +the probability of creating such a collision. +`Empirically `_, it has been +determined that if one shares the lower 58 bits of state and shares an +increment, then the pair of streams, when interleaved, will fail +`PractRand `_ in +a reasonable amount of time, after drawing a few gigabytes of data. Following +the standard Birthday Paradox calculations for a collision of 58 bits, we can +see that we can create :math:`2^{29}`, or about half a billion, streams which is when +the probability of such a collision becomes high. Half a billion streams is +quite high, and the amount of data each stream needs to draw before the +statistical correlations become apparent to even the strict ``PractRand`` tests +is in the gigabytes. But this is on the horizon for very large applications +like distributed reinforcement learning. There are reasons to expect that even +in these applications a collision probably will not have a practical effect in +the total result, since the statistical problem is constrained to just the +colliding pair. + +Now, let us consider the case when the increment is not constrained to be the +same. Our implementation of `PCG64` seeds both the state and the increment; +that is, two calls to `default_rng` (almost certainly) have different states +and increments. Upon our first release, we believed that having the seeded +increment would provide a certain amount of extra protection, that one would +have to be "close" in both the state space and increment space in order to +observe correlations (``PractRand`` failures) in a pair of streams. If that were +true, then the "bottleneck" for collisions would be the 128-bit entropy pool +size inside of `SeedSequence` (and 128-bit collisions are in the +"preposterously unlikely" category). Unfortunately, this is not true. + +One of the known properties of an LCG is that different increments create +*distinct* streams, but with a known relationship. Each LCG has an orbit that +traverses all :math:`2^{128}` different 128-bit states. Two LCGs with different +increments are related in that one can "rotate" the orbit of the first LCG +(advance it by a number of steps that we can compute from the two increments) +such that then both LCGs will always then have the same state, up to an +additive constant and maybe an inversion of the bits. If you then iterate both +streams in lockstep, then the states will *always* remain related by that same +additive constant (and the inversion, if present). Recall that `PCG64` is +constructed from both a transition function (the LCG) and an output function. +It was expected that the scrambling effect of the output function would have +been strong enough to make the distinct streams practically independent (i.e. +"passing the ``PractRand`` tests") unless the two increments were +pathologically related to each other (e.g. 1 and 3). The output function XSL-RR +of the then-standard PCG algorithm that we implemented in `PCG64` turns out to +be too weak to cover up for the 58-bit collision of the underlying LCG that we +described above. For any given pair of increments, the size of the "colliding" +space of states is the same, so for this weakness, the extra distinctness +provided by the increments does not translate into extra protection from +statistical correlations that ``PractRand`` can detect. + +Fortunately, strengthening the output function is able to correct this weakness +and *does* turn the extra distinctness provided by differing increments into +additional protection from these low-bit collisions. To the `PCG author's +credit `_, +she had developed a stronger output function in response to related discussions +during the long birth of the new `BitGenerator` system. We NumPy developers +chose to be "conservative" and use the XSL-RR variant that had undergone +a longer period of testing at that time. The DXSM output function adopts +a "xorshift-multiply" construction used in strong integer hashes that has much +better avalanche properties than the XSL-RR output function. While there are +"pathological" pairs of increments that induce "bad" additive constants that +relate the two streams, the vast majority of pairs induce "good" additive +constants that make the merely-distinct streams of LCG states into +practically-independent output streams. Indeed, now the claim we once made +about `PCG64` is actually true of `PCG64DXSM`: collisions are possible, but +both streams have to simultaneously be both "close" in the 128 bit state space +*and* "close" in the 127-bit increment space, so that would be less likely than +the negligible chance of colliding in the 128-bit internal `SeedSequence` pool. +The DXSM output function is more computationally intensive than XSL-RR, but +some optimizations in the LCG more than make up for the performance hit on most +machines, so `PCG64DXSM` is a good, safe upgrade. There are, of course, an +infinite number of stronger output functions that one could consider, but most +will have a greater computational cost, and the DXSM output function has now +received many CPU cycles of testing via ``PractRand`` at this time. diff --git a/doc/source/reference/routines.array-creation.rst b/doc/source/reference/routines.array-creation.rst index e718f0052872..30780c286c41 100644 --- a/doc/source/reference/routines.array-creation.rst +++ b/doc/source/reference/routines.array-creation.rst @@ -7,8 +7,8 @@ Array creation routines .. currentmodule:: numpy -Ones and zeros --------------- +From shape or value +------------------- .. autosummary:: :toctree: generated/ diff --git a/doc/source/reference/routines.linalg.rst b/doc/source/reference/routines.linalg.rst index 86e168b262a2..76b7ab82c940 100644 --- a/doc/source/reference/routines.linalg.rst +++ b/doc/source/reference/routines.linalg.rst @@ -30,10 +30,26 @@ flexible broadcasting options. For example, `numpy.linalg.solve` can handle "stacked" arrays, while `scipy.linalg.solve` accepts only a single square array as its first argument. +.. note:: + + The term *matrix* as it is used on this page indicates a 2d `numpy.array` + object, and *not* a `numpy.matrix` object. The latter is no longer + recommended, even for linear algebra. See + :ref:`the matrix object documentation` for + more information. + +The ``@`` operator +------------------ + +Introduced in NumPy 1.10.0, the ``@`` operator is preferable to +other methods when computing the matrix product between 2d arrays. The +:func:`numpy.matmul` function implements the ``@`` operator. + .. currentmodule:: numpy Matrix and vector products -------------------------- + .. autosummary:: :toctree: generated/ diff --git a/doc/source/reference/routines.polynomials.classes.rst b/doc/source/reference/routines.polynomials.classes.rst index 10331e9c17a4..5f575bed13d4 100644 --- a/doc/source/reference/routines.polynomials.classes.rst +++ b/doc/source/reference/routines.polynomials.classes.rst @@ -290,7 +290,8 @@ polynomials up to degree 5 are plotted below. >>> import matplotlib.pyplot as plt >>> from numpy.polynomial import Chebyshev as T >>> x = np.linspace(-1, 1, 100) - >>> for i in range(6): ax = plt.plot(x, T.basis(i)(x), lw=2, label="$T_%d$"%i) + >>> for i in range(6): + ... ax = plt.plot(x, T.basis(i)(x), lw=2, label=f"$T_{i}$") ... >>> plt.legend(loc="upper left") @@ -304,7 +305,8 @@ The same plots over the range -2 <= `x` <= 2 look very different: >>> import matplotlib.pyplot as plt >>> from numpy.polynomial import Chebyshev as T >>> x = np.linspace(-2, 2, 100) - >>> for i in range(6): ax = plt.plot(x, T.basis(i)(x), lw=2, label="$T_%d$"%i) + >>> for i in range(6): + ... ax = plt.plot(x, T.basis(i)(x), lw=2, label=f"$T_{i}$") ... >>> plt.legend(loc="lower right") diff --git a/doc/source/reference/routines.polynomials.rst b/doc/source/reference/routines.polynomials.rst index e74c5a683b57..ecfb012f0c6a 100644 --- a/doc/source/reference/routines.polynomials.rst +++ b/doc/source/reference/routines.polynomials.rst @@ -9,24 +9,165 @@ of the `numpy.polynomial` package, introduced in NumPy 1.4. Prior to NumPy 1.4, `numpy.poly1d` was the class of choice and it is still available in order to maintain backward compatibility. -However, the newer Polynomial package is more complete than `numpy.poly1d` -and its convenience classes are better behaved in the numpy environment. +However, the newer `polynomial package ` is more complete +and its `convenience classes ` provide a +more consistent, better-behaved interface for working with polynomial +expressions. Therefore :mod:`numpy.polynomial` is recommended for new coding. -Transition notice ------------------ -The various routines in the Polynomial package all deal with -series whose coefficients go from degree zero upward, -which is the *reverse order* of the Poly1d convention. -The easy way to remember this is that indexes -correspond to degree, i.e., coef[i] is the coefficient of the term of -degree i. +.. note:: **Terminology** + The term *polynomial module* refers to the old API defined in + `numpy.lib.polynomial`, which includes the :class:`numpy.poly1d` class and + the polynomial functions prefixed with *poly* accessible from the `numpy` + namespace (e.g. `numpy.polyadd`, `numpy.polyval`, `numpy.polyfit`, etc.). + + The term *polynomial package* refers to the new API definied in + `numpy.polynomial`, which includes the convenience classes for the + different kinds of polynomials (`numpy.polynomial.Polynomial`, + `numpy.polynomial.Chebyshev`, etc.). + +Transitioning from `numpy.poly1d` to `numpy.polynomial` +------------------------------------------------------- + +As noted above, the :class:`poly1d class ` and associated +functions defined in ``numpy.lib.polynomial``, such as `numpy.polyfit` +and `numpy.poly`, are considered legacy and should **not** be used in new +code. +Since NumPy version 1.4, the `numpy.polynomial` package is preferred for +working with polynomials. + +Quick Reference +~~~~~~~~~~~~~~~ + +The following table highlights some of the main differences between the +legacy polynomial module and the polynomial package for common tasks. +The `~numpy.polynomial.polynomial.Polynomial` class is imported for brevity:: + + from numpy.polynomial import Polynomial + + ++------------------------+------------------------------+---------------------------------------+ +| **How to...** | Legacy (`numpy.poly1d`) | `numpy.polynomial` | ++------------------------+------------------------------+---------------------------------------+ +| Create a | ``p = np.poly1d([1, 2, 3])`` | ``p = Polynomial([3, 2, 1])`` | +| polynomial object | | | +| from coefficients [1]_ | | | ++------------------------+------------------------------+---------------------------------------+ +| Create a polynomial | ``r = np.poly([-1, 1])`` | ``p = Polynomial.fromroots([-1, 1])`` | +| object from roots | ``p = np.poly1d(r)`` | | ++------------------------+------------------------------+---------------------------------------+ +| Fit a polynomial of | | | +| degree ``deg`` to data | ``np.polyfit(x, y, deg)`` | ``Polynomial.fit(x, y, deg)`` | ++------------------------+------------------------------+---------------------------------------+ + + +.. [1] Note the reversed ordering of the coefficients + +Transition Guide +~~~~~~~~~~~~~~~~ + +There are significant differences between ``numpy.lib.polynomial`` and +`numpy.polynomial`. +The most significant difference is the ordering of the coefficients for the +polynomial expressions. +The various routines in `numpy.polynomial` all +deal with series whose coefficients go from degree zero upward, +which is the *reverse order* of the poly1d convention. +The easy way to remember this is that indices +correspond to degree, i.e., ``coef[i]`` is the coefficient of the term of +degree *i*. + +Though the difference in convention may be confusing, it is straightforward to +convert from the legacy polynomial API to the new. +For example, the following demonstrates how you would convert a `numpy.poly1d` +instance representing the expression :math:`x^{2} + 2x + 3` to a +`~numpy.polynomial.polynomial.Polynomial` instance representing the same +expression:: + + >>> p1d = np.poly1d([1, 2, 3]) + >>> p = np.polynomial.Polynomial(p1d.coef[::-1]) + +In addition to the ``coef`` attribute, polynomials from the polynomial +package also have ``domain`` and ``window`` attributes. +These attributes are most relevant when fitting +polynomials to data, though it should be noted that polynomials with +different ``domain`` and ``window`` attributes are not considered equal, and +can't be mixed in arithmetic:: + + >>> p1 = np.polynomial.Polynomial([1, 2, 3]) + >>> p1 + Polynomial([1., 2., 3.], domain=[-1, 1], window=[-1, 1]) + >>> p2 = np.polynomial.Polynomial([1, 2, 3], domain=[-2, 2]) + >>> p1 == p2 + False + >>> p1 + p2 + Traceback (most recent call last): + ... + TypeError: Domains differ + +See the documentation for the +`convenience classes `_ for further details on +the ``domain`` and ``window`` attributes. + +Another major difference bewteen the legacy polynomial module and the +polynomial package is polynomial fitting. In the old module, fitting was +done via the `~numpy.polyfit` function. In the polynomial package, the +`~numpy.polynomial.polynomial.Polynomial.fit` class method is preferred. For +example, consider a simple linear fit to the following data: + +.. ipython:: python + + rng = np.random.default_rng() + x = np.arange(10) + y = np.arange(10) + rng.standard_normal(10) + +With the legacy polynomial module, a linear fit (i.e. polynomial of degree 1) +could be applied to these data with `~numpy.polyfit`: + +.. ipython:: python + + np.polyfit(x, y, deg=1) + +With the new polynomial API, the `~numpy.polynomial.polynomial.Polynomial.fit` +class method is preferred: + +.. ipython:: python + + p_fitted = np.polynomial.Polynomial.fit(x, y, deg=1) + p_fitted + +Note that the coefficients are given *in the scaled domain* defined by the +linear mapping between the ``window`` and ``domain``. +`~numpy.polynomial.polynomial.Polynomial.convert` can be used to get the +coefficients in the unscaled data domain. + +.. ipython:: python + + p_fitted.convert() + +Documentation for the `~numpy.polynomial` Package +------------------------------------------------- + +In addition to standard power series polynomials, the polynomial package +provides several additional kinds of polynomials including Chebyshev, +Hermite (two subtypes), Laguerre, and Legendre polynomials. +Each of these has an associated +`convenience class ` available from the +`numpy.polynomial` namespace that provides a consistent interface for working +with polynomials regardless of their type. .. toctree:: :maxdepth: 1 routines.polynomials.classes + +Documentation pertaining to specific functions defined for each kind of +polynomial individually can be found in the corresponding module documentation: + +.. toctree:: + :maxdepth: 1 + routines.polynomials.polynomial routines.polynomials.chebyshev routines.polynomials.hermite @@ -36,6 +177,9 @@ degree i. routines.polynomials.polyutils +Documentation for Legacy Polynomials +------------------------------------ + .. toctree:: :maxdepth: 2 diff --git a/doc/source/reference/routines.testing.rst b/doc/source/reference/routines.testing.rst index 98ce3f37718b..d9e98e94188d 100644 --- a/doc/source/reference/routines.testing.rst +++ b/doc/source/reference/routines.testing.rst @@ -18,9 +18,6 @@ Asserts .. autosummary:: :toctree: generated/ - assert_almost_equal - assert_approx_equal - assert_array_almost_equal assert_allclose assert_array_almost_equal_nulp assert_array_max_ulp @@ -32,6 +29,19 @@ Asserts assert_warns assert_string_equal +Asserts (not recommended) +------------------------- +It is recommended to use one of `assert_allclose`, +`assert_array_almost_equal_nulp` or `assert_array_max_ulp` instead of these +functions for more consistent floating point comparisons. + +.. autosummary:: + :toctree: generated/ + + assert_almost_equal + assert_approx_equal + assert_array_almost_equal + Decorators ---------- .. autosummary:: diff --git a/doc/source/reference/simd/simd-optimizations.py b/doc/source/reference/simd/simd-optimizations.py index 5d6da50e3aff..a78302db5e89 100644 --- a/doc/source/reference/simd/simd-optimizations.py +++ b/doc/source/reference/simd/simd-optimizations.py @@ -8,7 +8,7 @@ from numpy.distutils.ccompiler_opt import CCompilerOpt class FakeCCompilerOpt(CCompilerOpt): - fake_info = "" + fake_info = ("arch", "compiler", "extra_args") # disable caching no need for it conf_nocache = True def __init__(self, *args, **kwargs): @@ -101,7 +101,7 @@ def features_table_sections(name, ftable=None, gtable=None, tab_size=4): return content def features_table(arch, cc="gcc", pretty_name=None, **kwargs): - FakeCCompilerOpt.fake_info = arch + cc + FakeCCompilerOpt.fake_info = (arch, cc, '') ccopt = FakeCCompilerOpt(cpu_baseline="max") features = ccopt.cpu_baseline_names() ftable = ccopt.gen_features_table(features, **kwargs) @@ -112,12 +112,12 @@ def features_table(arch, cc="gcc", pretty_name=None, **kwargs): return features_table_sections(pretty_name, ftable, gtable, **kwargs) def features_table_diff(arch, cc, cc_vs="gcc", pretty_name=None, **kwargs): - FakeCCompilerOpt.fake_info = arch + cc + FakeCCompilerOpt.fake_info = (arch, cc, '') ccopt = FakeCCompilerOpt(cpu_baseline="max") fnames = ccopt.cpu_baseline_names() features = {f:ccopt.feature_implies(f) for f in fnames} - FakeCCompilerOpt.fake_info = arch + cc_vs + FakeCCompilerOpt.fake_info = (arch, cc_vs, '') ccopt_vs = FakeCCompilerOpt(cpu_baseline="max") fnames_vs = ccopt_vs.cpu_baseline_names() features_vs = {f:ccopt_vs.feature_implies(f) for f in fnames_vs} diff --git a/doc/source/reference/simd/simd-optimizations.rst b/doc/source/reference/simd/simd-optimizations.rst index 59a4892b24d8..956824321023 100644 --- a/doc/source/reference/simd/simd-optimizations.rst +++ b/doc/source/reference/simd/simd-optimizations.rst @@ -96,8 +96,8 @@ NOTES arguments must be enclosed in quotes. - The operand ``+`` is only added for nominal reasons, For example: - ``--cpu-basline= "min avx2"`` is equivalent to ``--cpu-basline="min + avx2"``. - ``--cpu-basline="min,avx2"`` is equivalent to ``--cpu-basline`="min,+avx2"`` + ``--cpu-baseline= "min avx2"`` is equivalent to ``--cpu-baseline="min + avx2"``. + ``--cpu-baseline="min,avx2"`` is equivalent to ``--cpu-baseline`="min,+avx2"`` - If the CPU feature is not supported by the user platform or compiler, it will be skipped rather than raising a fatal error. diff --git a/doc/source/reference/ufuncs.rst b/doc/source/reference/ufuncs.rst index 06fbe28ddcba..3eae4e1598bb 100644 --- a/doc/source/reference/ufuncs.rst +++ b/doc/source/reference/ufuncs.rst @@ -266,7 +266,7 @@ can generate this table for your system with the code given in the Figure. S - - - - - - - - - - - - - - - - - - - - Y Y Y Y - - U - - - - - - - - - - - - - - - - - - - - - Y Y Y - - V - - - - - - - - - - - - - - - - - - - - - - Y Y - - - O - - - - - - - - - - - - - - - - - - - - - - Y Y - - + O - - - - - - - - - - - - - - - - - - - - - - - Y - - M - - - - - - - - - - - - - - - - - - - - - - Y Y Y - m - - - - - - - - - - - - - - - - - - - - - - Y Y - Y @@ -430,8 +430,10 @@ advanced usage and will not typically be used. .. versionadded:: 1.6 - Overrides the dtype of the calculation and output arrays. Similar to - *signature*. + Overrides the DType of the output arrays the same way as the *signature*. + This should ensure a matching precision of the calculation. The exact + calculation DTypes chosen may depend on the ufunc and the inputs may be + cast to this DType to perform the calculation. *subok* @@ -442,20 +444,31 @@ advanced usage and will not typically be used. *signature* - Either a data-type, a tuple of data-types, or a special signature - string indicating the input and output types of a ufunc. This argument - allows you to provide a specific signature for the 1-d loop to use - in the underlying calculation. If the loop specified does not exist - for the ufunc, then a TypeError is raised. Normally, a suitable loop is - found automatically by comparing the input types with what is - available and searching for a loop with data-types to which all inputs - can be cast safely. This keyword argument lets you bypass that - search and choose a particular loop. A list of available signatures is - provided by the **types** attribute of the ufunc object. For backwards - compatibility this argument can also be provided as *sig*, although - the long form is preferred. Note that this should not be confused with - the generalized ufunc :ref:`signature ` that is - stored in the **signature** attribute of the of the ufunc object. + Either a Dtype, a tuple of DTypes, or a special signature string + indicating the input and output types of a ufunc. + + This argument allows the user to specify exact DTypes to be used for the + calculation. Casting will be used as necessary. The actual DType of the + input arrays is not considered unless ``signature`` is ``None`` for + that array. + + When all DTypes are fixed, a specific loop is chosen or an error raised + if no matching loop exists. + If some DTypes are not specified and left ``None``, the behaviour may + depend on the ufunc. + At this time, a list of available signatures is provided by the **types** + attribute of the ufunc. (This list may be missing DTypes not defined + by NumPy.) + + The ``signature`` only specifies the DType class/type. For example, it + can specifiy that the operation should be ``datetime64`` or ``float64`` + operation. It does not specify the ``datetime64`` time-unit or the + ``float64`` byte-order. + + For backwards compatibility this argument can also be provided as *sig*, + although the long form is preferred. Note that this should not be + confused with the generalized ufunc :ref:`signature ` + that is stored in the **signature** attribute of the of the ufunc object. *extobj* @@ -628,8 +641,8 @@ Math operations for large calculations. If your arrays are large, complicated expressions can take longer than absolutely necessary due to the creation and (later) destruction of temporary calculation - spaces. For example, the expression ``G = a * b + c`` is equivalent to - ``t1 = A * B; G = T1 + C; del t1``. It will be more quickly executed + spaces. For example, the expression ``G = A * B + C`` is equivalent to + ``T1 = A * B; G = T1 + C; del T1``. It will be more quickly executed as ``G = A * B; add(G, C, G)`` which is the same as ``G = A * B; G += C``. diff --git a/doc/source/release.rst b/doc/source/release.rst index 3ef1b06bd34a..6d208d395b90 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -5,12 +5,18 @@ Release Notes .. toctree:: :maxdepth: 3 + 1.21.0 + 1.20.3 + 1.20.2 + 1.20.1 1.20.0 + 1.19.5 1.19.4 1.19.3 1.19.2 1.19.1 1.19.0 + 1.18.5 1.18.4 1.18.3 1.18.2 diff --git a/doc/source/release/1.18.5-notes.rst b/doc/source/release/1.18.5-notes.rst new file mode 100644 index 000000000000..e704c001a452 --- /dev/null +++ b/doc/source/release/1.18.5-notes.rst @@ -0,0 +1,31 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.18.5 Release Notes +========================== + +This is a short release to allow pickle ``protocol=5`` to be used in +Python3.5. It is motivated by the recent backport of pickle5 to Python3.5. + +The Python versions supported in this release are 3.5-3.8. Downstream +developers should use Cython >= 0.29.15 for Python 3.8 support and +OpenBLAS >= 3.7 to avoid errors on the Skylake architecture. + +Contributors +============ + +A total of 3 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Matti Picus +* Siyuan Zhuang + + +Pull requests merged +==================== + +A total of 2 pull requests were merged for this release. + +* `#16439 `__: ENH: enable pickle protocol 5 support for python3.5 +* `#16441 `__: BUG: relpath fails for different drives on windows + diff --git a/doc/source/release/1.19.5-notes.rst b/doc/source/release/1.19.5-notes.rst new file mode 100644 index 000000000000..048f2718cddf --- /dev/null +++ b/doc/source/release/1.19.5-notes.rst @@ -0,0 +1,42 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.19.5 Release Notes +========================== + +NumPy 1.19.5 is a short bugfix release. Apart from fixing several bugs, the +main improvement is the update to OpenBLAS 0.3.13 that works around the windows +2004 bug while not breaking execution on other platforms. This release supports +Python 3.6-3.9 and is planned to be the last release in the 1.19.x cycle. + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Charles Harris +* Christoph Gohlke +* Matti Picus +* Raghuveer Devulapalli +* Sebastian Berg +* Simon Graham + +* Veniamin Petrenko + +* Bernie Gray + + +Pull requests merged +==================== + +A total of 11 pull requests were merged for this release. + +* `#17756 `__: BUG: Fix segfault due to out of bound pointer in floatstatus... +* `#17774 `__: BUG: fix np.timedelta64('nat').__format__ throwing an exception +* `#17775 `__: BUG: Fixed file handle leak in array_tofile. +* `#17786 `__: BUG: Raise recursion error during dimension discovery +* `#17917 `__: BUG: Fix subarray dtype used with too large count in fromfile +* `#17918 `__: BUG: 'bool' object has no attribute 'ndim' +* `#17919 `__: BUG: ensure _UFuncNoLoopError can be pickled +* `#17924 `__: BLD: use BUFFERSIZE=20 in OpenBLAS +* `#18026 `__: BLD: update to OpenBLAS 0.3.13 +* `#18036 `__: BUG: make a variable volatile to work around clang compiler bug +* `#18114 `__: REL: Prepare for the NumPy 1.19.5 release. diff --git a/doc/source/release/1.20.0-notes.rst b/doc/source/release/1.20.0-notes.rst index 73b470f3fc29..b8b7a0c79a6e 100644 --- a/doc/source/release/1.20.0-notes.rst +++ b/doc/source/release/1.20.0-notes.rst @@ -3,8 +3,8 @@ ========================== NumPy 1.20.0 Release Notes ========================== -This NumPy release is the largest so made to date, some 648 PRs contributed by -182 people have been merged. See the list of highlights below for more details. +This NumPy release is the largest so made to date, some 684 PRs contributed by +184 people have been merged. See the list of highlights below for more details. The Python versions supported for this release are 3.7-3.9, support for Python 3.6 has been dropped. Highlights are @@ -75,32 +75,60 @@ Using the aliases of builtin types like ``np.int`` is deprecated ---------------------------------------------------------------- For a long time, ``np.int`` has been an alias of the builtin ``int``. This is -repeatedly a cause of confusion for newcomers, and is also simply not useful. +repeatedly a cause of confusion for newcomers, and existed mainly for historic +reasons. These aliases have been deprecated. The table below shows the full list of deprecated aliases, along with their exact meaning. Replacing uses of items in the first column with the contents of the second column will work identically and silence the deprecation warning. -In many cases, it may have been intended to use the types from the third column. -Be aware that use of these types may result in subtle but desirable behavior -changes. - -================== ================================= ================================================================== -Deprecated name Identical to Possibly intended numpy type -================== ================================= ================================================================== -``numpy.bool`` ``bool`` `numpy.bool_` -``numpy.int`` ``int`` `numpy.int_` (default int dtype), `numpy.cint` (C ``int``) -``numpy.float`` ``float`` `numpy.float_`, `numpy.double` (equivalent) -``numpy.complex`` ``complex`` `numpy.complex_`, `numpy.cdouble` (equivalent) -``numpy.object`` ``object`` `numpy.object_` -``numpy.str`` ``str`` `numpy.str_` -``numpy.long`` ``int`` (``long`` on Python 2) `numpy.int_` (C ``long``), `numpy.longlong` (largest integer type) -``numpy.unicode`` ``str`` (``unicode`` on Python 2) `numpy.unicode_` -================== ================================= ================================================================== - -Note that for technical reasons these deprecation warnings will only be emitted -on Python 3.7 and above. +The third column lists alternative NumPy names which may occasionally be +preferential. See also :ref:`basics.types` for additional details. + +================= ============ ================================================================== +Deprecated name Identical to NumPy scalar type names +================= ============ ================================================================== +``numpy.bool`` ``bool`` `numpy.bool_` +``numpy.int`` ``int`` `numpy.int_` (default), ``numpy.int64``, or ``numpy.int32`` +``numpy.float`` ``float`` `numpy.float64`, `numpy.float_`, `numpy.double` (equivalent) +``numpy.complex`` ``complex`` `numpy.complex128`, `numpy.complex_`, `numpy.cdouble` (equivalent) +``numpy.object`` ``object`` `numpy.object_` +``numpy.str`` ``str`` `numpy.str_` +``numpy.long`` ``int`` `numpy.int_` (C ``long``), `numpy.longlong` (largest integer type) +``numpy.unicode`` ``str`` `numpy.unicode_` +================= ============ ================================================================== + +To give a clear guideline for the vast majority of cases, for the types +``bool``, ``object``, ``str`` (and ``unicode``) using the plain version +is shorter and clear, and generally a good replacement. +For ``float`` and ``complex`` you can use ``float64`` and ``complex128`` +if you wish to be more explicit about the precision. + +For ``np.int`` a direct replacement with ``np.int_`` or ``int`` is also +good and will not change behavior, but the precision will continue to depend +on the computer and operating system. +If you want to be more explicit and review the current use, you have the +following alternatives: + +* ``np.int64`` or ``np.int32`` to specify the precision exactly. + This ensures that results cannot depend on the computer or operating system. +* ``np.int_`` or ``int`` (the default), but be aware that it depends on + the computer and operating system. +* The C types: ``np.cint`` (int), ``np.int_`` (long), ``np.longlong``. +* ``np.intp`` which is 32bit on 32bit machines 64bit on 64bit machines. + This can be the best type to use for indexing. + +When used with ``np.dtype(...)`` or ``dtype=...`` changing it to the +NumPy name as mentioned above will have no effect on the output. +If used as a scalar with:: + + np.float(123) + +changing it can subtly change the result. In this case, the Python version +``float(123)`` or ``int(12.)`` is normally preferable, although the NumPy +version may be useful for consistency with NumPy arrays (for example, +NumPy behaves differently for things like division by zero). (`gh-14882 `__) @@ -184,6 +212,43 @@ Use ``next(it)`` instead of ``it.ndincr()``. (`gh-17233 `__) +ArrayLike objects which do not define ``__len__`` and ``__getitem__`` +--------------------------------------------------------------------- +Objects which define one of the protocols ``__array__``, +``__array_interface__``, or ``__array_struct__`` but are not sequences +(usually defined by having a ``__len__`` and ``__getitem__``) will behave +differently during array-coercion in the future. + +When nested inside sequences, such as ``np.array([array_like])``, these +were handled as a single Python object rather than an array. +In the future they will behave identically to:: + + np.array([np.array(array_like)]) + +This change should only have an effect if ``np.array(array_like)`` is not 0-D. +The solution to this warning may depend on the object: + +* Some array-likes may expect the new behaviour, and users can ignore the + warning. The object can choose to expose the sequence protocol to opt-in + to the new behaviour. +* For example, ``shapely`` will allow conversion to an array-like using + ``line.coords`` rather than ``np.asarray(line)``. Users may work around + the warning, or use the new convention when it becomes available. + +Unfortunately, using the new behaviour can only be achieved by +calling ``np.array(array_like)``. + +If you wish to ensure that the old behaviour remains unchanged, please create +an object array and then fill it explicitly, for example:: + + arr = np.empty(3, dtype=object) + arr[:] = [array_like1, array_like2, array_like3] + +This will ensure NumPy knows to not enter the array-like and use it as +a object instead. + +(`gh-17973 `__) + Future Changes ============== @@ -253,6 +318,18 @@ library. Compatibility notes =================== +``isinstance(dtype, np.dtype)`` and not ``type(dtype) is not np.dtype`` +----------------------------------------------------------------------- +NumPy dtypes are not direct instances of ``np.dtype`` anymore. Code that +may have used ``type(dtype) is np.dtype`` will always return ``False`` and +must be updated to use the correct version ``isinstance(dtype, np.dtype)``. + +This change also affects the C-side macro ``PyArray_DescrCheck`` if compiled +against a NumPy older than 1.16.6. If code uses this macro and wishes to +compile against an older version of NumPy, it must replace the macro +(see also `C API changes`_ section). + + Same kind casting in concatenate with ``axis=None`` --------------------------------------------------- When `~numpy.concatenate` is called with ``axis=None``, @@ -337,9 +414,15 @@ Things will now be more consistent with:: np.array([np.array(array_like1)]) -This could potentially subtly change output for badly defined array-likes. -We are not aware of any such case where the results were not clearly -incorrect previously. +This can subtly change output for some badly defined array-likes. +One example for this are array-like objects which are not also sequences +of matching shape. +In NumPy 1.20, a warning will be given when an array-like is not also a +sequence (but behaviour remains identical, see deprecations). +If an array like is also a sequence (defines ``__getitem__`` and ``__len__``) +NumPy will now only use the result given by ``__array__``, +``__array_interface__``, or ``__array_struct__``. This will result in +differences when the (nested) sequence describes a different shape. (`gh-16200 `__) @@ -524,6 +607,23 @@ cannot represent ``b"1"`` faithfully. C API changes ============= +The ``PyArray_DescrCheck`` macro is modified +-------------------------------------------- +The ``PyArray_DescrCheck`` macro has been updated since NumPy 1.16.6 to be:: + + #define PyArray_DescrCheck(op) PyObject_TypeCheck(op, &PyArrayDescr_Type) + +Starting with NumPy 1.20 code that is compiled against an earlier version +will be API incompatible with NumPy 1.20. +The fix is to either compile against 1.16.6 (if the NumPy 1.16 release is +the oldest release you wish to support), or manually inline the macro by +replacing it with the new definition:: + + PyObject_TypeCheck(op, &PyArrayDescr_Type) + +which is compatible with all NumPy versions. + + Size of ``np.ndarray`` and ``np.void_`` changed ----------------------------------------------- The size of the ``PyArrayObject`` and ``PyVoidScalarObject`` @@ -900,5 +1000,3 @@ The former result can still be obtained with:: (`gh-16841 `__) - - diff --git a/doc/source/release/1.20.1-notes.rst b/doc/source/release/1.20.1-notes.rst new file mode 100644 index 000000000000..f95b5847ddcd --- /dev/null +++ b/doc/source/release/1.20.1-notes.rst @@ -0,0 +1,53 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.20.1 Release Notes +========================== + +NumPy 1,20.1 is a rapid bugfix release fixing several bugs and regressions +reported after the 1.20.0 release. + + +Highlights +========== + +- The distutils bug that caused problems with downstream projects is fixed. +- The ``random.shuffle`` regression is fixed. + + +Contributors +============ + +A total of 8 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Bas van Beek +* Charles Harris +* Nicholas McKibben + +* Pearu Peterson +* Ralf Gommers +* Sebastian Berg +* Tyler Reddy +* @Aerysv + + + +Pull requests merged +==================== + +A total of 15 pull requests were merged for this release. + +* `#18306 `__: MAINT: Add missing placeholder annotations +* `#18310 `__: BUG: Fix typo in ``numpy.__init__.py`` +* `#18326 `__: BUG: don't mutate list of fake libraries while iterating over... +* `#18327 `__: MAINT: gracefully shuffle memoryviews +* `#18328 `__: BUG: Use C linkage for random distributions +* `#18336 `__: CI: fix when GitHub Actions builds trigger, and allow ci skips +* `#18337 `__: BUG: Allow unmodified use of isclose, allclose, etc. with timedelta +* `#18345 `__: BUG: Allow pickling all relevant DType types/classes +* `#18351 `__: BUG: Fix missing signed_char dependency. Closes #18335. +* `#18352 `__: DOC: Change license date 2020 -> 2021 +* `#18353 `__: CI: CircleCI seems to occasionally time out, increase the limit +* `#18354 `__: BUG: Fix f2py bugs when wrapping F90 subroutines. +* `#18356 `__: MAINT: crackfortran regex simplify +* `#18357 `__: BUG: threads.h existence test requires GLIBC > 2.12. +* `#18359 `__: REL: Prepare for the NumPy 1.20.1 release. diff --git a/doc/source/release/1.20.2-notes.rst b/doc/source/release/1.20.2-notes.rst new file mode 100644 index 000000000000..10d39f7f6389 --- /dev/null +++ b/doc/source/release/1.20.2-notes.rst @@ -0,0 +1,48 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.20.2 Release Notes +========================== + +NumPy 1.20.2 is a bugfix release containing several fixes merged to the main +branch after the NumPy 1.20.1 release. + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Allan Haldane +* Bas van Beek +* Charles Harris +* Christoph Gohlke +* Mateusz Sokół + +* Michael Lamparski +* Sebastian Berg + +Pull requests merged +==================== + +A total of 20 pull requests were merged for this release. + +* `#18382 `__: MAINT: Update f2py from master. +* `#18459 `__: BUG: ``diagflat`` could overflow on windows or 32-bit platforms +* `#18460 `__: BUG: Fix refcount leak in f2py ``complex_double_from_pyobj``. +* `#18461 `__: BUG: Fix tiny memory leaks when ``like=`` overrides are used +* `#18462 `__: BUG: Remove temporary change of descr/flags in VOID functions +* `#18469 `__: BUG: Segfault in nditer buffer dealloc for Object arrays +* `#18485 `__: BUG: Remove suspicious type casting +* `#18486 `__: BUG: remove nonsensical comparison of pointer < 0 +* `#18487 `__: BUG: verify pointer against NULL before using it +* `#18488 `__: BUG: check if PyArray_malloc succeeded +* `#18546 `__: BUG: incorrect error fallthrough in nditer +* `#18559 `__: CI: Backport CI fixes from main. +* `#18599 `__: MAINT: Add annotations for `dtype.__getitem__`, `__mul__` and... +* `#18611 `__: BUG: NameError in numpy.distutils.fcompiler.compaq +* `#18612 `__: BUG: Fixed ``where`` keyword for ``np.mean`` & ``np.var`` methods +* `#18617 `__: CI: Update apt package list before Python install +* `#18636 `__: MAINT: Ensure that re-exported sub-modules are properly annotated +* `#18638 `__: BUG: Fix ma coercion list-of-ma-arrays if they do not cast to... +* `#18661 `__: BUG: Fix small valgrind-found issues +* `#18671 `__: BUG: Fix small issues found with pytest-leaks diff --git a/doc/source/release/1.20.3-notes.rst b/doc/source/release/1.20.3-notes.rst new file mode 100644 index 000000000000..8c25b3cc3215 --- /dev/null +++ b/doc/source/release/1.20.3-notes.rst @@ -0,0 +1,43 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.20.3 Release Notes +========================== + +NumPy 1.20.3 is a bugfix release containing several fixes merged to the main +branch after the NumPy 1.20.2 release. + +Contributors +============ + +A total of 7 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* Anne Archibald +* Bas van Beek +* Charles Harris +* Dong Keun Oh + +* Kamil Choudhury + +* Sayed Adel +* Sebastian Berg + +Pull requests merged +==================== + +A total of 15 pull requests were merged for this release. + +* `#18763 `__: BUG: Correct ``datetime64`` missing type overload for ``datetime.date``... +* `#18764 `__: MAINT: Remove ``__all__`` in favor of explicit re-exports +* `#18768 `__: BLD: Strip extra newline when dumping gfortran version on MacOS +* `#18769 `__: BUG: fix segfault in object/longdouble operations +* `#18794 `__: MAINT: Use towncrier build explicitly +* `#18887 `__: MAINT: Relax certain integer-type constraints +* `#18915 `__: MAINT: Remove unsafe unions and ABCs from return-annotations +* `#18921 `__: MAINT: Allow more recursion depth for scalar tests. +* `#18922 `__: BUG: Initialize the full nditer buffer in case of error +* `#18923 `__: BLD: remove unnecessary flag ``-faltivec`` on macOS +* `#18924 `__: MAINT, CI: treats _SIMD module build warnings as errors through... +* `#18925 `__: BUG: for MINGW, threads.h existence test requires GLIBC > 2.12 +* `#18941 `__: BUG: Make changelog recognize gh- as a PR number prefix. +* `#18948 `__: REL, DOC: Prepare for the NumPy 1.20.3 release. +* `#18953 `__: BUG: Fix failing mypy test in 1.20.x. diff --git a/doc/source/release/1.21.0-notes.rst b/doc/source/release/1.21.0-notes.rst new file mode 100644 index 000000000000..270cc32de918 --- /dev/null +++ b/doc/source/release/1.21.0-notes.rst @@ -0,0 +1,579 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.21.0 Release Notes +========================== +The NumPy 1.21.0 release highlights are + +* continued SIMD work covering more functions and platforms, +* initial work on the new dtype infrastructure and casting, +* universal2 wheels for Python 3.8 and Python 3.9 on Mac, +* improved documentation, +* improved annotations, +* new ``PCG64DXSM`` bitgenerator for random numbers. + +In addition there are the usual large number of bug fixes and other improvements. + +The Python versions supported for this release are 3.7-3.9. Official support +for Python 3.10 will be added when it is released. + +.. warning:: + There are unresolved problems compiling NumPy 1.20.0 with gcc-11.1. + + * Optimization level `-O3` results in many incorrect warnings when + running the tests. + * On some hardware NumPY will hang in an infinite loop. + + + + + +New functions +============= + +.. currentmodule:: numpy.random + +Add `PCG64DXSM` `BitGenerator` +------------------------------ + +Uses of the ``PCG64`` ``BitGenerator`` in a massively-parallel context have been +shown to have statistical weaknesses that were not apparent at the first +release in numpy 1.17. Most users will never observe this weakness and are +safe to continue to use ``PCG64``. We have introduced a new ``PCG64DXSM`` +``BitGenerator`` that will eventually become the new default ``BitGenerator`` +implementation used by ``default_rng`` in future releases. ``PCG64DXSM`` solves +the statistical weakness while preserving the performance and the features of +``PCG64``. + +See :ref:`upgrading-pcg64` for more details. + +.. currentmodule:: numpy + +(`gh-18906 `__) + + +Expired deprecations +==================== + +* The ``shape`` argument `~numpy.unravel_index` cannot be passed + as ``dims`` keyword argument anymore. (Was deprecated in NumPy 1.16.) + + (`gh-17900 `__) + +* The function ``PyUFunc_GenericFunction`` has been disabled. + It was deprecated in NumPy 1.19. Users should call the ufunc + directly using the Python API. + + (`gh-18697 `__) + +* The function ``PyUFunc_SetUsesArraysAsData`` has been disabled. + It was deprecated in NumPy 1.19. + + (`gh-18697 `__) + +* The class ``PolyBase`` has been removed (deprecated in numpy 1.9.0). Please + use the abstract ``ABCPolyBase`` class instead. + + (`gh-18963 `__) + +* The unused ``PolyError`` and ``PolyDomainError`` exceptions are + removed. + + (`gh-18963 `__) + + +Deprecations +============ + +The ``.dtype`` attribute must return a ``dtype`` +------------------------------------------------ + +A ``DeprecationWarning`` is now given if the ``.dtype`` attribute +of an object passed into ``np.dtype`` or as a ``dtype=obj`` argument +is not a dtype. NumPy will stop attempting to recursively coerce the +result of ``.dtype``. + +(`gh-13578 `__) + +Inexact matches for ``numpy.convolve`` and ``numpy.correlate`` are deprecated +----------------------------------------------------------------------------- + +`~numpy.convolve` and `~numpy.correlate` now emit a warning when there are case +insensitive and/or inexact matches found for ``mode`` argument in the functions. +Pass full ``"same"``, ``"valid"``, ``"full"`` strings instead of +``"s"``, ``"v"``, ``"f"`` for the ``mode`` argument. + +(`gh-17492 `__) + +``np.typeDict`` has been formally deprecated +-------------------------------------------- +``np.typeDict`` is a deprecated alias for ``np.sctypeDict`` and +has been so for over 14 years (6689502_). +A deprecation warning will now be issued whenever getting ``np.typeDict``. + +.. _6689502: https://github.com/numpy/numpy/commit/668950285c407593a368336ff2e737c5da84af7d + +(`gh-17586 `__) + +Exceptions will be raised during array-like creation +---------------------------------------------------- +When an object raised an exception during access of the special +attributes ``__array__`` or ``__array_interface__``, this exception +was usually ignored. +A warning is now given when the exception is anything but AttributeError. +To silence the warning, the type raising the exception has to be adapted +to raise an ``AttributeError``. + +(`gh-19001 `__) + +Four ``ndarray.ctypes`` methods have been deprecated +---------------------------------------------------- +Four methods of the `ndarray.ctypes` object have been deprecated, +as they are (undocumentated) implementation artifacts of their respective +properties. + +The methods in question are: + +* ``_ctypes.get_data`` (use ``_ctypes.data`` instead) +* ``_ctypes.get_shape`` (use ``_ctypes.shape`` instead) +* ``_ctypes.get_strides`` (use ``_ctypes.strides`` instead) +* ``_ctypes.get_as_parameter`` (use ``_ctypes._as_parameter_`` instead) + +(`gh-19031 `__) + + +Expired deprecations +==================== + +* The ``shape`` argument `numpy.unravel_index` cannot be passed + as ``dims`` keyword argument anymore. (Was deprecated in NumPy 1.16.) + + (`gh-17900 `__) + +* The function ``PyUFunc_GenericFunction`` has been disabled. + It was deprecated in NumPy 1.19. Users should call the ufunc + directly using the Python API. + + (`gh-18697 `__) + +* The function ``PyUFunc_SetUsesArraysAsData`` has been disabled. + It was deprecated in NumPy 1.19. + + (`gh-18697 `__) + +Remove deprecated ``PolyBase`` and unused ``PolyError`` and ``PolyDomainError`` +------------------------------------------------------------------------------- + +The class ``PolyBase`` has been removed (deprecated in numpy 1.9.0). Please use +the abstract ``ABCPolyBase`` class instead. + +Furthermore, the unused ``PolyError`` and ``PolyDomainError`` exceptions are +removed from the `numpy.polynomial`. + +(`gh-18963 `__) + + +Compatibility notes +=================== + +Error type changes in universal functions +----------------------------------------- +The universal functions may now raise different errors on invalid input in some +cases. The main changes should be that a ``RuntimeError`` was replaced with a +more fitting ``TypeError``. When multiple errors were present in the same +call, NumPy may now raise a different one. + +(`gh-15271 `__) + +``__array_ufunc__`` argument validation +--------------------------------------- +NumPy will now partially validate arguments before calling ``__array_ufunc__``. +Previously, it was possible to pass on invalid arguments (such as a +non-existing keyword argument) when dispatch was known to occur. + +(`gh-15271 `__) + +``__array_ufunc__`` and additional positional arguments +------------------------------------------------------- +Previously, all positionally passed arguments were checked for +``__array_ufunc__`` support. In the case of ``reduce``, ``accumulate``, and +``reduceat`` all arguments may be passed by position. This means that when +they were passed by position, they could previously have been asked to handle +the ufunc call via ``__array_ufunc__``. Since this depended on the way the +arguments were passed (by position or by keyword), NumPy will now only dispatch +on the input and output array. For example, NumPy will never dispatch on the +``where`` array in a reduction such as ``np.add.reduce``. + +(`gh-15271 `__) + +Validate input values in ``Generator.uniform`` +---------------------------------------------- +Checked that ``high - low >= 0`` in ``np.random.Generator.uniform``. Raises +``ValueError`` if ``low > high``. Previously out-of-order inputs were accepted +and silently swapped, so that if ``low > high``, the value generated was +``high + (low - high) * random()``. + +(`gh-17921 `__) + +``/usr/include`` removed from default include paths +--------------------------------------------------- +The default include paths when building a package with ``numpy.distutils`` no +longer include ``/usr/include``. This path is normally added by the compiler, +and hardcoding it can be problematic. In case this causes a problem, please +open an issue. A workaround is documented in PR 18658. + +(`gh-18658 `__) + +Changes to comparisons with ``dtype=...`` +----------------------------------------- +When the ``dtype=`` (or ``signature``) arguments to comparison +ufuncs (``equal``, ``less``, etc.) is used, this will denote +the desired output dtype in the future. +This means that: + + np.equal(2, 3, dtype=object) + +will give a ``FutureWarning`` that it will return an ``object`` +array in the future, which currently happens for: + + np.equal(None, None, dtype=object) + +due to the fact that ``np.array(None)`` is already an object +array. (This also happens for some other dtypes.) + +Since comparisons normally only return boolean arrays, providing +any other dtype will always raise an error in the future and +give a ``DeprecationWarning`` now. + +(`gh-18718 `__) + +Changes to ``dtype`` and ``signature`` arguments in ufuncs +---------------------------------------------------------- +The universal function arguments ``dtype`` and ``signature`` +which are also valid for reduction such as ``np.add.reduce`` +(which is the implementation for ``np.sum``) will now issue +a warning when the ``dtype`` provided is not a "basic" dtype. + +NumPy almost always ignored metadata, byteorder or time units +on these inputs. NumPy will now always ignore it and raise an +error if byteorder or time unit changed. +The following are the most important examples of changes which +will give the error. In some cases previously the information +stored was not ignored, in all of these an error is now raised:: + + # Previously ignored the byte-order (affect if non-native) + np.add(3, 5, dtype=">i32") + + # The biggest impact is for timedelta or datetimes: + arr = np.arange(10, dtype="m8[s]") + # The examples always ignored the time unit "ns": + np.add(arr, arr, dtype="m8[ns]") + np.maximum.reduce(arr, dtype="m8[ns]") + + # The following previously did use "ns" (as opposed to `arr.dtype`) + np.add(3, 5, dtype="m8[ns]") # Now return generic time units + np.maximum(arr, arr, dtype="m8[ns]") # Now returns "s" (from `arr`) + +The same applies for functions like ``np.sum`` which use these internally. +This change is necessary to achieve consistent handling within NumPy. + +If you run into these, in most cases pass for example ``dtype=np.timedelta64`` +which clearly denotes a general ``timedelta64`` without any unit or byte-order +defined. If you need to specify the output dtype precisely, you may do so +by either casting the inputs or providing an output array using `out=`. + +NumPy may choose to allow providing an exact output ``dtype`` here in the +future, which would be preceded by a ``FutureWarning``. + +(`gh-18718 `__) + +Ufunc ``signature=...`` and ``dtype=`` generalization and ``casting`` +--------------------------------------------------------------------- +The behaviour for ``np.ufunc(1.0, 1.0, signature=...)`` or +``np.ufunc(1.0, 1.0, dtype=...)`` can now yield different loops in 1.21 +compared to 1.20 because of changes in promotion. +When ``signature`` was previously used, the casting check on inputs +was relaxed, which could lead to downcasting inputs unsafely especially +if combined with ``casting="unsafe"``. + +Casting is now guaranteed to be safe. If a signature is only +partially provided, for example using ``signature=("float64", None, None)``, +this could lead to no loop being found (an error). +In that case, it is necessary to provide the complete signature +to enforce casting the inputs. +If ``dtype="float64"`` is used or only outputs are set (e.g. +``signature=(None, None, "float64")`` the is unchanged. +We expect that very few users are affected by this change. + +Further, the meaning of ``dtype="float64"`` has been slightly modified and +now strictly enforces only the correct output (and not input) DTypes. +This means it is now always equivalent to:: + + signature=(None, None, "float64") + +(If the ufunc has two inputs and one output). Since this could lead +to no loop being found in some cases, NumPy will normally also search +for the loop:: + + signature=("float64", "float64", "float64") + +if the first search failed. +In the future, this behaviour may be customized to achieve the expected +results for more complex ufuncs. (For some universal functions such as +``np.ldexp`` inputs can have different DTypes.) + +(`gh-18880 `__) + +Distutils forces strict floating point model on clang +----------------------------------------------------- +NumPy distutils will now always add the ``-ffp-exception-behavior=strict`` +compiler flag when compiling with clang. Clang defaults to a non-strict +version, which allows the compiler to generate code that does not set +floating point warnings/errors correctly. + +(`gh-19049 `__) + + +C API changes +============= + +Use of ``ufunc->type_resolver`` and "type tuple" +------------------------------------------------ +NumPy now normalizes the "type tuple" argument to the type resolver functions +before calling it. Note that in the use of this type resolver is legacy +behaviour and NumPy will not do so when possible. Calling +``ufunc->type_resolver`` or ``PyUFunc_DefaultTypeResolver`` is strongly +discouraged and will now enforce a normalized type tuple if done. Note that +this does not affect providing a type resolver, which is expected to keep +working in most circumstances. If you have an unexpected use-case for calling +the type resolver, please inform the NumPy developers so that a solution can be +found. + +(`gh-18718 `__) + + +New Features +============ + +Added a mypy plugin for handling platform-specific ``numpy.number`` precisions +------------------------------------------------------------------------------ +A mypy_ plugin is now available for automatically assigning the (platform-dependent) +precisions of certain `~numpy.number` subclasses, including the likes of +`~numpy.int_`, `~numpy.intp` and `~numpy.longlong`. See the documentation on +:ref:`scalar types ` for a comprehensive overview +of the affected classes. + +Note that while usage of the plugin is completely optional, without it the +precision of above-mentioned classes will be inferred as `~typing.Any`. + +To enable the plugin, one must add it to their mypy `configuration file`_: + +.. code-block:: ini + + [mypy] + plugins = numpy.typing.mypy_plugin + + +.. _mypy: http://mypy-lang.org/ +.. _configuration file: https://mypy.readthedocs.io/en/stable/config_file.html + +(`gh-17843 `__) + +Let the mypy plugin manage extended-precision ``numpy.number`` subclasses +------------------------------------------------------------------------- +The mypy_ plugin, introduced in `numpy/numpy#17843`_, has been expanded: +the plugin now removes annotations for platform-specific extended-precision +types that are not available to the platform in question. +For example, it will remove `~numpy.float128` when not available. + +Without the plugin *all* extended-precision types will, as far as mypy is concerned, +be available on all platforms. + +To enable the plugin, one must add it to their mypy `configuration file`_: + +.. code-block:: ini + + [mypy] + plugins = numpy.typing.mypy_plugin + + +.. _mypy: http://mypy-lang.org/ +.. _configuration file: https://mypy.readthedocs.io/en/stable/config_file.html +.. _`numpy/numpy#17843`: https://github.com/numpy/numpy/pull/17843 + +(`gh-18322 `__) + +New ``min_digits`` argument for printing float values +----------------------------------------------------- +A new ``min_digits`` argument has been added to the dragon4 float printing +functions `~numpy.format_float_positional` and `~numpy.format_float_scientific` +. This kwd guarantees that at least the given number of digits will be printed +when printing in unique=True mode, even if the extra digits are unnecessary to +uniquely specify the value. It is the counterpart to the precision argument +which sets the maximum number of digits to be printed. When unique=False in +fixed precision mode, it has no effect and the precision argument fixes the +number of digits. + +(`gh-18629 `__) + +f2py now recognizes Fortran abstract interface blocks +----------------------------------------------------- +`~numpy.f2py` can now parse abstract interface blocks. + +(`gh-18695 `__) + +BLAS and LAPACK configuration via environment variables +------------------------------------------------------- +Autodetection of installed BLAS and LAPACK libraries can be bypassed by using +the ``NPY_BLAS_LIBS`` and ``NPY_LAPACK_LIBS`` environment variables. Instead, +the link flags in these environment variables will be used directly, and the +language is assumed to be F77. This is especially useful in automated builds +where the BLAS and LAPACK that are installed are known exactly. A use case is +replacing the actual implementation at runtime via stub library links. + +If ``NPY_CBLAS_LIBS`` is set (optional in addition to ``NPY_BLAS_LIBS``), this +will be used as well, by defining ``HAVE_CBLAS`` and appending the environment +variable content to the link flags. + +(`gh-18737 `__) + +A runtime-subcriptable alias has been added for ``ndarray`` +----------------------------------------------------------- +``numpy.typing.NDArray`` has been added, a runtime-subscriptable alias for +``np.ndarray[Any, np.dtype[~Scalar]]``. The new type alias can be used +for annotating arrays with a given dtype and unspecified shape. :sup:`1` + +:sup:`1` NumPy does not support the annotating of array shapes as of 1.21, +this is expected to change in the future though (see :pep:`646`). + +Examples +~~~~~~~~ + +.. code-block:: python + + >>> import numpy as np + >>> import numpy.typing as npt + + >>> print(npt.NDArray) + numpy.ndarray[typing.Any, numpy.dtype[~ScalarType]] + + >>> print(npt.NDArray[np.float64]) + numpy.ndarray[typing.Any, numpy.dtype[numpy.float64]] + + >>> NDArrayInt = npt.NDArray[np.int_] + >>> a: NDArrayInt = np.arange(10) + + >>> def func(a: npt.ArrayLike) -> npt.NDArray[Any]: + ... return np.array(a) + +(`gh-18935 `__) + + +Improvements +============ + +Arbitrary ``period`` option for ``numpy.unwrap`` +------------------------------------------------ +The size of the interval over which phases are unwrapped is no longer restricted to ``2 * pi``. +This is especially useful for unwrapping degrees, but can also be used for other intervals. + +.. code:: python + + >>> phase_deg = np.mod(np.linspace(0,720,19), 360) - 180 + >>> phase_deg + array([-180., -140., -100., -60., -20., 20., 60., 100., 140., + -180., -140., -100., -60., -20., 20., 60., 100., 140., + -180.]) + + >>> unwrap(phase_deg, period=360) + array([-180., -140., -100., -60., -20., 20., 60., 100., 140., + 180., 220., 260., 300., 340., 380., 420., 460., 500., + 540.]) + +(`gh-16987 `__) + +``np.unique`` now returns single ``NaN`` +---------------------------------------- +When ``np.unique`` operated on an array with multiple ``NaN`` entries, +its return included a ``NaN`` for each entry that was ``NaN`` in the original array. +This is now improved such that the returned array contains just one ``NaN`` as the +last element. + +Also for complex arrays all ``NaN`` values are considered equivalent +(no matter whether the ``NaN`` is in the real or imaginary part). As the +representant for the returned array the smallest one in the +lexicographical order is chosen - see ``np.sort`` for how the lexicographical +order is defined for complex arrays. + +(`gh-18070 `__) + +``Generator.rayleigh`` and ``Generator.geometric`` performance improved +----------------------------------------------------------------------- +The performance of Rayleigh and geometric random variate generation +in ``Generator`` has improved. These are both transformation of exponential +random variables and the slow log-based inverse cdf transformation has +been replaced with the Ziggurat-based exponential variate generator. + +This change breaks the stream of variates generated when variates from +either of these distributions are produced. + +(`gh-18666 `__) + +Placeholder annotations have been improved +------------------------------------------ +All placeholder annotations, that were previously annotated as ``typing.Any``, +have been improved. Where appropiate they have been replaced with explicit +function definitions, classes or other miscellaneous objects. + +(`gh-18934 `__) + + +Performance improvements +======================== + +Improved performance in integer division of NumPy arrays +-------------------------------------------------------- +Integer division of NumPy arrays now uses +`libdivide `__ when the divisor is a constant. With the +usage of libdivide and other minor optimizations, there is a large speedup. +The ``//`` operator and ``np.floor_divide`` makes use of the new changes. + +(`gh-17727 `__) + +Improve performance of ``np.save`` and ``np.load`` for small arrays +------------------------------------------------------------------- +``np.save`` is now a lot faster for small arrays. + +``np.load`` is also faster for small arrays, +but only when serializing with a version >= ``(3, 0)``. + +Both are done by removing checks that are only relevant for Python 2, +while still maintaining compatibility with arrays +which might have been created by Python 2. + +(`gh-18657 `__) + + +Changes +======= + +`numpy.piecewise` output class now matches the input class +---------------------------------------------------------- +When `~numpy.ndarray` subclasses are used on input to `~numpy.piecewise`, +they are passed on to the functions. The output will now be of the +same subclass as well. + +(`gh-18110 `__) + +Enable Accelerate Framework +---------------------------- +With the release of macOS 11.3, several different issues that numpy was +encountering when using Accelerate Framework's implementation of BLAS and +LAPACK should be resolved. This change enables the Accelerate Framework as an +option on macOS. If additional issues are found, please file a bug report +against Accelerate using the developer feedback assistant tool +(https://developer.apple.com/bug-reporting/). We intend to address issues +promptly and plan to continue supporting and updating our BLAS and LAPACK +libraries. + +(`gh-18874 `__) diff --git a/doc/source/user/absolute_beginners.rst b/doc/source/user/absolute_beginners.rst index 126f5f2a395e..d97b92617b77 100644 --- a/doc/source/user/absolute_beginners.rst +++ b/doc/source/user/absolute_beginners.rst @@ -63,17 +63,13 @@ at `SciPy `_. How to import NumPy ------------------- -Any time you want to use a package or library in your code, you first need to -make it accessible. - -In order to start using NumPy and all of the functions available in NumPy, -you'll need to import it. This can be easily done with this import statement:: +To access NumPy and its functions import it in your Python code like this:: import numpy as np -(We shorten ``numpy`` to ``np`` in order to save time and also to keep code -standardized so that anyone working with your code can easily understand and -run it.) +We shorten the imported name to ``np`` for better readability of code using +NumPy. This is a widely adopted convention that you should follow so that +anyone working with your code can easily understand it. Reading the example code ------------------------ @@ -775,12 +771,12 @@ If you start with this array:: >>> b = np.array([[1, 1], [2, 2]]) -You can sum the rows with:: +You can sum over the axis of rows with:: >>> b.sum(axis=0) array([3, 3]) -You can sum the columns with:: +You can sum over the axis of columns with:: >>> b.sum(axis=1) array([2, 4]) @@ -871,10 +867,11 @@ Creating matrices You can pass Python lists of lists to create a 2-D array (or "matrix") to represent them in NumPy. :: - >>> data = np.array([[1, 2], [3, 4]]) + >>> data = np.array([[1, 2], [3, 4], [5, 6]]) >>> data array([[1, 2], - [3, 4]]) + [3, 4], + [5, 6]]) .. image:: images/np_create_matrix.png @@ -883,7 +880,8 @@ Indexing and slicing operations are useful when you're manipulating matrices:: >>> data[0, 1] 2 >>> data[1:3] - array([[3, 4]]) + array([[3, 4], + [5, 6]]) >>> data[0:2, 0] array([1, 3]) @@ -892,11 +890,11 @@ Indexing and slicing operations are useful when you're manipulating matrices:: You can aggregate matrices the same way you aggregated vectors:: >>> data.max() - 4 + 6 >>> data.min() 1 >>> data.sum() - 10 + 21 .. image:: images/np_matrix_aggregation.png @@ -904,9 +902,9 @@ You can aggregate all the values in a matrix and you can aggregate them across columns or rows using the ``axis`` parameter:: >>> data.max(axis=0) - array([3, 4]) + array([5, 6]) >>> data.max(axis=1) - array([2, 4]) + array([2, 4, 6]) .. image:: images/np_matrix_aggregation_row.png @@ -1672,9 +1670,8 @@ For example, you can plot a 1D array like this:: With Matplotlib, you have access to an enormous number of visualization options. :: - >>> from mpl_toolkits.mplot3d import Axes3D >>> fig = plt.figure() - >>> ax = Axes3D(fig) + >>> ax = fig.add_subplot(projection='3d') >>> X = np.arange(-5, 5, 0.15) >>> Y = np.arange(-5, 5, 0.15) >>> X, Y = np.meshgrid(X, Y) diff --git a/doc/source/user/basics.creation.rst b/doc/source/user/basics.creation.rst index 671a8ec59e59..ccd6de184ce8 100644 --- a/doc/source/user/basics.creation.rst +++ b/doc/source/user/basics.creation.rst @@ -9,55 +9,102 @@ Array creation Introduction ============ -There are 5 general mechanisms for creating arrays: +There are 6 general mechanisms for creating arrays: -1) Conversion from other Python structures (e.g., lists, tuples) -2) Intrinsic numpy array creation objects (e.g., arange, ones, zeros, +1) Conversion from other Python structures (i.e. lists and tuples) +2) Intrinsic NumPy array creation functions (e.g. arange, ones, zeros, etc.) -3) Reading arrays from disk, either from standard or custom formats -4) Creating arrays from raw bytes through the use of strings or buffers -5) Use of special library functions (e.g., random) +3) Replicating, joining, or mutating existing arrays +4) Reading arrays from disk, either from standard or custom formats +5) Creating arrays from raw bytes through the use of strings or buffers +6) Use of special library functions (e.g., random) -This section will not cover means of replicating, joining, or otherwise -expanding or mutating existing arrays. Nor will it cover creating object -arrays or structured arrays. Both of those are covered in their own sections. +You can use these methods to create ndarrays or :ref:`structured_arrays`. +This document will cover general methods for ndarray creation. -Converting Python array_like Objects to NumPy Arrays -==================================================== - -In general, numerical data arranged in an array-like structure in Python can -be converted to arrays through the use of the array() function. The most -obvious examples are lists and tuples. See the documentation for array() for -details for its use. Some objects may support the array-protocol and allow -conversion to arrays this way. A simple way to find out if the object can be -converted to a numpy array using array() is simply to try it interactively and -see if it works! (The Python Way). +1) Converting Python sequences to NumPy Arrays +============================================== -Examples: :: +NumPy arrays can be defined using Python sequences such as lists and +tuples. Lists and tuples are defined using ``[...]`` and ``(...)``, +respectively. Lists and tuples can define ndarray creation: - >>> x = np.array([2,3,1,0]) - >>> x = np.array([2, 3, 1, 0]) - >>> x = np.array([[1,2.0],[0,0],(1+1j,3.)]) # note mix of tuple and lists, - and types - >>> x = np.array([[ 1.+0.j, 2.+0.j], [ 0.+0.j, 0.+0.j], [ 1.+1.j, 3.+0.j]]) +* a list of numbers will create a 1D array, +* a list of lists will create a 2D array, +* further nested lists will create higher-dimensional arrays. In general, any array object is called an **ndarray** in NumPy. -Intrinsic NumPy Array Creation -============================== - -NumPy has built-in functions for creating arrays from scratch: +:: -zeros(shape) will create an array filled with 0 values with the specified -shape. The default dtype is float64. :: + >>> a1D = np.array([1, 2, 3, 4]) + >>> a2D = np.array([[1, 2], [3, 4]]) + >>> a3D = np.array([[[1, 2], [3, 4]], + [[5, 6], [7, 8]]]) - >>> np.zeros((2, 3)) - array([[ 0., 0., 0.], [ 0., 0., 0.]]) +When you use :func:`numpy.array` to define a new array, you should +consider the :doc:`dtype ` of the elements in the array, +which can be specified explicitly. This feature gives you +more control over the underlying data structures and how the elements +are handled in C/C++ functions. If you are not careful with ``dtype`` +assignments, you can get unwanted overflow, as such -ones(shape) will create an array filled with 1 values. It is identical to -zeros in all other respects. +:: -arange() will create arrays with regularly incrementing values. Check the -docstring for complete information on the various ways it can be used. A few -examples will be given here: :: + >>> a = np.array([127, 128, 129], dtype=np.int8) + >>> a + array([ 127, -128, -127], dtype=int8) + +An 8-bit signed integer represents integers from -128 to 127. +Assigning the ``int8`` array to integers outside of this range results +in overflow. This feature can often be misunderstood. If you +perform calculations with mismatching ``dtypes``, you can get unwanted +results, for example:: + + >>> a = array([2, 3, 4], dtype = np.uint32) + >>> b = array([5, 6, 7], dtype = np.uint32) + >>> c_unsigned32 = a - b + >>> print('unsigned c:', c_unsigned32, c_unsigned32.dtype) + unsigned c: [4294967293 4294967293 4294967293] uint32 + >>> c_signed32 = a - b.astype(np.int32) + >>> print('signed c:', c_signed32, c_signed32.dtype) + signed c: [-3 -3 -3] int64 + +Notice when you perform operations with two arrays of the same +``dtype``: ``uint32``, the resulting array is the same type. When you +perform operations with different ``dtype``, NumPy will +assign a new type that satisfies all of the array elements involved in +the computation, here ``uint32`` and ``int32`` can both be represented in +as ``int64``. + +The default NumPy behavior is to create arrays in either 64-bit signed +integers or double precision floating point numbers, ``int64`` and +``float``, respectively. If you expect your arrays to be a certain type, +then you need to specify the ``dtype`` while you create the array. + +2) Intrinsic NumPy array creation functions +=========================================== +.. + 40 functions seems like a small number, but the routies.array-creation + has ~47. I'm sure there are more. + +NumPy has over 40 built-in functions for creating arrays as laid +out in the :ref:`Array creation routines `. +These functions can be split into roughly three categories, based on the +dimension of the array they create: + +1) 1D arrays +2) 2D arrays +3) ndarrays + +1 - 1D array creation functions +------------------------------- + +The 1D array creation functions e.g. :func:`numpy.linspace` and +:func:`numpy.arange` generally need at least two inputs, ``start`` and +``stop``. + +:func:`numpy.arange` creates arrays with regularly incrementing values. +Check the documentation for complete information and examples. A few +examples are shown:: >>> np.arange(10) array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -66,43 +113,216 @@ examples will be given here: :: >>> np.arange(2, 3, 0.1) array([ 2. , 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9]) -Note that there are some subtleties regarding the last usage that the user -should be aware of that are described in the arange docstring. +Note: best practice for :func:`numpy.arange` is to use integer start, end, and +step values. There are some subtleties regarding ``dtype``. In the second +example, the ``dtype`` is defined. In the third example, the array is +``dtype=float`` to accomodate the step size of ``0.1``. Due to roundoff error, +the ``stop`` value is sometimes included. -linspace() will create arrays with a specified number of elements, and +:func:`numpy.linspace` will create arrays with a specified number of elements, and spaced equally between the specified beginning and end values. For example: :: >>> np.linspace(1., 4., 6) array([ 1. , 1.6, 2.2, 2.8, 3.4, 4. ]) -The advantage of this creation function is that one can guarantee the -number of elements and the starting and end point, which arange() -generally will not do for arbitrary start, stop, and step values. +The advantage of this creation function is that you guarantee the +number of elements and the starting and end point. The previous +``arange(start, stop, step)`` will not include the value ``stop``. + +2 - 2D array creation functions +------------------------------- + +The 2D array creation functions e.g. :func:`numpy.eye`, :func:`numpy.diag`, and :func:`numpy.vander` +define properties of special matrices represented as 2D arrays. + +``np.eye(n, m)`` defines a 2D identity matrix. The elements where i=j (row index and column index are equal) are 1 +and the rest are 0, as such:: + + >>> np.eye(3) + array([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + >>> np.eye(3, 5) + array([[1., 0., 0., 0., 0.], + [0., 1., 0., 0., 0.], + [0., 0., 1., 0., 0.]]) + +:func:`numpy.diag` can define either a square 2D array with given values along +the diagonal *or* if given a 2D array returns a 1D array that is +only the diagonal elements. The two array creation functions can be helpful while +doing linear algebra, as such:: + + >>> np.diag([1, 2, 3]) + array([[1, 0, 0], + [0, 2, 0], + [0, 0, 3]]) + >>> np.diag([1, 2, 3], 1) + array([[0, 1, 0, 0], + [0, 0, 2, 0], + [0, 0, 0, 3], + [0, 0, 0, 0]]) + >>> a = np.array([[1, 2], [3, 4]]) + >>> np.diag(a) + array([1, 4]) + +``vander(x, n)`` defines a Vandermonde matrix as a 2D NumPy array. Each column +of the Vandermonde matrix is a decreasing power of the input 1D array or +list or tuple, +``x`` where the highest polynomial order is ``n-1``. This array creation +routine is helpful in generating linear least squares models, as such:: + + >>> np.vander(np.linspace(0, 2, 5), 2) + array([[0. , 0. , 1. ], + [0.25, 0.5 , 1. ], + [1. , 1. , 1. ], + [2.25, 1.5 , 1. ], + [4. , 2. , 1. ]]) + >>> np.vander([1, 2, 3, 4], 2) + array([[1, 1], + [2, 1], + [3, 1], + [4, 1]]) + >>> np.vander((1, 2, 3, 4), 4) + array([[ 1, 1, 1, 1], + [ 8, 4, 2, 1], + [27, 9, 3, 1], + [64, 16, 4, 1]]) + +3 - general ndarray creation functions +-------------------------------------- + +The ndarray creation functions e.g. :func:`numpy.ones`, +:func:`numpy.zeros`, and :meth:`~numpy.random.Generator.random` define +arrays based upon the desired shape. The ndarray creation functions +can create arrays with any dimension by specifying how many dimensions +and length along that dimension in a tuple or list. + +:func:`numpy.zeros` will create an array filled with 0 values with the +specified shape. The default dtype is ``float64``:: -indices() will create a set of arrays (stacked as a one-higher dimensioned -array), one per dimension with each representing variation in that dimension. -An example illustrates much better than a verbal description: :: + >>> np.zeros((2, 3)) + array([[0., 0., 0.], + [0., 0., 0.]]) + >>> np.zeros((2, 3, 2)) + array([[[0., 0.], + [0., 0.], + [0., 0.]], + + [[0., 0.], + [0., 0.], + [0., 0.]]]) + +:func:`numpy.ones` will create an array filled with 1 values. It is identical to +``zeros`` in all other respects as such:: + + >>> np.ones((2, 3)) + array([[ 1., 1., 1.], + [ 1., 1., 1.]]) + >>> np.ones((2, 3, 2)) + array([[[1., 1.], + [1., 1.], + [1., 1.]], + + [[1., 1.], + [1., 1.], + [1., 1.]]]) + +The :meth:`~numpy.random.Generator.random` method of the result of +``default_rng`` will create an array filled with random +values between 0 and 1. It is included with the :func:`numpy.random` +library. Below, two arrays are created with shapes (2,3) and (2,3,2), +respectively. The seed is set to 42 so you can reproduce these +pseudorandom numbers:: + + >>> import numpy.random.default_rng + >>> default_rng(42).random((2,3)) + array([[0.77395605, 0.43887844, 0.85859792], + [0.69736803, 0.09417735, 0.97562235]]) + >>> default_rng(42).random((2,3,2)) + array([[[0.77395605, 0.43887844], + [0.85859792, 0.69736803], + [0.09417735, 0.97562235]], + [[0.7611397 , 0.78606431], + [0.12811363, 0.45038594], + [0.37079802, 0.92676499]]]) + +:func:`numpy.indices` will create a set of arrays (stacked as a one-higher +dimensioned array), one per dimension with each representing variation in that +dimension: :: >>> np.indices((3,3)) - array([[[0, 0, 0], [1, 1, 1], [2, 2, 2]], [[0, 1, 2], [0, 1, 2], [0, 1, 2]]]) + array([[[0, 0, 0], + [1, 1, 1], + [2, 2, 2]], + [[0, 1, 2], + [0, 1, 2], + [0, 1, 2]]]) This is particularly useful for evaluating functions of multiple dimensions on a regular grid. -Reading Arrays From Disk -======================== +3) Replicating, joining, or mutating existing arrays +==================================================== -This is presumably the most common case of large array creation. The details, -of course, depend greatly on the format of data on disk and so this section -can only give general pointers on how to handle various formats. +Once you have created arrays, you can replicate, join, or mutate those +existing arrays to create new arrays. When you assign an array or its +elements to a new variable, you have to explicitly :func:`numpy.copy` the array, +otherwise the variable is a view into the original array. Consider the +following example:: + + >>> a = np.array([1, 2, 3, 4, 5, 6]) + >>> b = a[:2] + >>> b += 1 + >>> print('a =', a, '; b =', b) + a = [2 3 3 4 5 6]; b = [2 3] + +In this example, you did not create a new array. You created a variable, +``b`` that viewed the first 2 elements of ``a``. When you added 1 to ``b`` you +would get the same result by adding 1 to ``a[:2]``. If you want to create a +*new* array, use the :func:`numpy.copy` array creation routine as such:: + + >>> a = np.array([1, 2, 3, 4]) + >>> b = a[:2].copy() + >>> b += 1 + >>> print('a = ', a, 'b = ', b) + a = [1 2 3 4 5 6] b = [2 3] + +For more information and examples look at :ref:`Copies and Views +`. + +There are a number of routines to join existing arrays e.g. :func:`numpy.vstack`, +:func:`numpy.hstack`, and :func:`numpy.block`. Here is an example of joining four 2-by-2 +arrays into a 4-by-4 array using ``block``:: + + >>> A = np.ones((2, 2)) + >>> B = np.eye((2, 2)) + >>> C = np.zeros((2, 2)) + >>> D = np.diag((-3, -4)) + >>> np.block([[A, B], + [C, D]]) + array([[ 1., 1., 1., 0. ], + [ 1., 1., 0., 1. ], + [ 0., 0., -3., 0. ], + [ 0., 0., 0., -4. ]]) + +Other routines use similar syntax to join ndarrays. Check the +routine's documentation for further examples and syntax. + +4) Reading arrays from disk, either from standard or custom formats +=================================================================== + +This is the most common case of large array creation. The details depend +greatly on the format of data on disk. This section gives general pointers on +how to handle various formats. For more detailed examples of IO look at +:ref:`How to Read and Write files `. Standard Binary Formats ----------------------- Various fields have standard formats for array data. The following lists the -ones with known python libraries to read them and return numpy arrays (there -may be others for which it is possible to read and convert to numpy arrays so +ones with known Python libraries to read them and return NumPy arrays (there +may be others for which it is possible to read and convert to NumPy arrays so check the last section as well) :: @@ -114,33 +334,51 @@ convert are those formats supported by libraries like PIL (able to read and write many image formats such as jpg, png, etc). Common ASCII Formats ------------------------- +-------------------- -Comma Separated Value files (CSV) are widely used (and an export and import -option for programs like Excel). There are a number of ways of reading these -files in Python. There are CSV functions in Python and functions in pylab -(part of matplotlib). +Delimited files such as comma separated value (csv) and tab separated +value (tsv) files are used for programs like Excel and LabView. Python +functions can read and parse these files line-by-line. NumPy has two +standard routines for importing a file with delimited data :func:`numpy.loadtxt` +and :func:`numpy.genfromtxt`. These functions have more involved use cases in +:doc:`how-to-io`. A simple example given a ``simple.csv``: -More generic ascii files can be read using the io package in scipy. +.. code-block:: bash -Custom Binary Formats ---------------------- + $ cat simple.csv + x, y + 0, 0 + 1, 1 + 2, 4 + 3, 9 + +Importing ``simple.csv`` is accomplished using :func:`loadtxt`:: + + >>> np.loadtxt('simple.csv', delimiter = ',', skiprows = 1) # doctest: +SKIP + array([[0., 0.], + [1., 1.], + [2., 4.], + [3., 9.]]) + + +More generic ASCII files can be read using `scipy.io` and `Pandas +`_. + +5) Creating arrays from raw bytes through the use of strings or buffers +======================================================================= There are a variety of approaches one can use. If the file has a relatively -simple format then one can write a simple I/O library and use the numpy -fromfile() function and .tofile() method to read and write numpy arrays +simple format then one can write a simple I/O library and use the NumPy +``fromfile()`` function and ``.tofile()`` method to read and write NumPy arrays directly (mind your byteorder though!) If a good C or C++ library exists that read the data, one can wrap that library with a variety of techniques though that certainly is much more work and requires significantly more advanced knowledge to interface with C or C++. -Use of Special Libraries ------------------------- - -There are libraries that can be used to generate arrays for special purposes -and it isn't possible to enumerate all of them. The most common uses are use -of the many array generation functions in random that can generate arrays of -random values, and some utility functions to generate special matrices (e.g. -diagonal). - +6) Use of special library functions (e.g., SciPy, Pandas, and OpenCV) +===================================================================== +NumPy is the fundamental library for array containers in the Python Scientific Computing +stack. Many Python libraries, including SciPy, Pandas, and OpenCV, use NumPy ndarrays +as the common format for data exchange, These libraries can create, +operate on, and work with NumPy arrays. diff --git a/doc/source/user/basics.dispatch.rst b/doc/source/user/basics.dispatch.rst index c0e1cf9ba577..089a7df17063 100644 --- a/doc/source/user/basics.dispatch.rst +++ b/doc/source/user/basics.dispatch.rst @@ -22,8 +22,8 @@ example that has rather narrow utility but illustrates the concepts involved. ... self._i = value ... def __repr__(self): ... return f"{self.__class__.__name__}(N={self._N}, value={self._i})" -... def __array__(self): -... return self._i * np.eye(self._N) +... def __array__(self, dtype=None): +... return self._i * np.eye(self._N, dtype=dtype) Our custom array can be instantiated like: @@ -56,7 +56,7 @@ array([[2., 0., 0., 0., 0.], Notice that the return type is a standard ``numpy.ndarray``. ->>> type(arr) +>>> type(np.multiply(arr, 2)) numpy.ndarray How can we pass our custom array type through this function? Numpy allows a @@ -84,8 +84,8 @@ For this example we will only handle the method ``__call__`` ... self._i = value ... def __repr__(self): ... return f"{self.__class__.__name__}(N={self._N}, value={self._i})" -... def __array__(self): -... return self._i * np.eye(self._N) +... def __array__(self, dtype=None): +... return self._i * np.eye(self._N, dtype=dtype) ... def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ... if method == '__call__': ... N = None @@ -133,8 +133,8 @@ conveniently by inheriting from the mixin ... self._i = value ... def __repr__(self): ... return f"{self.__class__.__name__}(N={self._N}, value={self._i})" -... def __array__(self): -... return self._i * np.eye(self._N) +... def __array__(self, dtype=None): +... return self._i * np.eye(self._N, dtype=dtype) ... def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ... if method == '__call__': ... N = None @@ -171,8 +171,8 @@ functions to our custom variants. ... self._i = value ... def __repr__(self): ... return f"{self.__class__.__name__}(N={self._N}, value={self._i})" -... def __array__(self): -... return self._i * np.eye(self._N) +... def __array__(self, dtype=None): +... return self._i * np.eye(self._N, dtype=dtype) ... def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ... if method == '__call__': ... N = None diff --git a/doc/source/user/basics.indexing.rst b/doc/source/user/basics.indexing.rst index 9545bb78c5a5..7ee61b13007f 100644 --- a/doc/source/user/basics.indexing.rst +++ b/doc/source/user/basics.indexing.rst @@ -11,7 +11,7 @@ Indexing :ref:`Indexing routines ` Array indexing refers to any use of the square brackets ([]) to index -array values. There are many options to indexing, which give numpy +array values. There are many options to indexing, which give NumPy indexing great power, but with power comes some complexity and the potential for confusion. This section is just an overview of the various options and issues related to indexing. Aside from single @@ -39,7 +39,7 @@ and accepts negative indices for indexing from the end of the array. :: >>> x[-2] 8 -Unlike lists and tuples, numpy arrays support multidimensional indexing +Unlike lists and tuples, NumPy arrays support multidimensional indexing for multidimensional arrays. That means that it is not necessary to separate each dimension's index into its own set of square brackets. :: @@ -285,7 +285,7 @@ with four True elements to select rows from a 3-D array of shape [20, 21, 22, 23, 24], [25, 26, 27, 28, 29]]) -For further details, consult the numpy reference documentation on array indexing. +For further details, consult the NumPy reference documentation on array indexing. Combining index arrays with slices ================================== diff --git a/doc/source/user/basics.rst b/doc/source/user/basics.rst index e0fc0ece36ce..66f3f9ee9988 100644 --- a/doc/source/user/basics.rst +++ b/doc/source/user/basics.rst @@ -1,14 +1,18 @@ -************ -NumPy basics -************ +****************** +NumPy fundamentals +****************** + +These documents clarify concepts, design decisions, and technical +constraints in NumPy. This is a great place to understand the +fundamental NumPy ideas and philosophy. .. toctree:: :maxdepth: 1 - basics.types basics.creation - basics.io basics.indexing + basics.io + basics.types basics.broadcasting basics.byteswapping basics.rec diff --git a/doc/source/user/basics.subclassing.rst b/doc/source/user/basics.subclassing.rst index 8ffa31688531..1b78809865aa 100644 --- a/doc/source/user/basics.subclassing.rst +++ b/doc/source/user/basics.subclassing.rst @@ -223,8 +223,8 @@ where our object creation housekeeping usually goes. new ndarray instance of its own class. In practice this means that we, the authors of the code, will need to make a call to ``ndarray.__new__(MySubClass,...)``, a class-hierarchy prepared call to - ``super(MySubClass, cls).__new__(cls, ...)``, or do view casting of an - existing array (see below) + ``super().__new__(cls, ...)``, or do view casting of an existing array + (see below) * For view casting and new-from-template, the equivalent of ``ndarray.__new__(MySubClass,...`` is called, at the C level. @@ -240,7 +240,7 @@ The following code allows us to look at the call sequences and arguments: class C(np.ndarray): def __new__(cls, *args, **kwargs): print('In __new__ with class %s' % cls) - return super(C, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): # in practice you probably will not need or want an __init__ @@ -312,9 +312,8 @@ Simple example - adding an extra attribute to ndarray # ndarray input arguments. This will call the standard # ndarray constructor, but return an object of our type. # It also triggers a call to InfoArray.__array_finalize__ - obj = super(InfoArray, subtype).__new__(subtype, shape, dtype, - buffer, offset, strides, - order) + obj = super().__new__(subtype, shape, dtype, + buffer, offset, strides, order) # set the new 'info' attribute to the value passed obj.info = info # Finally, we must return the newly created object: @@ -486,8 +485,7 @@ following. if out_no: info['outputs'] = out_no - results = super(A, self).__array_ufunc__(ufunc, method, - *args, **kwargs) + results = super().__array_ufunc__(ufunc, method, *args, **kwargs) if results is NotImplemented: return NotImplemented @@ -600,7 +598,7 @@ some print statements: print(' self is %s' % repr(self)) print(' arr is %s' % repr(out_arr)) # then just call the parent - return super(MySubClass, self).__array_wrap__(self, out_arr, context) + return super().__array_wrap__(self, out_arr, context) We run a ufunc on an instance of our new array: diff --git a/doc/source/user/basics.types.rst b/doc/source/user/basics.types.rst index ec2af409ae86..354f003fbb28 100644 --- a/doc/source/user/basics.types.rst +++ b/doc/source/user/basics.types.rst @@ -1,3 +1,5 @@ +.. _basics.types: + ********** Data types ********** @@ -96,70 +98,8 @@ The primitive types supported are tied closely to those in C: Since many of these have platform-dependent definitions, a set of fixed-size -aliases are provided: - -.. list-table:: - :header-rows: 1 - - * - Numpy type - - C type - - Description - - * - `numpy.int8` - - ``int8_t`` - - Byte (-128 to 127) - - * - `numpy.int16` - - ``int16_t`` - - Integer (-32768 to 32767) - - * - `numpy.int32` - - ``int32_t`` - - Integer (-2147483648 to 2147483647) - - * - `numpy.int64` - - ``int64_t`` - - Integer (-9223372036854775808 to 9223372036854775807) - - * - `numpy.uint8` - - ``uint8_t`` - - Unsigned integer (0 to 255) - - * - `numpy.uint16` - - ``uint16_t`` - - Unsigned integer (0 to 65535) +aliases are provided (See :ref:`sized-aliases`). - * - `numpy.uint32` - - ``uint32_t`` - - Unsigned integer (0 to 4294967295) - - * - `numpy.uint64` - - ``uint64_t`` - - Unsigned integer (0 to 18446744073709551615) - - * - `numpy.intp` - - ``intptr_t`` - - Integer used for indexing, typically the same as ``ssize_t`` - - * - `numpy.uintp` - - ``uintptr_t`` - - Integer large enough to hold a pointer - - * - `numpy.float32` - - ``float`` - - - - * - `numpy.float64` / `numpy.float_` - - ``double`` - - Note that this matches the precision of the builtin python `float`. - - * - `numpy.complex64` - - ``float complex`` - - Complex number, represented by two 32-bit floats (real and imaginary components) - - * - `numpy.complex128` / `numpy.complex_` - - ``double complex`` - - Note that this matches the precision of the builtin python `complex`. NumPy numerical types are instances of ``dtype`` (data-type) objects, each @@ -171,7 +111,7 @@ having unique characteristics. Once you have imported NumPy using the dtypes are available as ``np.bool_``, ``np.float32``, etc. -Advanced types, not listed in the table above, are explored in +Advanced types, not listed above, are explored in section :ref:`structured_arrays`. There are 5 basic numerical types representing booleans (bool), integers (int), @@ -261,12 +201,14 @@ identical behaviour between arrays and scalars, irrespective of whether the value is inside an array or not. NumPy scalars also have many of the same methods arrays do. +.. _overflow-errors: + Overflow Errors =============== The fixed size of NumPy numeric types may cause overflow errors when a value requires more memory than available in the data type. For example, -`numpy.power` evaluates ``100 * 10 ** 8`` correctly for 64-bit integers, +`numpy.power` evaluates ``100 ** 8`` correctly for 64-bit integers, but gives 1874919424 (incorrect) for a 32-bit integer. >>> np.power(100, 8, dtype=np.int64) diff --git a/doc/source/user/building.rst b/doc/source/user/building.rst index 47399139e6e8..52d7330bf753 100644 --- a/doc/source/user/building.rst +++ b/doc/source/user/building.rst @@ -6,7 +6,7 @@ Building from source A general overview of building NumPy from source is given here, with detailed instructions for specific platforms given separately. -.. +.. This page is referenced from numpy/numpy/__init__.py. Please keep its location in sync with the link there. @@ -123,6 +123,9 @@ in the ``site.cfg.example`` file. BLAS ~~~~ +Note that both BLAS and CBLAS interfaces are needed for a properly +optimized build of NumPy. + The default order for the libraries are: 1. MKL @@ -131,6 +134,12 @@ The default order for the libraries are: 4. ATLAS 5. BLAS (NetLIB) +The detection of BLAS libraries may be bypassed by defining the environment +variable ``NPY_BLAS_LIBS`` , which should contain the exact linker flags you +want to use (interface is assumed to be Fortran 77). Also define +``NPY_CBLAS_LIBS`` (even empty if CBLAS is contained in your BLAS library) to +trigger use of CBLAS and avoid slow fallback code for matrix calculations. + If you wish to build against OpenBLAS but you also have BLIS available one may predefine the order of searching via the environment variable ``NPY_BLAS_ORDER`` which is a comma-separated list of the above names which @@ -163,6 +172,9 @@ The default order for the libraries are: 4. ATLAS 5. LAPACK (NetLIB) +The detection of LAPACK libraries may be bypassed by defining the environment +variable ``NPY_LAPACK_LIBS``, which should contain the exact linker flags you +want to use (language is assumed to be Fortran 77). If you wish to build against OpenBLAS but you also have MKL available one may predefine the order of searching via the environment variable @@ -185,7 +197,6 @@ list is retained. One cannot mix negation and positives, nor have multiple negations, such cases will raise an error. - .. deprecated:: 1.20 The native libraries on macOS, provided by Accelerate, are not fit for use in NumPy since they have bugs that cause wrong output under easily reproducible diff --git a/doc/source/user/c-info.beyond-basics.rst b/doc/source/user/c-info.beyond-basics.rst index 124162d6ce17..289a7951b792 100644 --- a/doc/source/user/c-info.beyond-basics.rst +++ b/doc/source/user/c-info.beyond-basics.rst @@ -172,8 +172,8 @@ iterators so that all that needs to be done to advance to the next element in each array is for PyArray_ITER_NEXT to be called for each of the inputs. This incrementing is automatically performed by :c:func:`PyArray_MultiIter_NEXT` ( ``obj`` ) macro (which can handle a -multiterator ``obj`` as either a :c:type:`PyArrayMultiObject *` or a -:c:type:`PyObject *`). The data from input number ``i`` is available using +multiterator ``obj`` as either a :c:expr:`PyArrayMultiObject *` or a +:c:expr:`PyObject *`). The data from input number ``i`` is available using :c:func:`PyArray_MultiIter_DATA` ( ``obj``, ``i`` ) and the total (broadcasted) size as :c:func:`PyArray_MultiIter_SIZE` ( ``obj``). An example of using this feature follows. @@ -400,7 +400,7 @@ describe the desired behavior of the type. Typically, a new C-structure is also created to contain the instance-specific information needed for each object of the type as well. For example, :c:data:`&PyArray_Type` is a pointer to the type-object table for the ndarray -while a :c:type:`PyArrayObject *` variable is a pointer to a particular instance +while a :c:expr:`PyArrayObject *` variable is a pointer to a particular instance of an ndarray (one of the members of the ndarray structure is, in turn, a pointer to the type- object table :c:data:`&PyArray_Type`). Finally :c:func:`PyType_Ready` () must be called for diff --git a/doc/source/user/c-info.how-to-extend.rst b/doc/source/user/c-info.how-to-extend.rst index 845ce0a74c31..ebb4b7518d1d 100644 --- a/doc/source/user/c-info.how-to-extend.rst +++ b/doc/source/user/c-info.how-to-extend.rst @@ -174,7 +174,7 @@ rule. There are several converter functions defined in the NumPy C-API that may be of use. In particular, the :c:func:`PyArray_DescrConverter` function is very useful to support arbitrary data-type specification. This function transforms any valid data-type Python object into a -:c:type:`PyArray_Descr *` object. Remember to pass in the address of the +:c:expr:`PyArray_Descr *` object. Remember to pass in the address of the C-variables that should be filled in. There are lots of examples of how to use :c:func:`PyArg_ParseTuple` @@ -192,7 +192,7 @@ It is important to keep in mind that you get a *borrowed* reference to the object when using the "O" format string. However, the converter functions usually require some form of memory handling. In this example, if the conversion is successful, *dtype* will hold a new -reference to a :c:type:`PyArray_Descr *` object, while *input* will hold a +reference to a :c:expr:`PyArray_Descr *` object, while *input* will hold a borrowed reference. Therefore, if this conversion were mixed with another conversion (say to an integer) and the data-type conversion was successful but the integer conversion failed, then you would need @@ -213,9 +213,9 @@ The :c:func:`Py_BuildValue` (format_string, c_variables...) function makes it easy to build tuples of Python objects from C variables. Pay special attention to the difference between 'N' and 'O' in the format string or you can easily create memory leaks. The 'O' format string -increments the reference count of the :c:type:`PyObject *` C-variable it +increments the reference count of the :c:expr:`PyObject *` C-variable it corresponds to, while the 'N' format string steals a reference to the -corresponding :c:type:`PyObject *` C-variable. You should use 'N' if you have +corresponding :c:expr:`PyObject *` C-variable. You should use 'N' if you have already created a reference for the object and just want to give that reference to the tuple. You should use 'O' if you only have a borrowed reference to an object and need to create one to provide for the @@ -510,7 +510,7 @@ by providing default values for common use cases. Getting at ndarray memory and accessing elements of the ndarray --------------------------------------------------------------- -If obj is an ndarray (:c:type:`PyArrayObject *`), then the data-area of the +If obj is an ndarray (:c:expr:`PyArrayObject *`), then the data-area of the ndarray is pointed to by the void* pointer :c:func:`PyArray_DATA` (obj) or the char* pointer :c:func:`PyArray_BYTES` (obj). Remember that (in general) this data-area may not be aligned according to the data-type, it may diff --git a/doc/source/user/depending_on_numpy.rst b/doc/source/user/depending_on_numpy.rst new file mode 100644 index 000000000000..d8e97ef1f967 --- /dev/null +++ b/doc/source/user/depending_on_numpy.rst @@ -0,0 +1,147 @@ +.. _for-downstream-package-authors: + +For downstream package authors +============================== + +This document aims to explain some best practices for authoring a package that +depends on NumPy. + + +Understanding NumPy's versioning and API/ABI stability +------------------------------------------------------ + +NumPy uses a standard, :pep:`440` compliant, versioning scheme: +``major.minor.bugfix``. A *major* release is highly unusual (NumPy is still at +version ``1.xx``) and if it happens it will likely indicate an ABI break. +*Minor* versions are released regularly, typically every 6 months. Minor +versions contain new features, deprecations, and removals of previously +deprecated code. *Bugfix* releases are made even more frequently; they do not +contain any new features or deprecations. + +It is important to know that NumPy, like Python itself and most other +well known scientific Python projects, does **not** use semantic versioning. +Instead, backwards incompatible API changes require deprecation warnings for at +least two releases. For more details, see :ref:`NEP23`. + +NumPy has both a Python API and a C API. The C API can be used directly or via +Cython, f2py, or other such tools. If your package uses the C API, then ABI +(application binary interface) stability of NumPy is important. NumPy's ABI is +forward but not backward compatible. This means: binaries compiled against a +given version of NumPy will still run correctly with newer NumPy versions, but +not with older versions. + + +Testing against the NumPy main branch or pre-releases +----------------------------------------------------- + +For large, actively maintained packages that depend on NumPy, we recommend +testing against the development version of NumPy in CI. To make this easy, +nightly builds are provided as wheels at +https://anaconda.org/scipy-wheels-nightly/. +This helps detect regressions in NumPy that need fixing before the next NumPy +release. Furthermore, we recommend to raise errors on warnings in CI for this +job, either all warnings or otherwise at least ``DeprecationWarning`` and +``FutureWarning``. This gives you an early warning about changes in NumPy to +adapt your code. + + +Adding a dependency on NumPy +---------------------------- + +Build-time dependency +````````````````````` + +If a package either uses the NumPy C API directly or it uses some other tool +that depends on it like Cython or Pythran, NumPy is a *build-time* dependency +of the package. Because the NumPy ABI is only forward compatible, you must +build your own binaries (wheels or other package formats) against the lowest +NumPy version that you support (or an even older version). + +Picking the correct NumPy version to build against for each Python version and +platform can get complicated. There are a couple of ways to do this. +Build-time dependencies are specified in ``pyproject.toml`` (see PEP 517), +which is the file used to build wheels by PEP 517 compliant tools (e.g., +when using ``pip wheel``). + +You can specify everything manually in ``pyproject.toml``, or you can instead +rely on the `oldest-supported-numpy `__ +metapackage. ``oldest-supported-numpy`` will specify the correct NumPy version +at build time for wheels, taking into account Python version, Python +implementation (CPython or PyPy), operating system and hardware platform. It +will specify the oldest NumPy version that supports that combination of +characteristics. Note: for platforms for which NumPy provides wheels on PyPI, +it will be the first version with wheels (even if some older NumPy version +happens to build). + +For conda-forge it's a little less complicated: there's dedicated handling for +NumPy in build-time and runtime dependencies, so typically this is enough +(see `here `__ for docs):: + + host: + - numpy + run: + - {{ pin_compatible('numpy') }} + +.. note:: + + ``pip`` has ``--no-use-pep517`` and ``--no-build-isolation`` flags that may + ignore ``pyproject.toml`` or treat it differently - if users use those + flags, they are responsible for installing the correct build dependencies + themselves. + + ``conda`` will always use ``-no-build-isolation``; dependencies for conda + builds are given in the conda recipe (``meta.yaml``), the ones in + ``pyproject.toml`` have no effect. + + Please do not use ``setup_requires`` (it is deprecated and may invoke + ``easy_install``). + +Because for NumPy you have to care about ABI compatibility, you +specify the version with ``==`` to the lowest supported version. For your other +build dependencies you can probably be looser, however it's still important to +set lower and upper bounds for each dependency. It's fine to specify either a +range or a specific version for a dependency like ``wheel`` or ``setuptools``. +It's recommended to set the upper bound of the range to the latest already +released version of ``wheel`` and ``setuptools`` - this prevents future +releases from breaking your packages on PyPI. + + +Runtime dependency & version ranges +``````````````````````````````````` + +NumPy itself and many core scientific Python packages have agreed on a schedule +for dropping support for old Python and NumPy versions: :ref:`NEP29`. We +recommend all packages depending on NumPy to follow the recommendations in NEP +29. + +For *run-time dependencies*, you specify the range of versions in +``install_requires`` in ``setup.py`` (assuming you use ``numpy.distutils`` or +``setuptools`` to build). Getting the upper bound right for NumPy is slightly +tricky. If we don't set any bound, a too-new version will be pulled in a few +years down the line, and NumPy may have deprecated and removed some API that +your package depended on by then. On the other hand if you set the upper bound +to the newest already-released version, then as soon as a new NumPy version is +released there will be no matching version of your package that works with it. + +What to do here depends on your release frequency. Given that NumPy releases +come in a 6-monthly cadence and that features that get deprecated in NumPy +should stay around for another two releases, a good upper bound is +``<1.(xx+3).0`` - where ``xx`` is the minor version of the latest +already-released NumPy. This is safe to do if you release at least once a year. +If your own releases are much less frequent, you may set the upper bound a +little further into the future - this is a trade-off between a future NumPy +version _maybe_ removing something you rely on, and the upper bound being +exceeded which _may_ lead to your package being hard to install in combination +with other packages relying on the latest NumPy. + + +.. note:: + + + SciPy has more documentation on how it builds wheels and deals with its + build-time and runtime dependencies + `here `__. + + NumPy and SciPy wheel build CI may also be useful as a reference, it can be + found `here for NumPy `__ and + `here for SciPy `__. diff --git a/doc/source/user/explanations_index.rst b/doc/source/user/explanations_index.rst deleted file mode 100644 index f515e36b1437..000000000000 --- a/doc/source/user/explanations_index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _explanations: - -################ -Explanations -################ - -These documents are intended to explain in detail the concepts and techniques -used in NumPy. For the reference documentation of the functions and classes -contained in the package, see the :ref:`API reference `. - -.. toctree:: - :maxdepth: 1 - - basics.broadcasting diff --git a/doc/source/user/how-to-how-to.rst b/doc/source/user/how-to-how-to.rst index de8afc28ae9e..16a2fc7a4acf 100644 --- a/doc/source/user/how-to-how-to.rst +++ b/doc/source/user/how-to-how-to.rst @@ -1,118 +1,118 @@ -.. _how-to-how-to: - -############################################################################## -How to write a NumPy how-to -############################################################################## - -How-tos get straight to the point -- they - - - answer a focused question, or - - narrow a broad question into focused questions that the user can - choose among. - -****************************************************************************** -A stranger has asked for directions... -****************************************************************************** - -**"I need to refuel my car."** - -****************************************************************************** -Give a brief but explicit answer -****************************************************************************** - - - `"Three kilometers/miles, take a right at Hayseed Road, it's on your left."` - -Add helpful details for newcomers ("Hayseed Road", even though it's the only -turnoff at three km/mi). But not irrelevant ones: - - - Don't also give directions from Route 7. - - Don't explain why the town has only one filling station. - -If there's related background (tutorial, explanation, reference, alternative -approach), bring it to the user's attention with a link ("Directions from Route 7," -"Why so few filling stations?"). - - -****************************************************************************** -Delegate -****************************************************************************** - - - `"Three km/mi, take a right at Hayseed Road, follow the signs."` - -If the information is already documented and succinct enough for a how-to, -just link to it, possibly after an introduction ("Three km/mi, take a right"). - -****************************************************************************** -If the question is broad, narrow and redirect it -****************************************************************************** - - **"I want to see the sights."** - -The `See the sights` how-to should link to a set of narrower how-tos: - -- Find historic buildings -- Find scenic lookouts -- Find the town center - -and these might in turn link to still narrower how-tos -- so the town center -page might link to - - - Find the court house - - Find city hall - -By organizing how-tos this way, you not only display the options for people -who need to narrow their question, you also have provided answers for users -who start with narrower questions ("I want to see historic buildings," "Which -way to city hall?"). - -****************************************************************************** -If there are many steps, break them up -****************************************************************************** - -If a how-to has many steps: - - - Consider breaking a step out into an individual how-to and linking to it. - - Include subheadings. They help readers grasp what's coming and return - where they left off. - -****************************************************************************** -Why write how-tos when there's Stack Overflow, Reddit, Gitter...? -****************************************************************************** - - - We have authoritative answers. - - How-tos make the site less forbidding to non-experts. - - How-tos bring people into the site and help them discover other information - that's here . - - Creating how-tos helps us see NumPy usability through new eyes. - -****************************************************************************** -Aren't how-tos and tutorials the same thing? -****************************************************************************** - -People use the terms "how-to" and "tutorial" interchangeably, but we draw a -distinction, following Daniele Procida's `taxonomy of documentation`_. - - .. _`taxonomy of documentation`: https://documentation.divio.com/ - -Documentation needs to meet users where they are. `How-tos` offer get-it-done -information; the user wants steps to copy and doesn't necessarily want to -understand NumPy. `Tutorials` are warm-fuzzy information; the user wants a -feel for some aspect of NumPy (and again, may or may not care about deeper -knowledge). - -We distinguish both tutorials and how-tos from `Explanations`, which are -deep dives intended to give understanding rather than immediate assistance, -and `References`, which give complete, autoritative data on some concrete -part of NumPy (like its API) but aren't obligated to paint a broader picture. - -For more on tutorials, see the `tutorial how-to`_. - -.. _`tutorial how-to`: https://github.com/numpy/numpy-tutorials/blob/master/tutorial_style.ipynb - - -****************************************************************************** -Is this page an example of a how-to? -****************************************************************************** - -Yes -- until the sections with question-mark headings; they explain rather -than giving directions. In a how-to, those would be links. \ No newline at end of file +.. _how-to-how-to: + +############################################################################## +How to write a NumPy how-to +############################################################################## + +How-tos get straight to the point -- they + + - answer a focused question, or + - narrow a broad question into focused questions that the user can + choose among. + +****************************************************************************** +A stranger has asked for directions... +****************************************************************************** + +**"I need to refuel my car."** + +****************************************************************************** +Give a brief but explicit answer +****************************************************************************** + + - `"Three kilometers/miles, take a right at Hayseed Road, it's on your left."` + +Add helpful details for newcomers ("Hayseed Road", even though it's the only +turnoff at three km/mi). But not irrelevant ones: + + - Don't also give directions from Route 7. + - Don't explain why the town has only one filling station. + +If there's related background (tutorial, explanation, reference, alternative +approach), bring it to the user's attention with a link ("Directions from Route 7," +"Why so few filling stations?"). + + +****************************************************************************** +Delegate +****************************************************************************** + + - `"Three km/mi, take a right at Hayseed Road, follow the signs."` + +If the information is already documented and succinct enough for a how-to, +just link to it, possibly after an introduction ("Three km/mi, take a right"). + +****************************************************************************** +If the question is broad, narrow and redirect it +****************************************************************************** + + **"I want to see the sights."** + +The `See the sights` how-to should link to a set of narrower how-tos: + +- Find historic buildings +- Find scenic lookouts +- Find the town center + +and these might in turn link to still narrower how-tos -- so the town center +page might link to + + - Find the court house + - Find city hall + +By organizing how-tos this way, you not only display the options for people +who need to narrow their question, you also have provided answers for users +who start with narrower questions ("I want to see historic buildings," "Which +way to city hall?"). + +****************************************************************************** +If there are many steps, break them up +****************************************************************************** + +If a how-to has many steps: + + - Consider breaking a step out into an individual how-to and linking to it. + - Include subheadings. They help readers grasp what's coming and return + where they left off. + +****************************************************************************** +Why write how-tos when there's Stack Overflow, Reddit, Gitter...? +****************************************************************************** + + - We have authoritative answers. + - How-tos make the site less forbidding to non-experts. + - How-tos bring people into the site and help them discover other information + that's here . + - Creating how-tos helps us see NumPy usability through new eyes. + +****************************************************************************** +Aren't how-tos and tutorials the same thing? +****************************************************************************** + +People use the terms "how-to" and "tutorial" interchangeably, but we draw a +distinction, following Daniele Procida's `taxonomy of documentation`_. + + .. _`taxonomy of documentation`: https://documentation.divio.com/ + +Documentation needs to meet users where they are. `How-tos` offer get-it-done +information; the user wants steps to copy and doesn't necessarily want to +understand NumPy. `Tutorials` are warm-fuzzy information; the user wants a +feel for some aspect of NumPy (and again, may or may not care about deeper +knowledge). + +We distinguish both tutorials and how-tos from `Explanations`, which are +deep dives intended to give understanding rather than immediate assistance, +and `References`, which give complete, autoritative data on some concrete +part of NumPy (like its API) but aren't obligated to paint a broader picture. + +For more on tutorials, see the `tutorial how-to`_. + +.. _`tutorial how-to`: https://github.com/numpy/numpy-tutorials/blob/master/tutorial_style.ipynb + + +****************************************************************************** +Is this page an example of a how-to? +****************************************************************************** + +Yes -- until the sections with question-mark headings; they explain rather +than giving directions. In a how-to, those would be links. diff --git a/doc/source/user/how-to-io.rst b/doc/source/user/how-to-io.rst index ca9fc41f07f8..d238ccbb605e 100644 --- a/doc/source/user/how-to-io.rst +++ b/doc/source/user/how-to-io.rst @@ -1,328 +1,328 @@ -.. _how-to-io: - -############################################################################## -Reading and writing files -############################################################################## - -This page tackles common applications; for the full collection of I/O -routines, see :ref:`routines.io`. - - -****************************************************************************** -Reading text and CSV_ files -****************************************************************************** - -.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values - -With no missing values -============================================================================== - -Use :func:`numpy.loadtxt`. - -With missing values -============================================================================== - -Use :func:`numpy.genfromtxt`. - -:func:`numpy.genfromtxt` will either - - - return a :ref:`masked array` - **masking out missing values** (if ``usemask=True``), or - - - **fill in the missing value** with the value specified in - ``filling_values`` (default is ``np.nan`` for float, -1 for int). - -With non-whitespace delimiters ------------------------------------------------------------------------------- -:: - - >>> print(open("csv.txt").read()) # doctest: +SKIP - 1, 2, 3 - 4,, 6 - 7, 8, 9 - - -Masked-array output -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:: - - >>> np.genfromtxt("csv.txt", delimiter=",", usemask=True) # doctest: +SKIP - masked_array( - data=[[1.0, 2.0, 3.0], - [4.0, --, 6.0], - [7.0, 8.0, 9.0]], - mask=[[False, False, False], - [False, True, False], - [False, False, False]], - fill_value=1e+20) - -Array output -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:: - - >>> np.genfromtxt("csv.txt", delimiter=",") # doctest: +SKIP - array([[ 1., 2., 3.], - [ 4., nan, 6.], - [ 7., 8., 9.]]) - -Array output, specified fill-in value -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:: - - >>> np.genfromtxt("csv.txt", delimiter=",", dtype=np.int8, filling_values=99) # doctest: +SKIP - array([[ 1, 2, 3], - [ 4, 99, 6], - [ 7, 8, 9]], dtype=int8) - -Whitespace-delimited -------------------------------------------------------------------------------- - -:func:`numpy.genfromtxt` can also parse whitespace-delimited data files -that have missing values if - -* **Each field has a fixed width**: Use the width as the `delimiter` argument. - :: - - # File with width=4. The data does not have to be justified (for example, - # the 2 in row 1), the last column can be less than width (for example, the 6 - # in row 2), and no delimiting character is required (for instance 8888 and 9 - # in row 3) - - >>> f = open("fixedwidth.txt").read() # doctest: +SKIP - >>> print(f) # doctest: +SKIP - 1 2 3 - 44 6 - 7 88889 - - # Showing spaces as ^ - >>> print(f.replace(" ","^")) # doctest: +SKIP - 1^^^2^^^^^^3 - 44^^^^^^6 - 7^^^88889 - - >>> np.genfromtxt("fixedwidth.txt", delimiter=4) # doctest: +SKIP - array([[1.000e+00, 2.000e+00, 3.000e+00], - [4.400e+01, nan, 6.000e+00], - [7.000e+00, 8.888e+03, 9.000e+00]]) - -* **A special value (e.g. "x") indicates a missing field**: Use it as the - `missing_values` argument. - :: - - >>> print(open("nan.txt").read()) # doctest: +SKIP - 1 2 3 - 44 x 6 - 7 8888 9 - - >>> np.genfromtxt("nan.txt", missing_values="x") # doctest: +SKIP - array([[1.000e+00, 2.000e+00, 3.000e+00], - [4.400e+01, nan, 6.000e+00], - [7.000e+00, 8.888e+03, 9.000e+00]]) - -* **You want to skip the rows with missing values**: Set - `invalid_raise=False`. - :: - - >>> print(open("skip.txt").read()) # doctest: +SKIP - 1 2 3 - 44 6 - 7 888 9 - - >>> np.genfromtxt("skip.txt", invalid_raise=False) # doctest: +SKIP - __main__:1: ConversionWarning: Some errors were detected ! - Line #2 (got 2 columns instead of 3) - array([[ 1., 2., 3.], - [ 7., 888., 9.]]) - - -* **The delimiter whitespace character is different from the whitespace that - indicates missing data**. For instance, if columns are delimited by ``\t``, - then missing data will be recognized if it consists of one - or more spaces. - :: - - >>> f = open("tabs.txt").read() # doctest: +SKIP - >>> print(f) # doctest: +SKIP - 1 2 3 - 44 6 - 7 888 9 - - # Tabs vs. spaces - >>> print(f.replace("\t","^")) # doctest: +SKIP - 1^2^3 - 44^ ^6 - 7^888^9 - - >>> np.genfromtxt("tabs.txt", delimiter="\t", missing_values=" +") # doctest: +SKIP - array([[ 1., 2., 3.], - [ 44., nan, 6.], - [ 7., 888., 9.]]) - -****************************************************************************** -Read a file in .npy or .npz format -****************************************************************************** - -Choices: - - - Use :func:`numpy.load`. It can read files generated by any of - :func:`numpy.save`, :func:`numpy.savez`, or :func:`numpy.savez_compressed`. - - - Use memory mapping. See `numpy.lib.format.open_memmap`. - -****************************************************************************** -Write to a file to be read back by NumPy -****************************************************************************** - -Binary -=============================================================================== - -Use -:func:`numpy.save`, or to store multiple arrays :func:`numpy.savez` -or :func:`numpy.savez_compressed`. - -For :ref:`security and portability `, set -``allow_pickle=False`` unless the dtype contains Python objects, which -requires pickling. - -Masked arrays :any:`can't currently be saved `, -nor can other arbitrary array subclasses. - -Human-readable -============================================================================== - -:func:`numpy.save` and :func:`numpy.savez` create binary files. To **write a -human-readable file**, use :func:`numpy.savetxt`. The array can only be 1- or -2-dimensional, and there's no ` savetxtz` for multiple files. - -Large arrays -============================================================================== - -See :ref:`how-to-io-large-arrays`. - -****************************************************************************** -Read an arbitrarily formatted binary file ("binary blob") -****************************************************************************** - -Use a :doc:`structured array `. - -**Example:** - -The ``.wav`` file header is a 44-byte block preceding ``data_size`` bytes of the -actual sound data:: - - chunk_id "RIFF" - chunk_size 4-byte unsigned little-endian integer - format "WAVE" - fmt_id "fmt " - fmt_size 4-byte unsigned little-endian integer - audio_fmt 2-byte unsigned little-endian integer - num_channels 2-byte unsigned little-endian integer - sample_rate 4-byte unsigned little-endian integer - byte_rate 4-byte unsigned little-endian integer - block_align 2-byte unsigned little-endian integer - bits_per_sample 2-byte unsigned little-endian integer - data_id "data" - data_size 4-byte unsigned little-endian integer - -The ``.wav`` file header as a NumPy structured dtype:: - - wav_header_dtype = np.dtype([ - ("chunk_id", (bytes, 4)), # flexible-sized scalar type, item size 4 - ("chunk_size", "`_.) - -.. _how-to-io-large-arrays: - -****************************************************************************** -Write or read large arrays -****************************************************************************** - -**Arrays too large to fit in memory** can be treated like ordinary in-memory -arrays using memory mapping. - -- Raw array data written with :func:`numpy.ndarray.tofile` or - :func:`numpy.ndarray.tobytes` can be read with :func:`numpy.memmap`:: - - array = numpy.memmap("mydata/myarray.arr", mode="r", dtype=np.int16, shape=(1024, 1024)) - -- Files output by :func:`numpy.save` (that is, using the numpy format) can be read - using :func:`numpy.load` with the ``mmap_mode`` keyword argument:: - - large_array[some_slice] = np.load("path/to/small_array", mmap_mode="r") - -Memory mapping lacks features like data chunking and compression; more -full-featured formats and libraries usable with NumPy include: - -* **HDF5**: `h5py `_ or `PyTables `_. -* **Zarr**: `here `_. -* **NetCDF**: :class:`scipy.io.netcdf_file`. - -For tradeoffs among memmap, Zarr, and HDF5, see -`pythonspeed.com `_. - -****************************************************************************** -Write files for reading by other (non-NumPy) tools -****************************************************************************** - -Formats for **exchanging data** with other tools include HDF5, Zarr, and -NetCDF (see :ref:`how-to-io-large-arrays`). - -****************************************************************************** -Write or read a JSON file -****************************************************************************** - -NumPy arrays are **not** directly -`JSON serializable `_. - - -.. _how-to-io-pickle-file: - -****************************************************************************** -Save/restore using a pickle file -****************************************************************************** - -Avoid when possible; :doc:`pickles ` are not secure -against erroneous or maliciously constructed data. - -Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, -unless the array dtype includes Python objects, in which case pickling is -required. - -****************************************************************************** -Convert from a pandas DataFrame to a NumPy array -****************************************************************************** - -See :meth:`pandas.DataFrame.to_numpy`. - -****************************************************************************** - Save/restore using `~numpy.ndarray.tofile` and `~numpy.fromfile` -****************************************************************************** - -In general, prefer :func:`numpy.save` and :func:`numpy.load`. - -:func:`numpy.ndarray.tofile` and :func:`numpy.fromfile` lose information on -endianness and precision and so are unsuitable for anything but scratch -storage. - +.. _how-to-io: + +############################################################################## +Reading and writing files +############################################################################## + +This page tackles common applications; for the full collection of I/O +routines, see :ref:`routines.io`. + + +****************************************************************************** +Reading text and CSV_ files +****************************************************************************** + +.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values + +With no missing values +============================================================================== + +Use :func:`numpy.loadtxt`. + +With missing values +============================================================================== + +Use :func:`numpy.genfromtxt`. + +:func:`numpy.genfromtxt` will either + + - return a :ref:`masked array` + **masking out missing values** (if ``usemask=True``), or + + - **fill in the missing value** with the value specified in + ``filling_values`` (default is ``np.nan`` for float, -1 for int). + +With non-whitespace delimiters +------------------------------------------------------------------------------ +:: + + >>> print(open("csv.txt").read()) # doctest: +SKIP + 1, 2, 3 + 4,, 6 + 7, 8, 9 + + +Masked-array output +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",", usemask=True) # doctest: +SKIP + masked_array( + data=[[1.0, 2.0, 3.0], + [4.0, --, 6.0], + [7.0, 8.0, 9.0]], + mask=[[False, False, False], + [False, True, False], + [False, False, False]], + fill_value=1e+20) + +Array output +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 4., nan, 6.], + [ 7., 8., 9.]]) + +Array output, specified fill-in value +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + >>> np.genfromtxt("csv.txt", delimiter=",", dtype=np.int8, filling_values=99) # doctest: +SKIP + array([[ 1, 2, 3], + [ 4, 99, 6], + [ 7, 8, 9]], dtype=int8) + +Whitespace-delimited +------------------------------------------------------------------------------- + +:func:`numpy.genfromtxt` can also parse whitespace-delimited data files +that have missing values if + +* **Each field has a fixed width**: Use the width as the `delimiter` argument. + :: + + # File with width=4. The data does not have to be justified (for example, + # the 2 in row 1), the last column can be less than width (for example, the 6 + # in row 2), and no delimiting character is required (for instance 8888 and 9 + # in row 3) + + >>> f = open("fixedwidth.txt").read() # doctest: +SKIP + >>> print(f) # doctest: +SKIP + 1 2 3 + 44 6 + 7 88889 + + # Showing spaces as ^ + >>> print(f.replace(" ","^")) # doctest: +SKIP + 1^^^2^^^^^^3 + 44^^^^^^6 + 7^^^88889 + + >>> np.genfromtxt("fixedwidth.txt", delimiter=4) # doctest: +SKIP + array([[1.000e+00, 2.000e+00, 3.000e+00], + [4.400e+01, nan, 6.000e+00], + [7.000e+00, 8.888e+03, 9.000e+00]]) + +* **A special value (e.g. "x") indicates a missing field**: Use it as the + `missing_values` argument. + :: + + >>> print(open("nan.txt").read()) # doctest: +SKIP + 1 2 3 + 44 x 6 + 7 8888 9 + + >>> np.genfromtxt("nan.txt", missing_values="x") # doctest: +SKIP + array([[1.000e+00, 2.000e+00, 3.000e+00], + [4.400e+01, nan, 6.000e+00], + [7.000e+00, 8.888e+03, 9.000e+00]]) + +* **You want to skip the rows with missing values**: Set + `invalid_raise=False`. + :: + + >>> print(open("skip.txt").read()) # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 + + >>> np.genfromtxt("skip.txt", invalid_raise=False) # doctest: +SKIP + __main__:1: ConversionWarning: Some errors were detected ! + Line #2 (got 2 columns instead of 3) + array([[ 1., 2., 3.], + [ 7., 888., 9.]]) + + +* **The delimiter whitespace character is different from the whitespace that + indicates missing data**. For instance, if columns are delimited by ``\t``, + then missing data will be recognized if it consists of one + or more spaces. + :: + + >>> f = open("tabs.txt").read() # doctest: +SKIP + >>> print(f) # doctest: +SKIP + 1 2 3 + 44 6 + 7 888 9 + + # Tabs vs. spaces + >>> print(f.replace("\t","^")) # doctest: +SKIP + 1^2^3 + 44^ ^6 + 7^888^9 + + >>> np.genfromtxt("tabs.txt", delimiter="\t", missing_values=" +") # doctest: +SKIP + array([[ 1., 2., 3.], + [ 44., nan, 6.], + [ 7., 888., 9.]]) + +****************************************************************************** +Read a file in .npy or .npz format +****************************************************************************** + +Choices: + + - Use :func:`numpy.load`. It can read files generated by any of + :func:`numpy.save`, :func:`numpy.savez`, or :func:`numpy.savez_compressed`. + + - Use memory mapping. See `numpy.lib.format.open_memmap`. + +****************************************************************************** +Write to a file to be read back by NumPy +****************************************************************************** + +Binary +=============================================================================== + +Use +:func:`numpy.save`, or to store multiple arrays :func:`numpy.savez` +or :func:`numpy.savez_compressed`. + +For :ref:`security and portability `, set +``allow_pickle=False`` unless the dtype contains Python objects, which +requires pickling. + +Masked arrays :any:`can't currently be saved `, +nor can other arbitrary array subclasses. + +Human-readable +============================================================================== + +:func:`numpy.save` and :func:`numpy.savez` create binary files. To **write a +human-readable file**, use :func:`numpy.savetxt`. The array can only be 1- or +2-dimensional, and there's no ` savetxtz` for multiple files. + +Large arrays +============================================================================== + +See :ref:`how-to-io-large-arrays`. + +****************************************************************************** +Read an arbitrarily formatted binary file ("binary blob") +****************************************************************************** + +Use a :doc:`structured array `. + +**Example:** + +The ``.wav`` file header is a 44-byte block preceding ``data_size`` bytes of the +actual sound data:: + + chunk_id "RIFF" + chunk_size 4-byte unsigned little-endian integer + format "WAVE" + fmt_id "fmt " + fmt_size 4-byte unsigned little-endian integer + audio_fmt 2-byte unsigned little-endian integer + num_channels 2-byte unsigned little-endian integer + sample_rate 4-byte unsigned little-endian integer + byte_rate 4-byte unsigned little-endian integer + block_align 2-byte unsigned little-endian integer + bits_per_sample 2-byte unsigned little-endian integer + data_id "data" + data_size 4-byte unsigned little-endian integer + +The ``.wav`` file header as a NumPy structured dtype:: + + wav_header_dtype = np.dtype([ + ("chunk_id", (bytes, 4)), # flexible-sized scalar type, item size 4 + ("chunk_size", "`_.) + +.. _how-to-io-large-arrays: + +****************************************************************************** +Write or read large arrays +****************************************************************************** + +**Arrays too large to fit in memory** can be treated like ordinary in-memory +arrays using memory mapping. + +- Raw array data written with :func:`numpy.ndarray.tofile` or + :func:`numpy.ndarray.tobytes` can be read with :func:`numpy.memmap`:: + + array = numpy.memmap("mydata/myarray.arr", mode="r", dtype=np.int16, shape=(1024, 1024)) + +- Files output by :func:`numpy.save` (that is, using the numpy format) can be read + using :func:`numpy.load` with the ``mmap_mode`` keyword argument:: + + large_array[some_slice] = np.load("path/to/small_array", mmap_mode="r") + +Memory mapping lacks features like data chunking and compression; more +full-featured formats and libraries usable with NumPy include: + +* **HDF5**: `h5py `_ or `PyTables `_. +* **Zarr**: `here `_. +* **NetCDF**: :class:`scipy.io.netcdf_file`. + +For tradeoffs among memmap, Zarr, and HDF5, see +`pythonspeed.com `_. + +****************************************************************************** +Write files for reading by other (non-NumPy) tools +****************************************************************************** + +Formats for **exchanging data** with other tools include HDF5, Zarr, and +NetCDF (see :ref:`how-to-io-large-arrays`). + +****************************************************************************** +Write or read a JSON file +****************************************************************************** + +NumPy arrays are **not** directly +`JSON serializable `_. + + +.. _how-to-io-pickle-file: + +****************************************************************************** +Save/restore using a pickle file +****************************************************************************** + +Avoid when possible; :doc:`pickles ` are not secure +against erroneous or maliciously constructed data. + +Use :func:`numpy.save` and :func:`numpy.load`. Set ``allow_pickle=False``, +unless the array dtype includes Python objects, in which case pickling is +required. + +****************************************************************************** +Convert from a pandas DataFrame to a NumPy array +****************************************************************************** + +See :meth:`pandas.DataFrame.to_numpy`. + +****************************************************************************** + Save/restore using `~numpy.ndarray.tofile` and `~numpy.fromfile` +****************************************************************************** + +In general, prefer :func:`numpy.save` and :func:`numpy.load`. + +:func:`numpy.ndarray.tofile` and :func:`numpy.fromfile` lose information on +endianness and precision and so are unsuitable for anything but scratch +storage. + diff --git a/doc/source/user/images/np_create_matrix.png b/doc/source/user/images/np_create_matrix.png index cd685c4f5a00..65e4535e5708 100644 Binary files a/doc/source/user/images/np_create_matrix.png and b/doc/source/user/images/np_create_matrix.png differ diff --git a/doc/source/user/images/np_indexing.png b/doc/source/user/images/np_indexing.png index 4303ec35b18b..863b2d46f66a 100644 Binary files a/doc/source/user/images/np_indexing.png and b/doc/source/user/images/np_indexing.png differ diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 28297d9eaa9b..b47d6634eb01 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -23,6 +23,7 @@ details are found in :ref:`reference`. c-info tutorials_index howtos_index + depending_on_numpy .. Links to these files are placed directly in the top-level html @@ -34,7 +35,6 @@ details are found in :ref:`reference`. .. toctree:: :hidden: - explanations_index ../f2py/index ../glossary ../dev/underthehood diff --git a/doc/source/user/install.rst b/doc/source/user/install.rst index e05cee2f130d..b9425701f352 100644 --- a/doc/source/user/install.rst +++ b/doc/source/user/install.rst @@ -1,7 +1,7 @@ -:orphan: - -**************** -Installing NumPy -**************** - +:orphan: + +**************** +Installing NumPy +**************** + See `Installing NumPy `_. \ No newline at end of file diff --git a/doc/source/user/misc.rst b/doc/source/user/misc.rst index 031ce4efac9a..f0a7f5e4c896 100644 --- a/doc/source/user/misc.rst +++ b/doc/source/user/misc.rst @@ -149,11 +149,12 @@ Only a survey of the choices. Little detail on how each works. - good numpy support: arrays have all these in their ctypes attribute: :: - a.ctypes.data a.ctypes.get_strides - a.ctypes.data_as a.ctypes.shape - a.ctypes.get_as_parameter a.ctypes.shape_as - a.ctypes.get_data a.ctypes.strides - a.ctypes.get_shape a.ctypes.strides_as + a.ctypes.data + a.ctypes.data_as + a.ctypes.shape + a.ctypes.shape_as + a.ctypes.strides + a.ctypes.strides_as - Minuses: diff --git a/doc/source/user/plots/matplotlib3.py b/doc/source/user/plots/matplotlib3.py index 20a8c0767321..7b56067ef463 100644 --- a/doc/source/user/plots/matplotlib3.py +++ b/doc/source/user/plots/matplotlib3.py @@ -1,9 +1,8 @@ import numpy as np import matplotlib.pyplot as plt -from mpl_toolkits.mplot3d import Axes3D fig = plt.figure() -ax = Axes3D(fig) +ax = fig.add_subplot(projection='3d') X = np.arange(-5, 5, 0.15) Y = np.arange(-5, 5, 0.15) X, Y = np.meshgrid(X, Y) diff --git a/doc/source/user/quickstart.rst b/doc/source/user/quickstart.rst index 8fdc6ec3649f..9f3d6a040dd0 100644 --- a/doc/source/user/quickstart.rst +++ b/doc/source/user/quickstart.rst @@ -20,8 +20,7 @@ in addition to NumPy. **Learner profile** -This is a quick overview of -algebra and arrays in NumPy. It demonstrates how n-dimensional +This is a quick overview of arrays in NumPy. It demonstrates how n-dimensional (:math:`n>=2`) arrays are represented and can be manipulated. In particular, if you don't know how to apply common functions to n-dimensional arrays (without using for-loops), or if you want to understand axis and shape properties for @@ -53,8 +52,8 @@ axis has a length of 2, the second axis has a length of 3. :: - [[ 1., 0., 0.], - [ 0., 1., 2.]] + [[1., 0., 0.], + [0., 1., 2.]] NumPy's array class is called ``ndarray``. It is also known by the alias ``array``. Note that ``numpy.array`` is not the same as the Standard @@ -128,7 +127,7 @@ from the type of the elements in the sequences. :: >>> import numpy as np - >>> a = np.array([2,3,4]) + >>> a = np.array([2, 3, 4]) >>> a array([2, 3, 4]) >>> a.dtype @@ -142,11 +141,11 @@ rather than providing a single sequence as an argument. :: - >>> a = np.array(1,2,3,4) # WRONG + >>> a = np.array(1, 2, 3, 4) # WRONG Traceback (most recent call last): ... TypeError: array() takes from 1 to 2 positional arguments but 4 were given - >>> a = np.array([1,2,3,4]) # RIGHT + >>> a = np.array([1, 2, 3, 4]) # RIGHT ``array`` transforms sequences of sequences into two-dimensional arrays, sequences of sequences of sequences into three-dimensional arrays, and @@ -154,7 +153,7 @@ so on. :: - >>> b = np.array([(1.5,2,3), (4,5,6)]) + >>> b = np.array([(1.5, 2, 3), (4, 5, 6)]) >>> b array([[1.5, 2. , 3. ], [4. , 5. , 6. ]]) @@ -163,7 +162,7 @@ The type of the array can also be explicitly specified at creation time: :: - >>> c = np.array( [ [1,2], [3,4] ], dtype=complex ) + >>> c = np.array([[1, 2], [3, 4]], dtype=complex) >>> c array([[1.+0.j, 2.+0.j], [3.+0.j, 4.+0.j]]) @@ -177,7 +176,7 @@ The function ``zeros`` creates an array full of zeros, the function ``ones`` creates an array full of ones, and the function ``empty`` creates an array whose initial content is random and depends on the state of the memory. By default, the dtype of the created array is -``float64``. +``float64``, but it can be specified via the key word argument ``dtype``. :: @@ -185,7 +184,7 @@ state of the memory. By default, the dtype of the created array is array([[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]]) - >>> np.ones( (2,3,4), dtype=np.int16 ) # dtype can also be specified + >>> np.ones((2, 3, 4), dtype=np.int16) array([[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]], @@ -193,9 +192,9 @@ state of the memory. By default, the dtype of the created array is [[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]]], dtype=int16) - >>> np.empty( (2,3) ) # uninitialized - array([[ 3.73603959e-262, 6.02658058e-154, 6.55490914e-260], # may vary - [ 5.30498948e-313, 3.14673309e-307, 1.00000000e+000]]) + >>> np.empty((2, 3)) + array([[3.73603959e-262, 6.02658058e-154, 6.55490914e-260], # may vary + [5.30498948e-313, 3.14673309e-307, 1.00000000e+000]]) To create sequences of numbers, NumPy provides the ``arange`` function which is analogous to the Python built-in ``range``, but returns an @@ -203,9 +202,9 @@ array. :: - >>> np.arange( 10, 30, 5 ) + >>> np.arange(10, 30, 5) array([10, 15, 20, 25]) - >>> np.arange( 0, 2, 0.3 ) # it accepts float arguments + >>> np.arange(0, 2, 0.3) # it accepts float arguments array([0. , 0.3, 0.6, 0.9, 1.2, 1.5, 1.8]) When ``arange`` is used with floating point arguments, it is generally @@ -215,9 +214,9 @@ to use the function ``linspace`` that receives as an argument the number of elements that we want, instead of the step:: >>> from numpy import pi - >>> np.linspace( 0, 2, 9 ) # 9 numbers from 0 to 2 + >>> np.linspace(0, 2, 9) # 9 numbers from 0 to 2 array([0. , 0.25, 0.5 , 0.75, 1. , 1.25, 1.5 , 1.75, 2. ]) - >>> x = np.linspace( 0, 2*pi, 100 ) # useful to evaluate function at lots of points + >>> x = np.linspace(0, 2 * pi, 100) # useful to evaluate function at lots of points >>> f = np.sin(x) .. seealso:: @@ -251,18 +250,18 @@ matrices and tridimensionals as lists of matrices. :: - >>> a = np.arange(6) # 1d array + >>> a = np.arange(6) # 1d array >>> print(a) [0 1 2 3 4 5] - >>> - >>> b = np.arange(12).reshape(4,3) # 2d array + >>> + >>> b = np.arange(12).reshape(4, 3) # 2d array >>> print(b) [[ 0 1 2] [ 3 4 5] [ 6 7 8] [ 9 10 11]] - >>> - >>> c = np.arange(24).reshape(2,3,4) # 3d array + >>> + >>> c = np.arange(24).reshape(2, 3, 4) # 3d array >>> print(c) [[[ 0 1 2 3] [ 4 5 6 7] @@ -280,8 +279,8 @@ central part of the array and only prints the corners:: >>> print(np.arange(10000)) [ 0 1 2 ... 9997 9998 9999] - >>> - >>> print(np.arange(10000).reshape(100,100)) + >>> + >>> print(np.arange(10000).reshape(100, 100)) [[ 0 1 2 ... 97 98 99] [ 100 101 102 ... 197 198 199] [ 200 201 202 ... 297 298 299] @@ -295,7 +294,7 @@ can change the printing options using ``set_printoptions``. :: - >>> np.set_printoptions(threshold=sys.maxsize) # sys module should be imported + >>> np.set_printoptions(threshold=sys.maxsize) # sys module should be imported .. _quickstart.basic-operations: @@ -308,35 +307,35 @@ created and filled with the result. :: - >>> a = np.array( [20,30,40,50] ) - >>> b = np.arange( 4 ) + >>> a = np.array([20, 30, 40, 50]) + >>> b = np.arange(4) >>> b array([0, 1, 2, 3]) - >>> c = a-b + >>> c = a - b >>> c array([20, 29, 38, 47]) >>> b**2 array([0, 1, 4, 9]) - >>> 10*np.sin(a) + >>> 10 * np.sin(a) array([ 9.12945251, -9.88031624, 7.4511316 , -2.62374854]) - >>> a<35 + >>> a < 35 array([ True, True, False, False]) Unlike in many matrix languages, the product operator ``*`` operates elementwise in NumPy arrays. The matrix product can be performed using the ``@`` operator (in python >=3.5) or the ``dot`` function or method:: - >>> A = np.array( [[1,1], - ... [0,1]] ) - >>> B = np.array( [[2,0], - ... [3,4]] ) - >>> A * B # elementwise product + >>> A = np.array([[1, 1], + ... [0, 1]]) + >>> B = np.array([[2, 0], + ... [3, 4]]) + >>> A * B # elementwise product array([[2, 0], [0, 4]]) - >>> A @ B # matrix product + >>> A @ B # matrix product array([[5, 4], [3, 4]]) - >>> A.dot(B) # another matrix product + >>> A.dot(B) # another matrix product array([[5, 4], [3, 4]]) @@ -345,9 +344,9 @@ existing array rather than create a new one. :: - >>> rg = np.random.default_rng(1) # create instance of default random number generator - >>> a = np.ones((2,3), dtype=int) - >>> b = rg.random((2,3)) + >>> rg = np.random.default_rng(1) # create instance of default random number generator + >>> a = np.ones((2, 3), dtype=int) + >>> b = rg.random((2, 3)) >>> a *= 3 >>> a array([[3, 3, 3], @@ -356,7 +355,7 @@ existing array rather than create a new one. >>> b array([[3.51182162, 3.9504637 , 3.14415961], [3.94864945, 3.31183145, 3.42332645]]) - >>> a += b # b is not automatically converted to integer type + >>> a += b # b is not automatically converted to integer type Traceback (most recent call last): ... numpy.core._exceptions._UFuncOutputCastingError: Cannot cast ufunc 'add' output from dtype('float64') to dtype('int64') with casting rule 'same_kind' @@ -368,15 +367,15 @@ as upcasting). :: >>> a = np.ones(3, dtype=np.int32) - >>> b = np.linspace(0,pi,3) + >>> b = np.linspace(0, pi, 3) >>> b.dtype.name 'float64' - >>> c = a+b + >>> c = a + b >>> c array([1. , 2.57079633, 4.14159265]) >>> c.dtype.name 'float64' - >>> d = np.exp(c*1j) + >>> d = np.exp(c * 1j) >>> d array([ 0.54030231+0.84147098j, -0.84147098+0.54030231j, -0.54030231-0.84147098j]) @@ -388,7 +387,7 @@ the array, are implemented as methods of the ``ndarray`` class. :: - >>> a = rg.random((2,3)) + >>> a = rg.random((2, 3)) >>> a array([[0.82770259, 0.40919914, 0.54959369], [0.02755911, 0.75351311, 0.53814331]]) @@ -404,19 +403,19 @@ of numbers, regardless of its shape. However, by specifying the ``axis`` parameter you can apply an operation along the specified axis of an array:: - >>> b = np.arange(12).reshape(3,4) + >>> b = np.arange(12).reshape(3, 4) >>> b array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) >>> - >>> b.sum(axis=0) # sum of each column + >>> b.sum(axis=0) # sum of each column array([12, 15, 18, 21]) >>> - >>> b.min(axis=1) # min of each row + >>> b.min(axis=1) # min of each row array([0, 4, 8]) >>> - >>> b.cumsum(axis=1) # cumulative sum along each row + >>> b.cumsum(axis=1) # cumulative sum along each row array([[ 0, 1, 3, 6], [ 4, 9, 15, 22], [ 8, 17, 27, 38]]) @@ -427,7 +426,7 @@ Universal Functions NumPy provides familiar mathematical functions such as sin, cos, and exp. In NumPy, these are called "universal -functions"(\ ``ufunc``). Within NumPy, these functions +functions" (\ ``ufunc``). Within NumPy, these functions operate elementwise on an array, producing an array as output. :: @@ -507,15 +506,15 @@ and other Python sequences. 8 >>> a[2:5] array([ 8, 27, 64]) - # equivalent to a[0:6:2] = 1000; - # from start to position 6, exclusive, set every 2nd element to 1000 + >>> # equivalent to a[0:6:2] = 1000; + >>> # from start to position 6, exclusive, set every 2nd element to 1000 >>> a[:6:2] = 1000 >>> a array([1000, 1, 1000, 27, 1000, 125, 216, 343, 512, 729]) - >>> a[ : :-1] # reversed a + >>> a[::-1] # reversed a array([ 729, 512, 343, 216, 125, 1000, 27, 1000, 1, 1000]) >>> for i in a: - ... print(i**(1/3.)) + ... print(i**(1 / 3.)) ... 9.999999999999998 1.0 @@ -532,23 +531,23 @@ and other Python sequences. **Multidimensional** arrays can have one index per axis. These indices are given in a tuple separated by commas:: - >>> def f(x,y): - ... return 10*x+y + >>> def f(x, y): + ... return 10 * x + y ... - >>> b = np.fromfunction(f,(5,4),dtype=int) + >>> b = np.fromfunction(f, (5, 4), dtype=int) >>> b array([[ 0, 1, 2, 3], [10, 11, 12, 13], [20, 21, 22, 23], [30, 31, 32, 33], [40, 41, 42, 43]]) - >>> b[2,3] + >>> b[2, 3] 23 - >>> b[0:5, 1] # each row in the second column of b + >>> b[0:5, 1] # each row in the second column of b array([ 1, 11, 21, 31, 41]) - >>> b[ : ,1] # equivalent to the previous example + >>> b[:, 1] # equivalent to the previous example array([ 1, 11, 21, 31, 41]) - >>> b[1:3, : ] # each column in the second and third row of b + >>> b[1:3, :] # each column in the second and third row of b array([[10, 11, 12, 13], [20, 21, 22, 23]]) @@ -557,34 +556,34 @@ indices are considered complete slices\ ``:`` :: - >>> b[-1] # the last row. Equivalent to b[-1,:] + >>> b[-1] # the last row. Equivalent to b[-1, :] array([40, 41, 42, 43]) The expression within brackets in ``b[i]`` is treated as an ``i`` followed by as many instances of ``:`` as needed to represent the remaining axes. NumPy also allows you to write this using dots as -``b[i,...]``. +``b[i, ...]``. The **dots** (``...``) represent as many colons as needed to produce a complete indexing tuple. For example, if ``x`` is an array with 5 axes, then -- ``x[1,2,...]`` is equivalent to ``x[1,2,:,:,:]``, -- ``x[...,3]`` to ``x[:,:,:,:,3]`` and -- ``x[4,...,5,:]`` to ``x[4,:,:,5,:]``. +- ``x[1, 2, ...]`` is equivalent to ``x[1, 2, :, :, :]``, +- ``x[..., 3]`` to ``x[:, :, :, :, 3]`` and +- ``x[4, ..., 5, :]`` to ``x[4, :, :, 5, :]``. :: - >>> c = np.array( [[[ 0, 1, 2], # a 3D array (two stacked 2D arrays) - ... [ 10, 12, 13]], - ... [[100,101,102], - ... [110,112,113]]]) + >>> c = np.array([[[ 0, 1, 2], # a 3D array (two stacked 2D arrays) + ... [ 10, 12, 13]], + ... [[100, 101, 102], + ... [110, 112, 113]]]) >>> c.shape (2, 2, 3) - >>> c[1,...] # same as c[1,:,:] or c[1] + >>> c[1, ...] # same as c[1, :, :] or c[1] array([[100, 101, 102], [110, 112, 113]]) - >>> c[...,2] # same as c[:,:,2] + >>> c[..., 2] # same as c[:, :, 2] array([[ 2, 13], [102, 113]]) @@ -647,7 +646,7 @@ Changing the shape of an array An array has a shape given by the number of elements along each axis:: - >>> a = np.floor(10*rg.random((3,4))) + >>> a = np.floor(10 * rg.random((3, 4))) >>> a array([[3., 7., 3., 4.], [1., 4., 2., 2.], @@ -661,7 +660,7 @@ the original array:: >>> a.ravel() # returns the array, flattened array([3., 7., 3., 4., 1., 4., 2., 2., 7., 2., 4., 9.]) - >>> a.reshape(6,2) # returns the array with a modified shape + >>> a.reshape(6, 2) # returns the array with a modified shape array([[3., 7.], [3., 4.], [1., 4.], @@ -678,14 +677,14 @@ the original array:: >>> a.shape (3, 4) -The order of the elements in the array resulting from ravel() is +The order of the elements in the array resulting from ``ravel`` is normally "C-style", that is, the rightmost index "changes the fastest", -so the element after a[0,0] is a[0,1]. If the array is reshaped to some +so the element after ``a[0, 0]`` is ``a[0, 1]``. If the array is reshaped to some other shape, again the array is treated as "C-style". NumPy normally -creates arrays stored in this order, so ravel() will usually not need to +creates arrays stored in this order, so ``ravel`` will usually not need to copy its argument, but if the array was made by taking slices of another array or created with unusual options, it may need to be copied. The -functions ravel() and reshape() can also be instructed, using an +functions ``ravel`` and ``reshape`` can also be instructed, using an optional argument, to use FORTRAN-style arrays, in which the leftmost index changes the fastest. @@ -698,15 +697,15 @@ itself:: array([[3., 7., 3., 4.], [1., 4., 2., 2.], [7., 2., 4., 9.]]) - >>> a.resize((2,6)) + >>> a.resize((2, 6)) >>> a array([[3., 7., 3., 4., 1., 4.], [2., 2., 7., 2., 4., 9.]]) -If a dimension is given as -1 in a reshaping operation, the other +If a dimension is given as ``-1`` in a reshaping operation, the other dimensions are automatically calculated:: - >>> a.reshape(3,-1) + >>> a.reshape(3, -1) array([[3., 7., 3., 4.], [1., 4., 2., 2.], [7., 2., 4., 9.]]) @@ -726,45 +725,44 @@ Stacking together different arrays Several arrays can be stacked together along different axes:: - >>> a = np.floor(10*rg.random((2,2))) + >>> a = np.floor(10 * rg.random((2, 2))) >>> a array([[9., 7.], [5., 2.]]) - >>> b = np.floor(10*rg.random((2,2))) + >>> b = np.floor(10 * rg.random((2, 2))) >>> b array([[1., 9.], [5., 1.]]) - >>> np.vstack((a,b)) + >>> np.vstack((a, b)) array([[9., 7.], [5., 2.], [1., 9.], [5., 1.]]) - >>> np.hstack((a,b)) + >>> np.hstack((a, b)) array([[9., 7., 1., 9.], [5., 2., 5., 1.]]) -The function `column_stack` -stacks 1D arrays as columns into a 2D array. It is equivalent to -`hstack` only for 2D arrays:: +The function `column_stack` stacks 1D arrays as columns into a 2D array. +It is equivalent to `hstack` only for 2D arrays:: >>> from numpy import newaxis - >>> np.column_stack((a,b)) # with 2D arrays + >>> np.column_stack((a, b)) # with 2D arrays array([[9., 7., 1., 9.], [5., 2., 5., 1.]]) - >>> a = np.array([4.,2.]) - >>> b = np.array([3.,8.]) - >>> np.column_stack((a,b)) # returns a 2D array + >>> a = np.array([4., 2.]) + >>> b = np.array([3., 8.]) + >>> np.column_stack((a, b)) # returns a 2D array array([[4., 3.], [2., 8.]]) - >>> np.hstack((a,b)) # the result is different + >>> np.hstack((a, b)) # the result is different array([4., 2., 3., 8.]) - >>> a[:,newaxis] # view `a` as a 2D column vector + >>> a[:, newaxis] # view `a` as a 2D column vector array([[4.], [2.]]) - >>> np.column_stack((a[:,newaxis],b[:,newaxis])) + >>> np.column_stack((a[:, newaxis], b[:, newaxis])) array([[4., 3.], [2., 8.]]) - >>> np.hstack((a[:,newaxis],b[:,newaxis])) # the result is the same + >>> np.hstack((a[:, newaxis], b[:, newaxis])) # the result is the same array([[4., 3.], [2., 8.]]) @@ -785,12 +783,10 @@ which the concatenation should happen. **Note** -In complex cases, `r_` and -`c_` are useful for creating arrays -by stacking numbers along one axis. They allow the use of range literals -(":") :: +In complex cases, `r_` and `c_` are useful for creating arrays by stacking +numbers along one axis. They allow the use of range literals ``:``. :: - >>> np.r_[1:4,0,4] + >>> np.r_[1:4, 0, 4] array([1, 2, 3, 0, 4]) When used with arrays as arguments, @@ -818,18 +814,18 @@ array along its horizontal axis, either by specifying the number of equally shaped arrays to return, or by specifying the columns after which the division should occur:: - >>> a = np.floor(10*rg.random((2,12))) + >>> a = np.floor(10 * rg.random((2, 12))) >>> a array([[6., 7., 6., 9., 0., 5., 4., 0., 6., 8., 5., 2.], [8., 5., 5., 7., 1., 8., 6., 7., 1., 8., 1., 0.]]) - # Split a into 3 - >>> np.hsplit(a,3) + >>> # Split `a` into 3 + >>> np.hsplit(a, 3) [array([[6., 7., 6., 9.], [8., 5., 5., 7.]]), array([[0., 5., 4., 0.], [1., 8., 6., 7.]]), array([[6., 8., 5., 2.], [1., 8., 1., 0.]])] - # Split a after the third and the fourth column - >>> np.hsplit(a,(3,4)) + >>> # Split `a` after the third and the fourth column + >>> np.hsplit(a, (3, 4)) [array([[6., 7., 6.], [8., 5., 5.]]), array([[9.], [7.]]), array([[0., 5., 4., 0., 6., 8., 5., 2.], @@ -871,7 +867,7 @@ copy. >>> def f(x): ... print(id(x)) ... - >>> id(a) # id is a unique identifier of an object + >>> id(a) # id is a unique identifier of an object 148293216 # may vary >>> f(a) 148293216 # may vary @@ -887,15 +883,15 @@ creates a new array object that looks at the same data. >>> c = a.view() >>> c is a False - >>> c.base is a # c is a view of the data owned by a + >>> c.base is a # c is a view of the data owned by a True >>> c.flags.owndata False >>> - >>> c = c.reshape((2, 6)) # a's shape doesn't change + >>> c = c.reshape((2, 6)) # a's shape doesn't change >>> a.shape (3, 4) - >>> c[0, 4] = 1234 # a's data changes + >>> c[0, 4] = 1234 # a's data changes >>> a array([[ 0, 1, 2, 3], [1234, 5, 6, 7], @@ -903,8 +899,8 @@ creates a new array object that looks at the same data. Slicing an array returns a view of it:: - >>> s = a[ : , 1:3] # spaces added for clarity; could also be written "s = a[:, 1:3]" - >>> s[:] = 10 # s[:] is a view of s. Note the difference between s = 10 and s[:] = 10 + >>> s = a[:, 1:3] + >>> s[:] = 10 # s[:] is a view of s. Note the difference between s = 10 and s[:] = 10 >>> a array([[ 0, 10, 10, 3], [1234, 10, 10, 7], @@ -917,12 +913,12 @@ The ``copy`` method makes a complete copy of the array and its data. :: - >>> d = a.copy() # a new array object with new data is created + >>> d = a.copy() # a new array object with new data is created >>> d is a False - >>> d.base is a # d doesn't share anything with a + >>> d.base is a # d doesn't share anything with a False - >>> d[0,0] = 9999 + >>> d[0, 0] = 9999 >>> a array([[ 0, 10, 10, 3], [1234, 10, 10, 7], @@ -1068,13 +1064,13 @@ Indexing with Arrays of Indices :: - >>> a = np.arange(12)**2 # the first 12 square numbers - >>> i = np.array([1, 1, 3, 8, 5]) # an array of indices - >>> a[i] # the elements of a at the positions i + >>> a = np.arange(12)**2 # the first 12 square numbers + >>> i = np.array([1, 1, 3, 8, 5]) # an array of indices + >>> a[i] # the elements of `a` at the positions `i` array([ 1, 1, 9, 64, 25]) - >>> - >>> j = np.array([[3, 4], [9, 7]]) # a bidimensional array of indices - >>> a[j] # the same shape as j + >>> + >>> j = np.array([[3, 4], [9, 7]]) # a bidimensional array of indices + >>> a[j] # the same shape as `j` array([[ 9, 16], [81, 49]]) @@ -1090,9 +1086,9 @@ using a palette. ... [0, 255, 0], # green ... [0, 0, 255], # blue ... [255, 255, 255]]) # white - >>> image = np.array([[0, 1, 2, 0], # each value corresponds to a color in the palette + >>> image = np.array([[0, 1, 2, 0], # each value corresponds to a color in the palette ... [0, 3, 4, 0]]) - >>> palette[image] # the (2, 4, 3) color image + >>> palette[image] # the (2, 4, 3) color image array([[[ 0, 0, 0], [255, 0, 0], [ 0, 255, 0], @@ -1108,25 +1104,25 @@ indices for each dimension must have the same shape. :: - >>> a = np.arange(12).reshape(3,4) + >>> a = np.arange(12).reshape(3, 4) >>> a array([[ 0, 1, 2, 3], [ 4, 5, 6, 7], [ 8, 9, 10, 11]]) - >>> i = np.array([[0, 1], # indices for the first dim of a + >>> i = np.array([[0, 1], # indices for the first dim of `a` ... [1, 2]]) - >>> j = np.array([[2, 1], # indices for the second dim + >>> j = np.array([[2, 1], # indices for the second dim ... [3, 3]]) - >>> - >>> a[i, j] # i and j must have equal shape + >>> + >>> a[i, j] # i and j must have equal shape array([[ 2, 5], [ 7, 11]]) - >>> + >>> >>> a[i, 2] array([[ 2, 6], [ 6, 10]]) - >>> - >>> a[:, j] # i.e., a[ : , j] + >>> + >>> a[:, j] array([[[ 2, 1], [ 3, 3]], @@ -1142,26 +1138,24 @@ put ``i`` and ``j`` in a ``tuple`` and then do the indexing with that. :: >>> l = (i, j) - # equivalent to a[i, j] + >>> # equivalent to a[i, j] >>> a[l] array([[ 2, 5], [ 7, 11]]) However, we can not do this by putting ``i`` and ``j`` into an array, because this array will be interpreted as indexing the first dimension -of a. +of ``a``. :: >>> s = np.array([i, j]) - - # not what we want + >>> # not what we want >>> a[s] Traceback (most recent call last): File "", line 1, in IndexError: index 3 is out of bounds for axis 0 with size 3 - - # same as a[i, j] + >>> # same as `a[i, j]` >>> a[tuple(s)] array([[ 2, 5], [ 7, 11]]) @@ -1169,8 +1163,8 @@ of a. Another common use of indexing with arrays is the search of the maximum value of time-dependent series:: - >>> time = np.linspace(20, 145, 5) # time scale - >>> data = np.sin(np.arange(20)).reshape(5,4) # 4 time-dependent series + >>> time = np.linspace(20, 145, 5) # time scale + >>> data = np.sin(np.arange(20)).reshape(5, 4) # 4 time-dependent series >>> time array([ 20. , 51.25, 82.5 , 113.75, 145. ]) >>> data @@ -1179,22 +1173,18 @@ value of time-dependent series:: [ 0.98935825, 0.41211849, -0.54402111, -0.99999021], [-0.53657292, 0.42016704, 0.99060736, 0.65028784], [-0.28790332, -0.96139749, -0.75098725, 0.14987721]]) - - # index of the maxima for each series + >>> # index of the maxima for each series >>> ind = data.argmax(axis=0) >>> ind array([2, 0, 3, 1]) - - # times corresponding to the maxima + >>> # times corresponding to the maxima >>> time_max = time[ind] - >>> - >>> data_max = data[ind, range(data.shape[1])] # => data[ind[0],0], data[ind[1],1]... - + >>> + >>> data_max = data[ind, range(data.shape[1])] # => data[ind[0], 0], data[ind[1], 1]... >>> time_max array([ 82.5 , 20. , 113.75, 51.25]) >>> data_max array([0.98935825, 0.84147098, 0.99060736, 0.6569866 ]) - >>> np.all(data_max == data.max(axis=0)) True @@ -1203,7 +1193,7 @@ You can also use indexing with arrays as a target to assign to:: >>> a = np.arange(5) >>> a array([0, 1, 2, 3, 4]) - >>> a[[1,3,4]] = 0 + >>> a[[1, 3, 4]] = 0 >>> a array([0, 0, 2, 0, 0]) @@ -1211,7 +1201,7 @@ However, when the list of indices contains repetitions, the assignment is done several times, leaving behind the last value:: >>> a = np.arange(5) - >>> a[[0,0,2]]=[1,2,3] + >>> a[[0, 0, 2]] = [1, 2, 3] >>> a array([2, 1, 3, 3, 4]) @@ -1219,13 +1209,13 @@ This is reasonable enough, but watch out if you want to use Python's ``+=`` construct, as it may not do what you expect:: >>> a = np.arange(5) - >>> a[[0,0,2]]+=1 + >>> a[[0, 0, 2]] += 1 >>> a array([1, 1, 3, 3, 4]) Even though 0 occurs twice in the list of indices, the 0th element is -only incremented once. This is because Python requires "a+=1" to be -equivalent to "a = a + 1". +only incremented once. This is because Python requires ``a += 1`` to be +equivalent to ``a = a + 1``. Indexing with Boolean Arrays ---------------------------- @@ -1238,18 +1228,18 @@ which ones we don't. The most natural way one can think of for boolean indexing is to use boolean arrays that have *the same shape* as the original array:: - >>> a = np.arange(12).reshape(3,4) + >>> a = np.arange(12).reshape(3, 4) >>> b = a > 4 - >>> b # b is a boolean with a's shape + >>> b # `b` is a boolean with `a`'s shape array([[False, False, False, False], [False, True, True, True], [ True, True, True, True]]) - >>> a[b] # 1d array with the selected elements + >>> a[b] # 1d array with the selected elements array([ 5, 6, 7, 8, 9, 10, 11]) This property can be very useful in assignments:: - >>> a[b] = 0 # All elements of 'a' higher than 4 become 0 + >>> a[b] = 0 # All elements of `a` higher than 4 become 0 >>> a array([[0, 1, 2, 3], [4, 0, 0, 0], @@ -1264,45 +1254,47 @@ set `__: >>> import numpy as np >>> import matplotlib.pyplot as plt - >>> def mandelbrot( h,w, maxit=20 ): + >>> def mandelbrot(h, w, maxit=20, r=2): ... """Returns an image of the Mandelbrot fractal of size (h,w).""" - ... y,x = np.ogrid[ -1.4:1.4:h*1j, -2:0.8:w*1j ] - ... c = x+y*1j - ... z = c + ... x = np.linspace(-2.5, 1.5, 4*h+1) + ... y = np.linspace(-1.5, 1.5, 3*w+1) + ... A, B = np.meshgrid(x, y) + ... C = A + B*1j + ... z = np.zeros_like(C) ... divtime = maxit + np.zeros(z.shape, dtype=int) ... ... for i in range(maxit): - ... z = z**2 + c - ... diverge = z*np.conj(z) > 2**2 # who is diverging - ... div_now = diverge & (divtime==maxit) # who is diverging now - ... divtime[div_now] = i # note when - ... z[diverge] = 2 # avoid diverging too much + ... z = z**2 + C + ... diverge = abs(z) > r # who is diverging + ... div_now = diverge & (divtime == maxit) # who is diverging now + ... divtime[div_now] = i # note when + ... z[diverge] = r # avoid diverging too much ... ... return divtime - >>> plt.imshow(mandelbrot(400,400)) + >>> plt.imshow(mandelbrot(400, 400)) The second way of indexing with booleans is more similar to integer indexing; for each dimension of the array we give a 1D boolean array selecting the slices we want:: - >>> a = np.arange(12).reshape(3,4) - >>> b1 = np.array([False,True,True]) # first dim selection - >>> b2 = np.array([True,False,True,False]) # second dim selection - >>> - >>> a[b1,:] # selecting rows + >>> a = np.arange(12).reshape(3, 4) + >>> b1 = np.array([False, True, True]) # first dim selection + >>> b2 = np.array([True, False, True, False]) # second dim selection + >>> + >>> a[b1, :] # selecting rows array([[ 4, 5, 6, 7], [ 8, 9, 10, 11]]) - >>> - >>> a[b1] # same thing + >>> + >>> a[b1] # same thing array([[ 4, 5, 6, 7], [ 8, 9, 10, 11]]) - >>> - >>> a[:,b2] # selecting columns + >>> + >>> a[:, b2] # selecting columns array([[ 0, 2], [ 4, 6], [ 8, 10]]) - >>> - >>> a[b1,b2] # a weird thing to do + >>> + >>> a[b1, b2] # a weird thing to do array([ 4, 10]) Note that the length of the 1D boolean array must coincide with the @@ -1319,10 +1311,10 @@ obtain the result for each n-uplet. For example, if you want to compute all the a+b\*c for all the triplets taken from each of the vectors a, b and c:: - >>> a = np.array([2,3,4,5]) - >>> b = np.array([8,5,4]) - >>> c = np.array([5,4,6,8,3]) - >>> ax,bx,cx = np.ix_(a,b,c) + >>> a = np.array([2, 3, 4, 5]) + >>> b = np.array([8, 5, 4]) + >>> c = np.array([5, 4, 6, 8, 3]) + >>> ax, bx, cx = np.ix_(a, b, c) >>> ax array([[[2]], @@ -1339,7 +1331,7 @@ and c:: array([[[5, 4, 6, 8, 3]]]) >>> ax.shape, bx.shape, cx.shape ((4, 1, 1), (1, 3, 1), (1, 1, 5)) - >>> result = ax+bx*cx + >>> result = ax + bx * cx >>> result array([[[42, 34, 50, 66, 26], [27, 22, 32, 42, 17], @@ -1356,9 +1348,9 @@ and c:: [[45, 37, 53, 69, 29], [30, 25, 35, 45, 20], [25, 21, 29, 37, 17]]]) - >>> result[3,2,4] + >>> result[3, 2, 4] 17 - >>> a[3]+b[2]*c[4] + >>> a[3] + b[2] * c[4] 17 You could also implement the reduce as follows:: @@ -1367,12 +1359,12 @@ You could also implement the reduce as follows:: ... vs = np.ix_(*vectors) ... r = ufct.identity ... for v in vs: - ... r = ufct(r,v) + ... r = ufct(r, v) ... return r and then use it as:: - >>> ufunc_reduce(np.add,a,b,c) + >>> ufunc_reduce(np.add, a, b, c) array([[[15, 14, 16, 18, 13], [12, 11, 13, 15, 10], [11, 10, 12, 14, 9]], @@ -1400,64 +1392,6 @@ Indexing with strings See :ref:`structured_arrays`. -Linear Algebra -============== - -Work in progress. Basic linear algebra to be included here. - -Simple Array Operations ------------------------ - -See linalg.py in numpy folder for more. - -:: - - >>> import numpy as np - >>> a = np.array([[1.0, 2.0], [3.0, 4.0]]) - >>> print(a) - [[1. 2.] - [3. 4.]] - - >>> a.transpose() - array([[1., 3.], - [2., 4.]]) - - >>> np.linalg.inv(a) - array([[-2. , 1. ], - [ 1.5, -0.5]]) - - >>> u = np.eye(2) # unit 2x2 matrix; "eye" represents "I" - >>> u - array([[1., 0.], - [0., 1.]]) - >>> j = np.array([[0.0, -1.0], [1.0, 0.0]]) - - >>> j @ j # matrix product - array([[-1., 0.], - [ 0., -1.]]) - - >>> np.trace(u) # trace - 2.0 - - >>> y = np.array([[5.], [7.]]) - >>> np.linalg.solve(a, y) - array([[-3.], - [ 4.]]) - - >>> np.linalg.eig(j) - (array([0.+1.j, 0.-1.j]), array([[0.70710678+0.j , 0.70710678-0.j ], - [0. -0.70710678j, 0. +0.70710678j]])) - -:: - - Parameters: - square matrix - Returns - The eigenvalues, each repeated according to its multiplicity. - The normalized (unit "length") eigenvectors, such that the - column ``v[:,i]`` is the eigenvector corresponding to the - eigenvalue ``w[i]`` . - Tricks and Tips =============== @@ -1496,13 +1430,13 @@ functions ``column_stack``, ``dstack``, ``hstack`` and ``vstack``, depending on the dimension in which the stacking is to be done. For example:: - >>> x = np.arange(0,10,2) + >>> x = np.arange(0, 10, 2) >>> y = np.arange(5) - >>> m = np.vstack([x,y]) + >>> m = np.vstack([x, y]) >>> m array([[0, 2, 4, 6, 8], [0, 1, 2, 3, 4]]) - >>> xy = np.hstack([x,y]) + >>> xy = np.hstack([x, y]) >>> xy array([0, 2, 4, 6, 8, 0, 1, 2, 3, 4]) @@ -1530,12 +1464,14 @@ that ``pylab.hist`` plots the histogram automatically, while >>> import matplotlib.pyplot as plt >>> # Build a vector of 10000 normal deviates with variance 0.5^2 and mean 2 >>> mu, sigma = 2, 0.5 - >>> v = rg.normal(mu,sigma,10000) + >>> v = rg.normal(mu, sigma, 10000) >>> # Plot a normalized histogram with 50 bins - >>> plt.hist(v, bins=50, density=1) # matplotlib version (plot) + >>> plt.hist(v, bins=50, density=True) # matplotlib version (plot) >>> # Compute the histogram with numpy and then plot it >>> (n, bins) = np.histogram(v, bins=50, density=True) # NumPy version (no plot) - >>> plt.plot(.5*(bins[1:]+bins[:-1]), n) + >>> plt.plot(.5 * (bins[1:] + bins[:-1]), n) + +With Matplotlib >=3.4 you can also use ``plt.stairs(n, bins)``. Further reading @@ -1546,3 +1482,4 @@ Further reading - `SciPy Tutorial `__ - `SciPy Lecture Notes `__ - A `matlab, R, IDL, NumPy/SciPy dictionary `__ +- :doc:`tutorial-svd` diff --git a/doc/source/user/theory.broadcasting.rst b/doc/source/user/theory.broadcasting.rst index b37edeacc121..a82d78e6c9a8 100644 --- a/doc/source/user/theory.broadcasting.rst +++ b/doc/source/user/theory.broadcasting.rst @@ -69,7 +69,7 @@ numpy on Windows 2000 with one million element arrays. *Figure 1* *In the simplest example of broadcasting, the scalar ``b`` is - stretched to become an array of with the same shape as ``a`` so the shapes + stretched to become an array of same shape as ``a`` so the shapes are compatible for element-by-element multiplication.* diff --git a/doc/source/user/tutorial-ma.rst b/doc/source/user/tutorial-ma.rst index 88bad3cbeed9..a21c4aae12f4 100644 --- a/doc/source/user/tutorial-ma.rst +++ b/doc/source/user/tutorial-ma.rst @@ -366,12 +366,13 @@ after the beginning of the records: .. ipython:: python - plt.plot(t, china_total); - plt.plot(t[china_total.mask], cubic_fit[china_total.mask], '--', color='orange'); - plt.plot(7, np.polyval(params, 7), 'r*'); + plt.plot(t, china_total, label='Mainland China'); + plt.plot(t[china_total.mask], cubic_fit[china_total.mask], '--', + color='orange', label='Cubic estimate'); + plt.plot(7, np.polyval(params, 7), 'r*', label='7 days after start'); plt.xticks([0, 7, 13], dates[[0, 7, 13]]); plt.yticks([0, np.polyval(params, 7), 10000, 17500]); - plt.legend(['Mainland China', 'Cubic estimate', '7 days after start']); + plt.legend(); @savefig plot_covid_5.png plt.title("COVID-19 cumulative cases from Jan 21 to Feb 3 2020 - Mainland China\n" "Cubic estimate for 7 days after start"); diff --git a/doc/source/user/tutorial-svd.rst b/doc/source/user/tutorial-svd.rst index fd9e366e0c77..7b905e51eede 100644 --- a/doc/source/user/tutorial-svd.rst +++ b/doc/source/user/tutorial-svd.rst @@ -431,7 +431,7 @@ Now, to build our approximation, we first need to make sure that our singular values are ready for multiplication, so we build our ``Sigma`` matrix similarly to what we did before. The ``Sigma`` array must have dimensions ``(3, 768, 1024)``. In order to add the singular values to the diagonal of -``Sigma``, we will use the ``fill_diagonal`` function from NumPy, using each of +``Sigma``, we will use the `numpy.fill_diagonal` function from NumPy, using each of the 3 rows in ``s`` as the diagonal for each of the 3 matrices in ``Sigma``: :: diff --git a/doc/source/user/who_covid_19_sit_rep_time_series.csv b/doc/source/user/who_covid_19_sit_rep_time_series.csv index ebf670b8c0b0..8ad5c2c238cf 100644 --- a/doc/source/user/who_covid_19_sit_rep_time_series.csv +++ b/doc/source/user/who_covid_19_sit_rep_time_series.csv @@ -1,115 +1,115 @@ -Province/States,Country/Region,WHO region,1/21/20,1/22/20,1/23/20,1/24/20,1/25/20,1/26/20,1/27/20,1/28/20,1/29/20,1/30/20,1/31/20,2/1/20,2/2/20,2/3/20,2/4/20,2/5/20,2/6/20,2/7/20,2/8/20,2/9/20,2/10/20,2/11/20,2/12/20,2/13/20,2/14/20,2/15/20,2/16/20,2/17/20,2/18/20,2/19/20,2/20/20,2/21/20,2/22/20,2/23/20,2/24/20,2/25/20,2/26/20,2/27/20,2/28/20,2/29/20,3/1/20,3/2/20,3/3/20 -Confirmed,Globally,,282,314,581,846,1320,2014,2798,4593,6065,7818,9826,11953,14557,17391,20630,24554,28276,31481,34886,37558,40554,43103,45171,46997,49053,50580,51857,71429,73332,75204,75748,76769,77794,78811,79331,80239,81109,82294,83652,85403,87137,88948,90870 -Confirmed,Mainland China,Western Pacific Region,278,309,571,830,1297,1985,2741,4537,5997,7736,9720,11821,14411,17238,20471,24363,28060,31211,34598,37251,40235,42708,44730,46550,48548,50054,51174,70635,72528,74280,74675,75569,76392,77042,77262,77780,78191,78630,78961,79394,79968,80174,80304 -Confirmed,Outside of China,,4,5,10,16,23,29,57,56,68,82,106,132,146,153,159,191,216,270,288,307,319,395,441,447,505,526,683,794,804,924,1073,1200,1402,1769,2069,2459,2918,3664,4691,6009,7169,8774,10566 -Suspected,Mainland China,Western Pacific Region,,,,,,,5794,6973,9239,12167,15238,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -Severe,Mainland China,Western Pacific Region,,,,,,,461,976,1239,1370,1527,1795,2110,2296,2788,3219,3859,4821,6101,6188,6484,7333,8204,,,,,,,,,,,,,,,,,,,, -Deaths,Mainland China,Western Pacific Region,,,,,,,80,106,132,170,213,259,304,361,425,491,564,637,723,812,909,1017,1114,1260,1381,1524,1666,1772,1870,2006,2121,2239,2348,2445,2595,2666,2718,2747,2791,2838,2873,2915,2946 -Hubei ,China,Western Pacific Region,258,270,375,375,,,,,,,,7153,9074,11177,13522,16678,19665,22112,24953,27100,29631,31728,33366,34874,51968,54406,56249,58182,59989,61682,62031,62662,63454,64084,64287,64786,65187,65596,65914,66337,66907,67103,67217 -Guangdong,China,Western Pacific Region,14,17,26,32,,,,,,,,520,604,683,797,870,944,1018,1075,1120,1151,1177,1219,1241,1261,1295,1316,1322,1328,1331,1332,1333,1339,1342,1345,1347,1347,1347,1348,1349,1349,1350,1350 -Henan,China,Western Pacific Region,,1,1,1,,,,,,,,422,493,566,675,764,851,914,981,1033,1073,1105,1135,1169,1184,1212,1231,1246,1257,1262,1265,1267,1270,1271,1271,1271,1271,1272,1272,1272,1272,1272,1272 -Zhejiang,China,Western Pacific Region,,5,5,5,,,,,,,,599,661,724,829,895,954,1006,1048,1075,1104,1117,1131,1145,1155,1162,1167,1171,1172,1173,1175,1203,1205,1205,1205,1205,1205,1205,1205,1205,1205,1206,1213 -Hunan,China,Western Pacific Region,,1,1,1,,,,,,,,389,463,521,593,661,711,772,803,838,879,912,946,968,988,1001,1004,1006,1007,1008,1010,1011,1013,1016,1016,1016,1016,1017,1017,1018,1018,1018,1018 -Anhui,China,Western Pacific Region,,,,,,,,,,,,297,340,408,480,530,591,665,733,779,830,860,889,910,934,950,962,973,982,986,987,988,989,989,989,989,989,989,990,990,990,990,990 -Jiangxi,China,Western Pacific Region,,1,2,2,,,,,,,,286,333,391,476,548,600,661,698,740,771,804,844,872,900,913,925,930,933,934,934,934,934,934,934,934,934,934,935,935,935,935,935 -Shandong,China,Western Pacific Region,,1,1,1,,,,,,,,202,225,246,270,298,343,379,407,435,459,486,497,506,519,530,537,541,543,544,546,748,750,754,755,755,756,756,756,756,756,758,758 -Jiangsu,China,Western Pacific Region,,,,,,,,,,,,202,231,271,308,341,373,408,439,468,492,515,543,570,593,604,617,626,629,631,631,631,631,631,631,631,631,631,631,631,631,631,631 -Chongqing,China,Western Pacific Region,,1,5,5,,,,,,,,238,262,300,337,366,389,411,426,446,468,486,505,518,529,537,544,551,553,555,560,567,572,573,575,576,576,576,576,576,576,576,576 -Sichuan,China,Western Pacific Region,,1,2,2,,,,,,,,207,236,254,282,301,321,344,363,386,405,417,436,451,463,470,481,495,508,514,520,525,526,526,527,529,531,534,538,538,538,538,538 -Heilongjiang,China,Western Pacific Region,,,,,,,,,,,,80,95,118,155,190,227,277,282,307,331,360,378,395,418,425,445,457,464,470,476,479,479,480,480,480,480,480,480,480,480,480,480 -Beijing,China,Western Pacific Region,5,5,10,10,,,,,,,,156,183,212,228,253,274,297,315,326,337,342,352,366,372,375,380,381,387,393,395,396,399,399,399,400,400,410,410,411,413,414,414 -Shanghai,China,Western Pacific Region,1,2,9,9,,,,,,,,153,177,193,208,233,254,269,281,292,295,302,306,313,318,326,328,331,333,333,333,334,334,335,335,335,336,337,337,337,337,337,338 -Hebei,China,Western Pacific Region,,,,,,,,,,,,96,104,113,126,135,157,171,195,206,218,239,251,265,283,291,300,301,302,306,307,308,309,311,311,311,312,317,318,318,318,318,318 -Fujian,China,Western Pacific Region,,,,,,,,,,,,144,159,179,194,205,215,224,239,250,261,267,272,279,281,285,287,290,292,293,293,293,293,293,293,294,294,296,296,296,296,296,296 -Guangxi,China,Western Pacific Region,,,,,,,,,,,,100,111,127,139,150,168,172,183,195,210,215,222,222,226,235,237,238,242,244,245,246,249,249,251,252,252,252,252,252,252,252,252 -Shaanxi,China,Western Pacific Region,,,,,,,,,,,,101,116,128,142,165,173,184,195,208,213,219,225,229,230,232,236,240,240,242,245,245,245,245,245,245,245,245,245,245,245,245,245 -Yunnan,China,Western Pacific Region,,1,1,1,,,,,,,,91,99,109,117,122,128,135,138,140,141,149,154,155,162,168,169,171,172,172,172,174,174,174,174,174,174,174,174,174,174,174,174 -Hainan,China,Western Pacific Region,,,,,,,,,,,,57,63,70,79,89,100,111,123,128,136,142,145,157,157,162,162,162,163,163,168,168,168,168,168,168,168,168,168,168,168,168,168 -Guizhou,China,Western Pacific Region,,,,,,,,,,,,29,38,46,56,64,69,77,89,96,109,118,131,135,140,143,144,146,146,146,146,146,146,146,146,146,146,146,146,146,146,146,146 -Tianjin,China,Western Pacific Region,,2,2,2,,,,,,,,34,40,49,63,67,70,94,81,88,91,96,106,112,119,120,122,124,125,128,130,131,133,135,135,135,135,135,136,136,136,136,136 -Shanxi,China,Western Pacific Region,,,,,,,,,,,,47,56,66,74,81,90,96,104,115,119,122,124,126,126,127,128,129,130,131,131,132,132,132,132,133,133,133,133,133,133,133,133 -Liaoning,China,Western Pacific Region,,,,,,,,,,,,60,64,70,74,81,89,94,99,105,107,108,111,116,117,119,120,121,121,121,121,121,121,121,121,121,121,121,121,121,122,122,125 -Hong Kong,China,Western Pacific Region,,,1,2,5,5,8,8,8,10,12,13,14,15,15,18,21,24,26,26,36,42,49,50,53,56,56,57,60,62,65,68,68,70,74,81,85,91,93,94,95,98,101 -Jilin,China,Western Pacific Region,,,,,,,,,,,,17,21,31,42,54,59,65,69,78,80,81,83,84,86,88,88,89,89,90,91,91,91,91,93,93,93,93,93,93,93,93,93 -Gansu,China,Western Pacific Region,,,,,,,,,,,,35,45,51,56,57,62,70,71,81,85,86,86,87,90,90,90,91,91,91,91,91,91,91,91,91,91,91,91,91,91,91,91 -Xinjiang,China,Western Pacific Region,,,,,,,,,,,,18,23,24,29,32,36,39,42,45,49,55,59,63,65,70,71,73,76,76,76,76,76,75,76,76,76,76,76,76,76,76,76 -Inner Mongolia,China,Western Pacific Region,,,,,,,,,,,,23,26,33,37,42,46,49,50,54,58,58,60,61,63,68,70,72,73,75,75,75,75,75,75,75,75,75,75,75,75,75,75 -Ningxia,China,Western Pacific Region,,,,,,,,,,,,26,28,31,34,34,40,43,45,45,49,53,58,64,67,70,70,70,70,71,71,71,71,71,71,71,71,72,72,73,73,74,74 -Taiwan,China,Western Pacific Region,,1,1,1,3,3,4,7,8,8,9,10,10,10,10,11,11,16,16,17,18,18,18,18,18,18,18,20,22,23,24,26,26,23,28,31,32,32,34,39,39,40,42 -Qinghai,China,Western Pacific Region,,,,,,,,,,,,8,9,13,15,17,18,18,18,18,18,18,18,18,18,18,18,18,18,12,18,18,18,18,18,18,18,18,18,18,18,18,18 -Macau,China,Western Pacific Region,,,1,2,2,2,5,7,7,7,7,7,7,8,8,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10 -Xizang,China,Western Pacific Region,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 -Unspecified*,China,Western Pacific Region,,,131,384,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -,Japan,Western Pacific Region,1,1,1,1,3,3,4,6,7,11,14,17,20,20,20,33,25,25,25,26,26,26,28,29,33,41,53,59,65,73,85,93,105,132,144,157,164,186,210,230,239,254,268 -,Republic of Korea,Western Pacific Region,1,1,1,2,2,2,4,4,4,4,11,12,15,15,16,18,23,24,24,27,27,28,28,28,28,28,29,30,31,51,104,204,346,602,763,977,1261,1766,2337,3150,3736,4212,4812 -,Thailand,South-East Asia Region,2,2,2,4,4,5,5,14,14,14,14,19,19,19,19,25,25,25,32,32,32,33,33,33,33,34,34,35,35,35,35,35,35,35,35,37,40,40,40,42,42,42,43 -,United States of America,Region of the Americas,,,1,1,2,2,5,5,5,5,6,7,8,11,11,11,12,12,12,12,12,13,13,14,15,15,15,15,15,15,15,15,35,35,35,53,53,59,59,62,62,62,64 -,Vietnam,Western Pacific Region,,,,2,2,2,2,2,2,2,5,6,7,8,9,10,10,12,13,14,14,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16 -,Singapore,Western Pacific Region,,,,1,3,3,4,7,7,10,13,16,18,18,18,24,28,30,33,40,43,45,47,50,58,67,72,75,77,81,84,85,86,89,89,90,91,93,96,98,102,106,108 -,Italy,European Region,,,,,,,,,,,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,9,76,124,229,322,400,650,888,1128,1689,2036 -,Nepal,South-East Asia Region,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 -,Australia,Western Pacific Region,,,,,3,3,4,5,7,7,9,12,12,12,12,13,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,21,21,21,22,23,23,23,24,25,27,33 -,Malaysia,Western Pacific Region,,,,,,3,4,4,4,7,8,8,8,8,10,10,12,14,15,17,18,18,18,18,19,21,22,22,22,22,22,22,22,22,22,22,22,22,24,24,24,24,29 -,Canada,Region of the Americas,,,,,,,1,2,3,3,3,4,4,4,4,5,5,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,9,9,10,10,11,11,14,19,19,27 -,Cambodia,Western Pacific Region,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 -,France,European Region,,,,,3,3,3,3,4,5,6,6,6,6,6,6,6,6,6,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,18,38,57,100,100,191 -,Sri Lanka,South-East Asia Region,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 -,Iran,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,5,18,28,43,61,95,141,245,388,593,978,1501 -,India,South-East Asia Region,,,,,,,,,,1,1,1,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5 -,Germany,European Region,,,,,,,,1,4,4,5,7,8,10,12,12,12,13,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,18,21,26,57,57,129,157 -,Philippines,Western Pacific Region,,,,,,,,,,1,1,1,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 -,Spain,European Region,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,12,25,32,45,45,114 -,United Kingdom,European Region,,,,,,,,,,,,2,2,2,2,2,2,3,3,3,4,8,8,9,9,9,9,9,9,9,9,9,9,9,9,13,13,13,16,20,23,36,39 -,Sweden,European Region,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,7,12,13,14,15 -,Switzerland,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,6,10,18,26,30 -,Austria,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,4,5,10,10,18 -,Norway,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,6,15,19,25 -,Kuwait,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,8,12,43,43,45,45,56,56 -,Bahrain,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,26,33,33,38,40,47,49 -,United Arab Emirates,Eastern Mediterranean Region,,,,,,,,,4,4,4,4,5,5,5,5,5,5,7,7,7,8,8,8,8,8,8,9,9,9,9,9,11,13,13,13,13,13,19,19,19,21,21 -,Israel,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,2,2,2,3,5,7,7,10 -,Iraq,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,5,6,7,8,13,19,26 -,Oman,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,4,4,6,6,6,6,6 -,Lebanon,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,2,2,2,2,10,13 -,Pakistan,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,2,4,4,5 -,Egypt,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2 -,Croatia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,3,3,5,7,7,9 -,Greece,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,3,3,3,7,7 -,Finland,European Region,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,6,7 -,Algeria,African Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1,5 -,Brazil,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,2,2,2 -,Russian,European Region,,,,,,,,,,,,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3 -,Belgium,European Region,,,,,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,8 -,Denmark,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,2,3,4,5 -,Estonia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1 -,Georgia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,2,3,3,3 -,North Macedonia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1 -,Romania,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,3,3,3,3 -,Afghanistan,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1,1,1 -,New Zealand,Western Pacific Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,2 -,Belarus,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1 -,Lithuania,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1 -,Netherlands,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,2,7,13,18 -,Nigeria,African Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1 -,Mexico,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,5,5 -,San Marino,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,8 -,Azerbaijan,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,3 -,Ireland,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1 -,Monaco,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1 -,Qatar,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,3,7 -,Ecuador,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,6 -,Czechia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3 -,Iceland,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,9 -,Armenia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1 -,Luxembourg,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1 -,Indonesia,South-East Asia Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2 -,Dominican Republic,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1 -,Portugal,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2 -,Andorra,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -,Latvia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -,Jordan,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -,Morocco,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -,Saudi Arabia,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -,Tunisia,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 -,Senegal,African Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +Province/States,Country/Region,WHO region,1/21/20,1/22/20,1/23/20,1/24/20,1/25/20,1/26/20,1/27/20,1/28/20,1/29/20,1/30/20,1/31/20,2/1/20,2/2/20,2/3/20,2/4/20,2/5/20,2/6/20,2/7/20,2/8/20,2/9/20,2/10/20,2/11/20,2/12/20,2/13/20,2/14/20,2/15/20,2/16/20,2/17/20,2/18/20,2/19/20,2/20/20,2/21/20,2/22/20,2/23/20,2/24/20,2/25/20,2/26/20,2/27/20,2/28/20,2/29/20,3/1/20,3/2/20,3/3/20 +Confirmed,Globally,,282,314,581,846,1320,2014,2798,4593,6065,7818,9826,11953,14557,17391,20630,24554,28276,31481,34886,37558,40554,43103,45171,46997,49053,50580,51857,71429,73332,75204,75748,76769,77794,78811,79331,80239,81109,82294,83652,85403,87137,88948,90870 +Confirmed,Mainland China,Western Pacific Region,278,309,571,830,1297,1985,2741,4537,5997,7736,9720,11821,14411,17238,20471,24363,28060,31211,34598,37251,40235,42708,44730,46550,48548,50054,51174,70635,72528,74280,74675,75569,76392,77042,77262,77780,78191,78630,78961,79394,79968,80174,80304 +Confirmed,Outside of China,,4,5,10,16,23,29,57,56,68,82,106,132,146,153,159,191,216,270,288,307,319,395,441,447,505,526,683,794,804,924,1073,1200,1402,1769,2069,2459,2918,3664,4691,6009,7169,8774,10566 +Suspected,Mainland China,Western Pacific Region,,,,,,,5794,6973,9239,12167,15238,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +Severe,Mainland China,Western Pacific Region,,,,,,,461,976,1239,1370,1527,1795,2110,2296,2788,3219,3859,4821,6101,6188,6484,7333,8204,,,,,,,,,,,,,,,,,,,, +Deaths,Mainland China,Western Pacific Region,,,,,,,80,106,132,170,213,259,304,361,425,491,564,637,723,812,909,1017,1114,1260,1381,1524,1666,1772,1870,2006,2121,2239,2348,2445,2595,2666,2718,2747,2791,2838,2873,2915,2946 +Hubei ,China,Western Pacific Region,258,270,375,375,,,,,,,,7153,9074,11177,13522,16678,19665,22112,24953,27100,29631,31728,33366,34874,51968,54406,56249,58182,59989,61682,62031,62662,63454,64084,64287,64786,65187,65596,65914,66337,66907,67103,67217 +Guangdong,China,Western Pacific Region,14,17,26,32,,,,,,,,520,604,683,797,870,944,1018,1075,1120,1151,1177,1219,1241,1261,1295,1316,1322,1328,1331,1332,1333,1339,1342,1345,1347,1347,1347,1348,1349,1349,1350,1350 +Henan,China,Western Pacific Region,,1,1,1,,,,,,,,422,493,566,675,764,851,914,981,1033,1073,1105,1135,1169,1184,1212,1231,1246,1257,1262,1265,1267,1270,1271,1271,1271,1271,1272,1272,1272,1272,1272,1272 +Zhejiang,China,Western Pacific Region,,5,5,5,,,,,,,,599,661,724,829,895,954,1006,1048,1075,1104,1117,1131,1145,1155,1162,1167,1171,1172,1173,1175,1203,1205,1205,1205,1205,1205,1205,1205,1205,1205,1206,1213 +Hunan,China,Western Pacific Region,,1,1,1,,,,,,,,389,463,521,593,661,711,772,803,838,879,912,946,968,988,1001,1004,1006,1007,1008,1010,1011,1013,1016,1016,1016,1016,1017,1017,1018,1018,1018,1018 +Anhui,China,Western Pacific Region,,,,,,,,,,,,297,340,408,480,530,591,665,733,779,830,860,889,910,934,950,962,973,982,986,987,988,989,989,989,989,989,989,990,990,990,990,990 +Jiangxi,China,Western Pacific Region,,1,2,2,,,,,,,,286,333,391,476,548,600,661,698,740,771,804,844,872,900,913,925,930,933,934,934,934,934,934,934,934,934,934,935,935,935,935,935 +Shandong,China,Western Pacific Region,,1,1,1,,,,,,,,202,225,246,270,298,343,379,407,435,459,486,497,506,519,530,537,541,543,544,546,748,750,754,755,755,756,756,756,756,756,758,758 +Jiangsu,China,Western Pacific Region,,,,,,,,,,,,202,231,271,308,341,373,408,439,468,492,515,543,570,593,604,617,626,629,631,631,631,631,631,631,631,631,631,631,631,631,631,631 +Chongqing,China,Western Pacific Region,,1,5,5,,,,,,,,238,262,300,337,366,389,411,426,446,468,486,505,518,529,537,544,551,553,555,560,567,572,573,575,576,576,576,576,576,576,576,576 +Sichuan,China,Western Pacific Region,,1,2,2,,,,,,,,207,236,254,282,301,321,344,363,386,405,417,436,451,463,470,481,495,508,514,520,525,526,526,527,529,531,534,538,538,538,538,538 +Heilongjiang,China,Western Pacific Region,,,,,,,,,,,,80,95,118,155,190,227,277,282,307,331,360,378,395,418,425,445,457,464,470,476,479,479,480,480,480,480,480,480,480,480,480,480 +Beijing,China,Western Pacific Region,5,5,10,10,,,,,,,,156,183,212,228,253,274,297,315,326,337,342,352,366,372,375,380,381,387,393,395,396,399,399,399,400,400,410,410,411,413,414,414 +Shanghai,China,Western Pacific Region,1,2,9,9,,,,,,,,153,177,193,208,233,254,269,281,292,295,302,306,313,318,326,328,331,333,333,333,334,334,335,335,335,336,337,337,337,337,337,338 +Hebei,China,Western Pacific Region,,,,,,,,,,,,96,104,113,126,135,157,171,195,206,218,239,251,265,283,291,300,301,302,306,307,308,309,311,311,311,312,317,318,318,318,318,318 +Fujian,China,Western Pacific Region,,,,,,,,,,,,144,159,179,194,205,215,224,239,250,261,267,272,279,281,285,287,290,292,293,293,293,293,293,293,294,294,296,296,296,296,296,296 +Guangxi,China,Western Pacific Region,,,,,,,,,,,,100,111,127,139,150,168,172,183,195,210,215,222,222,226,235,237,238,242,244,245,246,249,249,251,252,252,252,252,252,252,252,252 +Shaanxi,China,Western Pacific Region,,,,,,,,,,,,101,116,128,142,165,173,184,195,208,213,219,225,229,230,232,236,240,240,242,245,245,245,245,245,245,245,245,245,245,245,245,245 +Yunnan,China,Western Pacific Region,,1,1,1,,,,,,,,91,99,109,117,122,128,135,138,140,141,149,154,155,162,168,169,171,172,172,172,174,174,174,174,174,174,174,174,174,174,174,174 +Hainan,China,Western Pacific Region,,,,,,,,,,,,57,63,70,79,89,100,111,123,128,136,142,145,157,157,162,162,162,163,163,168,168,168,168,168,168,168,168,168,168,168,168,168 +Guizhou,China,Western Pacific Region,,,,,,,,,,,,29,38,46,56,64,69,77,89,96,109,118,131,135,140,143,144,146,146,146,146,146,146,146,146,146,146,146,146,146,146,146,146 +Tianjin,China,Western Pacific Region,,2,2,2,,,,,,,,34,40,49,63,67,70,94,81,88,91,96,106,112,119,120,122,124,125,128,130,131,133,135,135,135,135,135,136,136,136,136,136 +Shanxi,China,Western Pacific Region,,,,,,,,,,,,47,56,66,74,81,90,96,104,115,119,122,124,126,126,127,128,129,130,131,131,132,132,132,132,133,133,133,133,133,133,133,133 +Liaoning,China,Western Pacific Region,,,,,,,,,,,,60,64,70,74,81,89,94,99,105,107,108,111,116,117,119,120,121,121,121,121,121,121,121,121,121,121,121,121,121,122,122,125 +Hong Kong,China,Western Pacific Region,,,1,2,5,5,8,8,8,10,12,13,14,15,15,18,21,24,26,26,36,42,49,50,53,56,56,57,60,62,65,68,68,70,74,81,85,91,93,94,95,98,101 +Jilin,China,Western Pacific Region,,,,,,,,,,,,17,21,31,42,54,59,65,69,78,80,81,83,84,86,88,88,89,89,90,91,91,91,91,93,93,93,93,93,93,93,93,93 +Gansu,China,Western Pacific Region,,,,,,,,,,,,35,45,51,56,57,62,70,71,81,85,86,86,87,90,90,90,91,91,91,91,91,91,91,91,91,91,91,91,91,91,91,91 +Xinjiang,China,Western Pacific Region,,,,,,,,,,,,18,23,24,29,32,36,39,42,45,49,55,59,63,65,70,71,73,76,76,76,76,76,75,76,76,76,76,76,76,76,76,76 +Inner Mongolia,China,Western Pacific Region,,,,,,,,,,,,23,26,33,37,42,46,49,50,54,58,58,60,61,63,68,70,72,73,75,75,75,75,75,75,75,75,75,75,75,75,75,75 +Ningxia,China,Western Pacific Region,,,,,,,,,,,,26,28,31,34,34,40,43,45,45,49,53,58,64,67,70,70,70,70,71,71,71,71,71,71,71,71,72,72,73,73,74,74 +Taiwan,China,Western Pacific Region,,1,1,1,3,3,4,7,8,8,9,10,10,10,10,11,11,16,16,17,18,18,18,18,18,18,18,20,22,23,24,26,26,23,28,31,32,32,34,39,39,40,42 +Qinghai,China,Western Pacific Region,,,,,,,,,,,,8,9,13,15,17,18,18,18,18,18,18,18,18,18,18,18,18,18,12,18,18,18,18,18,18,18,18,18,18,18,18,18 +Macau,China,Western Pacific Region,,,1,2,2,2,5,7,7,7,7,7,7,8,8,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10 +Xizang,China,Western Pacific Region,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +Unspecified*,China,Western Pacific Region,,,131,384,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +,Japan,Western Pacific Region,1,1,1,1,3,3,4,6,7,11,14,17,20,20,20,33,25,25,25,26,26,26,28,29,33,41,53,59,65,73,85,93,105,132,144,157,164,186,210,230,239,254,268 +,Republic of Korea,Western Pacific Region,1,1,1,2,2,2,4,4,4,4,11,12,15,15,16,18,23,24,24,27,27,28,28,28,28,28,29,30,31,51,104,204,346,602,763,977,1261,1766,2337,3150,3736,4212,4812 +,Thailand,South-East Asia Region,2,2,2,4,4,5,5,14,14,14,14,19,19,19,19,25,25,25,32,32,32,33,33,33,33,34,34,35,35,35,35,35,35,35,35,37,40,40,40,42,42,42,43 +,United States of America,Region of the Americas,,,1,1,2,2,5,5,5,5,6,7,8,11,11,11,12,12,12,12,12,13,13,14,15,15,15,15,15,15,15,15,35,35,35,53,53,59,59,62,62,62,64 +,Vietnam,Western Pacific Region,,,,2,2,2,2,2,2,2,5,6,7,8,9,10,10,12,13,14,14,15,15,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16 +,Singapore,Western Pacific Region,,,,1,3,3,4,7,7,10,13,16,18,18,18,24,28,30,33,40,43,45,47,50,58,67,72,75,77,81,84,85,86,89,89,90,91,93,96,98,102,106,108 +,Italy,European Region,,,,,,,,,,,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,9,76,124,229,322,400,650,888,1128,1689,2036 +,Nepal,South-East Asia Region,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +,Australia,Western Pacific Region,,,,,3,3,4,5,7,7,9,12,12,12,12,13,14,15,15,15,15,15,15,15,15,15,15,15,15,15,15,17,21,21,21,22,23,23,23,24,25,27,33 +,Malaysia,Western Pacific Region,,,,,,3,4,4,4,7,8,8,8,8,10,10,12,14,15,17,18,18,18,18,19,21,22,22,22,22,22,22,22,22,22,22,22,22,24,24,24,24,29 +,Canada,Region of the Americas,,,,,,,1,2,3,3,3,4,4,4,4,5,5,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,9,9,10,10,11,11,14,19,19,27 +,Cambodia,Western Pacific Region,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +,France,European Region,,,,,3,3,3,3,4,5,6,6,6,6,6,6,6,6,6,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,12,18,38,57,100,100,191 +,Sri Lanka,South-East Asia Region,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 +,Iran,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,5,18,28,43,61,95,141,245,388,593,978,1501 +,India,South-East Asia Region,,,,,,,,,,1,1,1,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,5 +,Germany,European Region,,,,,,,,1,4,4,5,7,8,10,12,12,12,13,14,14,14,14,16,16,16,16,16,16,16,16,16,16,16,16,16,16,18,21,26,57,57,129,157 +,Philippines,Western Pacific Region,,,,,,,,,,1,1,1,2,2,2,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 +,Spain,European Region,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,12,25,32,45,45,114 +,United Kingdom,European Region,,,,,,,,,,,,2,2,2,2,2,2,3,3,3,4,8,8,9,9,9,9,9,9,9,9,9,9,9,9,13,13,13,16,20,23,36,39 +,Sweden,European Region,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,7,12,13,14,15 +,Switzerland,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,6,10,18,26,30 +,Austria,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,4,5,10,10,18 +,Norway,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,4,6,15,19,25 +,Kuwait,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,8,12,43,43,45,45,56,56 +,Bahrain,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,8,26,33,33,38,40,47,49 +,United Arab Emirates,Eastern Mediterranean Region,,,,,,,,,4,4,4,4,5,5,5,5,5,5,7,7,7,8,8,8,8,8,8,9,9,9,9,9,11,13,13,13,13,13,19,19,19,21,21 +,Israel,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,2,2,2,3,5,7,7,10 +,Iraq,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,5,6,7,8,13,19,26 +,Oman,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,4,4,6,6,6,6,6 +,Lebanon,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,2,2,2,2,10,13 +,Pakistan,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,2,4,4,5 +,Egypt,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2 +,Croatia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,3,3,5,7,7,9 +,Greece,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,3,3,3,7,7 +,Finland,European Region,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,6,7 +,Algeria,African Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1,5 +,Brazil,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,2,2,2 +,Russian,European Region,,,,,,,,,,,,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,3 +,Belgium,European Region,,,,,,,,,,,,,,,,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,8 +,Denmark,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,2,3,4,5 +,Estonia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1 +,Georgia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,2,3,3,3 +,North Macedonia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1 +,Romania,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,3,3,3,3 +,Afghanistan,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1,1,1,1 +,New Zealand,Western Pacific Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,2 +,Belarus,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1 +,Lithuania,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1 +,Netherlands,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,2,7,13,18 +,Nigeria,African Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,1,1 +,Mexico,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2,5,5 +,San Marino,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1,8 +,Azerbaijan,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3,3 +,Ireland,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1 +,Monaco,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,1 +,Qatar,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,3,7 +,Ecuador,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1,6 +,Czechia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,3,3 +,Iceland,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,9 +,Armenia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1 +,Luxembourg,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1 +,Indonesia,South-East Asia Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2,2 +,Dominican Republic,Region of the Americas,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1,1 +,Portugal,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,2 +,Andorra,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +,Latvia,European Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +,Jordan,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +,Morocco,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +,Saudi Arabia,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +,Tunisia,Eastern Mediterranean Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 +,Senegal,African Region,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1 Case on an international conveyance,Other,Other,,,,,,,,,,,,,,,,,20,61,64,64,70,135,175,174,218,218,355,454,454,542,621,634,634,634,695,691,691,705,705,705,706,706,706 \ No newline at end of file diff --git a/doc_requirements.txt b/doc_requirements.txt index f69cc5dcb406..3403668549b5 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -1,7 +1,7 @@ -sphinx>=2.2.0,<3.0 +sphinx==4.0.1 numpydoc==1.1.0 ipython scipy matplotlib pandas -pydata-sphinx-theme==0.4.1 +pydata-sphinx-theme==0.5.2 diff --git a/environment.yml b/environment.yml new file mode 100644 index 000000000000..22ce617af4b2 --- /dev/null +++ b/environment.yml @@ -0,0 +1,36 @@ +# To use: +# +# $ conda env create -f environment.yml # `mamba` works too for this command +# $ conda activate numpy-dev +# +name: numpy-dev +channels: + - conda-forge +dependencies: + - python + - cython + - compilers + - openblas + - nomkl + # For testing + - pytest + - pytest-cov + - pytest-xdist + - hypothesis + # For type annotations + - mypy=0.812 + - typing_extensions + # For building docs + - sphinx=4.0.1 + - numpydoc=1.1.0 + - ipython + - scipy + - pandas + - matplotlib + - pydata-sphinx-theme=0.5.2 + # For linting + - pycodestyle=2.7.0 + - gitpython + # Used in some tests + - cffi + - pytz diff --git a/linter_requirements.txt b/linter_requirements.txt new file mode 100644 index 000000000000..51a769ee0aa4 --- /dev/null +++ b/linter_requirements.txt @@ -0,0 +1,2 @@ +pycodestyle==2.7.0 +GitPython==3.1.13 \ No newline at end of file diff --git a/numpy/__init__.cython-30.pxd b/numpy/__init__.cython-30.pxd index a2c451bc1761..42a46d0b832b 100644 --- a/numpy/__init__.cython-30.pxd +++ b/numpy/__init__.cython-30.pxd @@ -910,8 +910,6 @@ cdef extern from "numpy/ufuncobject.h": void **, char *, int, int, int, int, char *, char *, int) int PyUFunc_RegisterLoopForType(ufunc, int, PyUFuncGenericFunction, int *, void *) - int PyUFunc_GenericFunction \ - (ufunc, PyObject *, PyObject *, PyArrayObject **) void PyUFunc_f_f_As_d_d \ (char **, npy_intp *, npy_intp *, void *) void PyUFunc_d_d \ diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index fd704b7e3db8..97f3da2e5673 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -868,8 +868,6 @@ cdef extern from "numpy/ufuncobject.h": void **, char *, int, int, int, int, char *, char *, int) int PyUFunc_RegisterLoopForType(ufunc, int, PyUFuncGenericFunction, int *, void *) - int PyUFunc_GenericFunction \ - (ufunc, PyObject *, PyObject *, PyArrayObject **) void PyUFunc_f_f_As_d_d \ (char **, npy_intp *, npy_intp *, void *) void PyUFunc_d_d \ diff --git a/numpy/__init__.py b/numpy/__init__.py index 3e5277318a47..baff5e1417e6 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -109,8 +109,9 @@ import sys import warnings -from ._globals import ModuleDeprecationWarning, VisibleDeprecationWarning -from ._globals import _NoValue +from ._globals import ( + ModuleDeprecationWarning, VisibleDeprecationWarning, _NoValue +) # We first need to detect if we're being called as part of the numpy setup # procedure itself in a reliable manner. @@ -130,12 +131,16 @@ your python interpreter from there.""" raise ImportError(msg) from e - from .version import git_revision as __git_revision__ - from .version import version as __version__ - __all__ = ['ModuleDeprecationWarning', 'VisibleDeprecationWarning'] + # get the version using versioneer + from ._version import get_versions + vinfo = get_versions() + __version__ = vinfo.get("closest-tag", vinfo["version"]) + __git_version__ = vinfo.get("full-revisionid") + del get_versions, vinfo + # mapping of {name: (value, deprecation_msg)} __deprecated_attrs__ = {} @@ -161,33 +166,62 @@ # Deprecations introduced in NumPy 1.20.0, 2020-06-06 import builtins as _builtins + + _msg = ( + "`np.{n}` is a deprecated alias for the builtin `{n}`. " + "To silence this warning, use `{n}` by itself. Doing this will not " + "modify any behavior and is safe. {extended_msg}\n" + "Deprecated in NumPy 1.20; for more details and guidance: " + "https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations") + + _specific_msg = ( + "If you specifically wanted the numpy scalar type, use `np.{}` here.") + + _int_extended_msg = ( + "When replacing `np.{}`, you may wish to use e.g. `np.int64` " + "or `np.int32` to specify the precision. If you wish to review " + "your current use, check the release note link for " + "additional information.") + + _type_info = [ + ("object", ""), # The NumPy scalar only exists by name. + ("bool", _specific_msg.format("bool_")), + ("float", _specific_msg.format("float64")), + ("complex", _specific_msg.format("complex128")), + ("str", _specific_msg.format("str_")), + ("int", _int_extended_msg.format("int"))] + __deprecated_attrs__.update({ - n: ( - getattr(_builtins, n), - "`np.{n}` is a deprecated alias for the builtin `{n}`. " - "Use `{n}` by itself, which is identical in behavior, to silence " - "this warning. " - "If you specifically wanted the numpy scalar type, use `np.{n}_` " - "here." - .format(n=n) - ) - for n in ["bool", "int", "float", "complex", "object", "str"] - }) - __deprecated_attrs__.update({ - n: ( - getattr(compat, n), - "`np.{n}` is a deprecated alias for `np.compat.{n}`. " - "Use `np.compat.{n}` by itself, which is identical in behavior, " - "to silence this warning. " - "In the likely event your code does not need to work on Python 2 " - "you can use the builtin ``{n2}`` for which ``np.compat.{n}`` is " - "itself an alias. " - "If you specifically wanted the numpy scalar type, use `np.{n2}_` " - "here." - .format(n=n, n2=n2) - ) - for n, n2 in [("long", "int"), ("unicode", "str")] + n: (getattr(_builtins, n), _msg.format(n=n, extended_msg=extended_msg)) + for n, extended_msg in _type_info }) + # Numpy 1.20.0, 2020-10-19 + __deprecated_attrs__["typeDict"] = ( + core.numerictypes.typeDict, + "`np.typeDict` is a deprecated alias for `np.sctypeDict`." + ) + + _msg = ( + "`np.{n}` is a deprecated alias for `np.compat.{n}`. " + "To silence this warning, use `np.compat.{n}` by itself. " + "In the likely event your code does not need to work on Python 2 " + "you can use the builtin `{n2}` for which `np.compat.{n}` is itself " + "an alias. Doing this will not modify any behaviour and is safe. " + "{extended_msg}\n" + "Deprecated in NumPy 1.20; for more details and guidance: " + "https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations") + + __deprecated_attrs__["long"] = ( + getattr(compat, "long"), + _msg.format(n="long", n2="int", + extended_msg=_int_extended_msg.format("long"))) + + __deprecated_attrs__["unicode"] = ( + getattr(compat, "unicode"), + _msg.format(n="unicode", n2="str", + extended_msg=_specific_msg.format("str_"))) + + del _msg, _specific_msg, _int_extended_msg, _type_info, _builtins from .core import round, abs, max, min # now that numpy modules are imported, can initialize limits @@ -384,3 +418,12 @@ def _mac_os_check(): # Note that this will currently only make a difference on Linux core.multiarray._set_madvise_hugepage(use_hugepage) + + # Give a warning if NumPy is reloaded or imported on a sub-interpreter + # We do this from python, since the C-module may not be reloaded and + # it is tidier organized. + core.multiarray._multiarray_umath._reload_guard() + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/numpy/__init__.pyi b/numpy/__init__.pyi index 2366e9b75a13..4ec46aea01a4 100644 --- a/numpy/__init__.pyi +++ b/numpy/__init__.pyi @@ -1,4 +1,5 @@ import builtins +import os import sys import datetime as dt from abc import abstractmethod @@ -7,24 +8,112 @@ from contextlib import ContextDecorator from numpy.core._internal import _ctypes from numpy.typing import ( + # Arrays ArrayLike, + NDArray, + _SupportsArray, + _NestedSequence, + _RecursiveSequence, + _SupportsArray, + _ArrayLikeBool_co, + _ArrayLikeUInt_co, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ArrayLikeNumber_co, + _ArrayLikeTD64_co, + _ArrayLikeDT64_co, + _ArrayLikeObject_co, + + # DTypes DTypeLike, - _Shape, - _ShapeLike, - _CharLike, - _BoolLike, - _IntLike, - _FloatLike, - _ComplexLike, - _NumberLike, _SupportsDType, _VoidDTypeLike, + + # Shapes + _Shape, + _ShapeLike, + + # Scalars + _CharLike_co, + _BoolLike_co, + _IntLike_co, + _FloatLike_co, + _ComplexLike_co, + _TD64Like_co, + _NumberLike_co, + _ScalarLike_co, + + # `number` precision NBitBase, + _256Bit, + _128Bit, + _96Bit, + _80Bit, _64Bit, _32Bit, _16Bit, _8Bit, + _NBitByte, + _NBitShort, + _NBitIntC, + _NBitIntP, + _NBitInt, + _NBitLongLong, + _NBitHalf, + _NBitSingle, + _NBitDouble, + _NBitLongDouble, + + # Character codes + _BoolCodes, + _UInt8Codes, + _UInt16Codes, + _UInt32Codes, + _UInt64Codes, + _Int8Codes, + _Int16Codes, + _Int32Codes, + _Int64Codes, + _Float16Codes, + _Float32Codes, + _Float64Codes, + _Complex64Codes, + _Complex128Codes, + _ByteCodes, + _ShortCodes, + _IntCCodes, + _IntPCodes, + _IntCodes, + _LongLongCodes, + _UByteCodes, + _UShortCodes, + _UIntCCodes, + _UIntPCodes, + _UIntCodes, + _ULongLongCodes, + _HalfCodes, + _SingleCodes, + _DoubleCodes, + _LongDoubleCodes, + _CSingleCodes, + _CDoubleCodes, + _CLongDoubleCodes, + _DT64Codes, + _TD64Codes, + _StrCodes, + _BytesCodes, + _VoidCodes, + _ObjectCodes, + + # Ufuncs + _UFunc_Nin1_Nout1, + _UFunc_Nin2_Nout1, + _UFunc_Nin1_Nout2, + _UFunc_Nin2_Nout2, + _GUFunc_Nin2_Nout1, ) + from numpy.typing._callable import ( _BoolOp, _BoolBitOp, @@ -47,6 +136,24 @@ from numpy.typing._callable import ( _FloatDivMod, _ComplexOp, _NumberOp, + _ComparisonOp, +) + +# NOTE: Numpy's mypy plugin is used for removing the types unavailable +# to the specific platform +from numpy.typing._extended_precision import ( + uint128 as uint128, + uint256 as uint256, + int128 as int128, + int256 as int256, + float80 as float80, + float96 as float96, + float128 as float128, + float256 as float256, + complex160 as complex160, + complex192 as complex192, + complex256 as complex256, + complex512 as complex512, ) from typing import ( @@ -61,6 +168,7 @@ from typing import ( Iterable, List, Mapping, + NoReturn, Optional, overload, Sequence, @@ -76,74 +184,71 @@ from typing import ( ) if sys.version_info >= (3, 8): - from typing import Literal, Protocol, SupportsIndex, Final + from typing import Literal as L, Protocol, SupportsIndex, Final else: - from typing_extensions import Literal, Protocol, Final - class SupportsIndex(Protocol): - def __index__(self) -> int: ... + from typing_extensions import Literal as L, Protocol, SupportsIndex, Final # Ensures that the stubs are picked up from numpy import ( - char, - ctypeslib, - emath, - fft, - lib, - linalg, - ma, - matrixlib, - polynomial, - random, - rec, - testing, - version, + char as char, + ctypeslib as ctypeslib, + fft as fft, + lib as lib, + linalg as linalg, + ma as ma, + matrixlib as matrixlib, + polynomial as polynomial, + random as random, + rec as rec, + testing as testing, + version as version, ) from numpy.core.function_base import ( - linspace, - logspace, - geomspace, + linspace as linspace, + logspace as logspace, + geomspace as geomspace, ) from numpy.core.fromnumeric import ( - take, - reshape, - choose, - repeat, - put, - swapaxes, - transpose, - partition, - argpartition, - sort, - argsort, - argmax, - argmin, - searchsorted, - resize, - squeeze, - diagonal, - trace, - ravel, - nonzero, - shape, - compress, - clip, - sum, - all, - any, - cumsum, - ptp, - amax, - amin, - prod, - cumprod, - ndim, - size, - around, - mean, - std, - var, + take as take, + reshape as reshape, + choose as choose, + repeat as repeat, + put as put, + swapaxes as swapaxes, + transpose as transpose, + partition as partition, + argpartition as argpartition, + sort as sort, + argsort as argsort, + argmax as argmax, + argmin as argmin, + searchsorted as searchsorted, + resize as resize, + squeeze as squeeze, + diagonal as diagonal, + trace as trace, + ravel as ravel, + nonzero as nonzero, + shape as shape, + compress as compress, + clip as clip, + sum as sum, + all as all, + any as any, + cumsum as cumsum, + ptp as ptp, + amax as amax, + amin as amin, + prod as prod, + cumprod as cumprod, + ndim as ndim, + size as size, + around as around, + mean as mean, + std as std, + var as var, ) from numpy.core._asarray import ( @@ -172,6 +277,23 @@ from numpy.core._ufunc_config import ( _ErrDictOptional, ) +from numpy.core.arrayprint import ( + set_printoptions as set_printoptions, + get_printoptions as get_printoptions, + array2string as array2string, + format_float_scientific as format_float_scientific, + format_float_positional as format_float_positional, + array_repr as array_repr, + array_str as array_str, + set_string_function as set_string_function, + printoptions as printoptions, +) + +from numpy.core.einsumfunc import ( + einsum as einsum, + einsum_path as einsum_path, +) + from numpy.core.numeric import ( zeros_like as zeros_like, ones as ones, @@ -212,6 +334,10 @@ from numpy.core.numerictypes import ( issubdtype as issubdtype, sctype2char as sctype2char, find_common_type as find_common_type, + nbytes as nbytes, + cast as cast, + ScalarType as ScalarType, + typecodes as typecodes, ) from numpy.core.shape_base import ( @@ -224,338 +350,572 @@ from numpy.core.shape_base import ( vstack as vstack, ) -# Add an object to `__all__` if their stubs are defined in an external file; -# their stubs will not be recognized otherwise. -# NOTE: This is redundant for objects defined within this file. -__all__ = [ - "linspace", - "logspace", - "geomspace", - "take", - "reshape", - "choose", - "repeat", - "put", - "swapaxes", - "transpose", - "partition", - "argpartition", - "sort", - "argsort", - "argmax", - "argmin", - "searchsorted", - "resize", - "squeeze", - "diagonal", - "trace", - "ravel", - "nonzero", - "shape", - "compress", - "clip", - "sum", - "all", - "any", - "cumsum", - "ptp", - "amax", - "amin", - "prod", - "cumprod", - "ndim", - "size", - "around", - "mean", - "std", - "var", -] +from numpy.lib import ( + emath as emath, +) + +from numpy.lib.arraypad import ( + pad as pad, +) + +from numpy.lib.arraysetops import ( + ediff1d as ediff1d, + intersect1d as intersect1d, + setxor1d as setxor1d, + union1d as union1d, + setdiff1d as setdiff1d, + unique as unique, + in1d as in1d, + isin as isin, +) + +from numpy.lib.arrayterator import ( + Arrayterator as Arrayterator, +) + +from numpy.lib.function_base import ( + select as select, + piecewise as piecewise, + trim_zeros as trim_zeros, + copy as copy, + iterable as iterable, + percentile as percentile, + diff as diff, + gradient as gradient, + angle as angle, + unwrap as unwrap, + sort_complex as sort_complex, + disp as disp, + flip as flip, + rot90 as rot90, + extract as extract, + place as place, + asarray_chkfinite as asarray_chkfinite, + average as average, + bincount as bincount, + digitize as digitize, + cov as cov, + corrcoef as corrcoef, + msort as msort, + median as median, + sinc as sinc, + hamming as hamming, + hanning as hanning, + bartlett as bartlett, + blackman as blackman, + kaiser as kaiser, + trapz as trapz, + i0 as i0, + add_newdoc as add_newdoc, + add_docstring as add_docstring, + meshgrid as meshgrid, + delete as delete, + insert as insert, + append as append, + interp as interp, + add_newdoc_ufunc as add_newdoc_ufunc, + quantile as quantile, +) + +from numpy.lib.index_tricks import ( + ravel_multi_index as ravel_multi_index, + unravel_index as unravel_index, + mgrid as mgrid, + ogrid as ogrid, + r_ as r_, + c_ as c_, + s_ as s_, + index_exp as index_exp, + ix_ as ix_, + fill_diagonal as fill_diagonal, + diag_indices as diag_indices, + diag_indices_from as diag_indices_from, +) + +from numpy.lib.nanfunctions import ( + nansum as nansum, + nanmax as nanmax, + nanmin as nanmin, + nanargmax as nanargmax, + nanargmin as nanargmin, + nanmean as nanmean, + nanmedian as nanmedian, + nanpercentile as nanpercentile, + nanvar as nanvar, + nanstd as nanstd, + nanprod as nanprod, + nancumsum as nancumsum, + nancumprod as nancumprod, + nanquantile as nanquantile, +) + +from numpy.lib.npyio import ( + savetxt as savetxt, + loadtxt as loadtxt, + genfromtxt as genfromtxt, + recfromtxt as recfromtxt, + recfromcsv as recfromcsv, + load as load, + loads as loads, + save as save, + savez as savez, + savez_compressed as savez_compressed, + packbits as packbits, + unpackbits as unpackbits, + fromregex as fromregex, +) + +from numpy.lib.polynomial import ( + poly as poly, + roots as roots, + polyint as polyint, + polyder as polyder, + polyadd as polyadd, + polysub as polysub, + polymul as polymul, + polydiv as polydiv, + polyval as polyval, + polyfit as polyfit, +) + +from numpy.lib.shape_base import ( + column_stack as column_stack, + row_stack as row_stack, + dstack as dstack, + array_split as array_split, + split as split, + hsplit as hsplit, + vsplit as vsplit, + dsplit as dsplit, + apply_over_axes as apply_over_axes, + expand_dims as expand_dims, + apply_along_axis as apply_along_axis, + kron as kron, + tile as tile, + get_array_wrap as get_array_wrap, + take_along_axis as take_along_axis, + put_along_axis as put_along_axis, +) + +from numpy.lib.stride_tricks import ( + broadcast_to as broadcast_to, + broadcast_arrays as broadcast_arrays, + broadcast_shapes as broadcast_shapes, +) + +from numpy.lib.twodim_base import ( + diag as diag, + diagflat as diagflat, + eye as eye, + fliplr as fliplr, + flipud as flipud, + tri as tri, + triu as triu, + tril as tril, + vander as vander, + histogram2d as histogram2d, + mask_indices as mask_indices, + tril_indices as tril_indices, + tril_indices_from as tril_indices_from, + triu_indices as triu_indices, + triu_indices_from as triu_indices_from, +) + +from numpy.lib.type_check import ( + mintypecode as mintypecode, + asfarray as asfarray, + real as real, + imag as imag, + iscomplex as iscomplex, + isreal as isreal, + iscomplexobj as iscomplexobj, + isrealobj as isrealobj, + nan_to_num as nan_to_num, + real_if_close as real_if_close, + typename as typename, + common_type as common_type, +) + +from numpy.lib.ufunclike import ( + fix as fix, + isposinf as isposinf, + isneginf as isneginf, +) + +from numpy.lib.utils import ( + issubclass_ as issubclass_, + issubsctype as issubsctype, + issubdtype as issubdtype, + deprecate as deprecate, + deprecate_with_doc as deprecate_with_doc, + get_include as get_include, + info as info, + source as source, + who as who, + lookfor as lookfor, + byte_bounds as byte_bounds, + safe_eval as safe_eval, +) + +__all__: List[str] +__path__: List[str] +__version__: str +__git_version__: str + +# TODO: Move placeholders to their respective module once +# their annotations are properly implemented +# +# Placeholders for classes +# TODO: Remove `__getattr__` once the classes are stubbed out +class MachAr: + def __init__( + self, + float_conv: Any = ..., + int_conv: Any = ..., + float_to_float: Any = ..., + float_to_str: Any = ..., + title: Any = ..., + ) -> None: ... + def __getattr__(self, key: str) -> Any: ... + +class busdaycalendar: + def __new__(cls, weekmask: Any = ..., holidays: Any = ...) -> Any: ... + def __getattr__(self, key: str) -> Any: ... + +class chararray(ndarray[_ShapeType, _DType_co]): + def __new__( + subtype, + shape: Any, + itemsize: Any = ..., + unicode: Any = ..., + buffer: Any = ..., + offset: Any = ..., + strides: Any = ..., + order: Any = ..., + ) -> Any: ... + def __array_finalize__(self, obj): ... + def argsort(self, axis=..., kind=..., order=...): ... + def capitalize(self): ... + def center(self, width, fillchar=...): ... + def count(self, sub, start=..., end=...): ... + def decode(self, encoding=..., errors=...): ... + def encode(self, encoding=..., errors=...): ... + def endswith(self, suffix, start=..., end=...): ... + def expandtabs(self, tabsize=...): ... + def find(self, sub, start=..., end=...): ... + def index(self, sub, start=..., end=...): ... + def isalnum(self): ... + def isalpha(self): ... + def isdigit(self): ... + def islower(self): ... + def isspace(self): ... + def istitle(self): ... + def isupper(self): ... + def join(self, seq): ... + def ljust(self, width, fillchar=...): ... + def lower(self): ... + def lstrip(self, chars=...): ... + def partition(self, sep): ... + def replace(self, old, new, count=...): ... + def rfind(self, sub, start=..., end=...): ... + def rindex(self, sub, start=..., end=...): ... + def rjust(self, width, fillchar=...): ... + def rpartition(self, sep): ... + def rsplit(self, sep=..., maxsplit=...): ... + def rstrip(self, chars=...): ... + def split(self, sep=..., maxsplit=...): ... + def splitlines(self, keepends=...): ... + def startswith(self, prefix, start=..., end=...): ... + def strip(self, chars=...): ... + def swapcase(self): ... + def title(self): ... + def translate(self, table, deletechars=...): ... + def upper(self): ... + def zfill(self, width): ... + def isnumeric(self): ... + def isdecimal(self): ... + +class finfo: + def __new__(cls, dtype: Any) -> Any: ... + def __getattr__(self, key: str) -> Any: ... + +class format_parser: + def __init__( + self, + formats: Any, + names: Any, + titles: Any, + aligned: Any = ..., + byteorder: Any = ..., + ) -> None: ... + +class iinfo: + def __init__(self, int_type: Any) -> None: ... + def __getattr__(self, key: str) -> Any: ... + +class matrix(ndarray[_ShapeType, _DType_co]): + def __new__( + subtype, + data: Any, + dtype: Any = ..., + copy: Any = ..., + ) -> Any: ... + def __array_finalize__(self, obj): ... + def __getitem__(self, index): ... + def __mul__(self, other): ... + def __rmul__(self, other): ... + def __imul__(self, other): ... + def __pow__(self, other): ... + def __ipow__(self, other): ... + def __rpow__(self, other): ... + def tolist(self): ... + def sum(self, axis=..., dtype=..., out=...): ... + def squeeze(self, axis=...): ... + def flatten(self, order=...): ... + def mean(self, axis=..., dtype=..., out=...): ... + def std(self, axis=..., dtype=..., out=..., ddof=...): ... + def var(self, axis=..., dtype=..., out=..., ddof=...): ... + def prod(self, axis=..., dtype=..., out=...): ... + def any(self, axis=..., out=...): ... + def all(self, axis=..., out=...): ... + def max(self, axis=..., out=...): ... + def argmax(self, axis=..., out=...): ... + def min(self, axis=..., out=...): ... + def argmin(self, axis=..., out=...): ... + def ptp(self, axis=..., out=...): ... + def ravel(self, order=...): ... + @property + def T(self): ... + @property + def I(self): ... + @property + def A(self): ... + @property + def A1(self): ... + @property + def H(self): ... + def getT(self): ... + def getA(self): ... + def getA1(self): ... + def getH(self): ... + def getI(self): ... + +class memmap(ndarray[_ShapeType, _DType_co]): + def __new__( + subtype, + filename: Any, + dtype: Any = ..., + mode: Any = ..., + offset: Any = ..., + shape: Any = ..., + order: Any = ..., + ) -> Any: ... + def __getattr__(self, key: str) -> Any: ... + +class nditer: + def __new__( + cls, + op: Any, + flags: Any = ..., + op_flags: Any = ..., + op_dtypes: Any = ..., + order: Any = ..., + casting: Any = ..., + op_axes: Any = ..., + itershape: Any = ..., + buffersize: Any = ..., + ) -> Any: ... + def __getattr__(self, key: str) -> Any: ... + +class poly1d: + def __init__( + self, + c_or_r: Any, + r: Any = ..., + variable: Any = ..., + ) -> None: ... + def __call__(self, val: Any) -> Any: ... + __hash__: Any + @property + def coeffs(self): ... + @coeffs.setter + def coeffs(self, value): ... + @property + def c(self): ... + @c.setter + def c(self, value): ... + @property + def coef(self): ... + @coef.setter + def coef(self, value): ... + @property + def coefficients(self): ... + @coefficients.setter + def coefficients(self, value): ... + @property + def variable(self): ... + @property + def order(self): ... + @property + def o(self): ... + @property + def roots(self): ... + @property + def r(self): ... + def __array__(self, t=...): ... + def __len__(self): ... + def __neg__(self): ... + def __pos__(self): ... + def __mul__(self, other): ... + def __rmul__(self, other): ... + def __add__(self, other): ... + def __radd__(self, other): ... + def __pow__(self, val): ... + def __sub__(self, other): ... + def __rsub__(self, other): ... + def __div__(self, other): ... + def __truediv__(self, other): ... + def __rdiv__(self, other): ... + def __rtruediv__(self, other): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def __getitem__(self, val): ... + def __setitem__(self, key, val): ... + def __iter__(self): ... + def integ(self, m=..., k=...): ... + def deriv(self, m=...): ... -DataSource: Any -MachAr: Any -ScalarType: Any -angle: Any -append: Any -apply_along_axis: Any -apply_over_axes: Any -arange: Any -array2string: Any -array_repr: Any -array_split: Any -array_str: Any -asarray_chkfinite: Any -asfarray: Any -asmatrix: Any -asscalar: Any -average: Any -bartlett: Any -bincount: Any -bitwise_not: Any -blackman: Any -bmat: Any -bool8: Any -broadcast: Any -broadcast_arrays: Any -broadcast_to: Any -busday_count: Any -busday_offset: Any -busdaycalendar: Any -byte: Any -byte_bounds: Any -bytes0: Any -c_: Any -can_cast: Any -cast: Any -cdouble: Any -cfloat: Any -chararray: Any -clongdouble: Any -clongfloat: Any -column_stack: Any -common_type: Any -compare_chararrays: Any -complex256: Any -complex_: Any -concatenate: Any -conj: Any -copy: Any -copyto: Any -corrcoef: Any -cov: Any -csingle: Any -cumproduct: Any -datetime_as_string: Any -datetime_data: Any -delete: Any -deprecate: Any -deprecate_with_doc: Any -diag: Any -diag_indices: Any -diag_indices_from: Any -diagflat: Any -diff: Any -digitize: Any -disp: Any -divide: Any -dot: Any -double: Any -dsplit: Any -dstack: Any -ediff1d: Any -einsum: Any -einsum_path: Any -expand_dims: Any -extract: Any -eye: Any -fill_diagonal: Any -finfo: Any -fix: Any -flip: Any -fliplr: Any -flipud: Any -float128: Any -float_: Any -format_float_positional: Any -format_float_scientific: Any -format_parser: Any -frombuffer: Any -fromfile: Any -fromiter: Any -frompyfunc: Any -fromregex: Any -fromstring: Any -genfromtxt: Any -get_include: Any -get_printoptions: Any -geterrobj: Any -gradient: Any -half: Any -hamming: Any -hanning: Any -histogram: Any -histogram2d: Any -histogram_bin_edges: Any -histogramdd: Any -hsplit: Any -i0: Any -iinfo: Any -imag: Any -in1d: Any -index_exp: Any -info: Any -inner: Any -insert: Any -int0: Any -int_: Any -intc: Any -interp: Any -intersect1d: Any -intp: Any -is_busday: Any -iscomplex: Any -iscomplexobj: Any -isin: Any -isneginf: Any -isposinf: Any -isreal: Any -isrealobj: Any -iterable: Any -ix_: Any -kaiser: Any -kron: Any -lexsort: Any -load: Any -loads: Any -loadtxt: Any -longcomplex: Any -longdouble: Any -longfloat: Any -longlong: Any -lookfor: Any -mafromtxt: Any -mask_indices: Any -mat: Any -matrix: Any -max: Any -may_share_memory: Any -median: Any -memmap: Any -meshgrid: Any -mgrid: Any -min: Any -min_scalar_type: Any -mintypecode: Any -mod: Any -msort: Any -nan_to_num: Any -nanargmax: Any -nanargmin: Any -nancumprod: Any -nancumsum: Any -nanmax: Any -nanmean: Any -nanmedian: Any -nanmin: Any -nanpercentile: Any -nanprod: Any -nanquantile: Any -nanstd: Any -nansum: Any -nanvar: Any -nbytes: Any -ndenumerate: Any -ndfromtxt: Any -ndindex: Any -nditer: Any -nested_iters: Any -newaxis: Any -numarray: Any -object0: Any -ogrid: Any -packbits: Any -pad: Any -percentile: Any -piecewise: Any -place: Any -poly: Any -poly1d: Any -polyadd: Any -polyder: Any -polydiv: Any -polyfit: Any -polyint: Any -polymul: Any -polysub: Any -polyval: Any -printoptions: Any -product: Any -promote_types: Any -put_along_axis: Any -putmask: Any -quantile: Any -r_: Any -ravel_multi_index: Any -real: Any -real_if_close: Any -recarray: Any -recfromcsv: Any -recfromtxt: Any -record: Any -result_type: Any -roots: Any -rot90: Any -round: Any -round_: Any -row_stack: Any -s_: Any -save: Any -savetxt: Any -savez: Any -savez_compressed: Any -select: Any -set_printoptions: Any -set_string_function: Any -setdiff1d: Any -seterrobj: Any -setxor1d: Any -shares_memory: Any -short: Any -show_config: Any -sinc: Any -single: Any -singlecomplex: Any -sort_complex: Any -source: Any -split: Any -string_: Any -take_along_axis: Any -tile: Any -trapz: Any -tri: Any -tril: Any -tril_indices: Any -tril_indices_from: Any -trim_zeros: Any -triu: Any -triu_indices: Any -triu_indices_from: Any -typeDict: Any -typecodes: Any -typename: Any -ubyte: Any -uint: Any -uint0: Any -uintc: Any -uintp: Any -ulonglong: Any -union1d: Any -unique: Any -unpackbits: Any -unravel_index: Any -unwrap: Any -ushort: Any -vander: Any -vdot: Any -vectorize: Any -void0: Any -vsplit: Any -where: Any -who: Any +class recarray(ndarray[_ShapeType, _DType_co]): + def __new__( + subtype, + shape: Any, + dtype: Any = ..., + buf: Any = ..., + offset: Any = ..., + strides: Any = ..., + formats: Any = ..., + names: Any = ..., + titles: Any = ..., + byteorder: Any = ..., + aligned: Any = ..., + order: Any = ..., + ) -> Any: ... + def __array_finalize__(self, obj): ... + def __getattribute__(self, attr): ... + def __setattr__(self, attr, val): ... + def __getitem__(self, indx): ... + def field(self, attr, val=...): ... + +class record(void): + def __getattribute__(self, attr): ... + def __setattr__(self, attr, val): ... + def __getitem__(self, indx): ... + def pprint(self): ... + +class vectorize: + pyfunc: Any + cache: Any + signature: Any + otypes: Any + excluded: Any + __doc__: Any + def __init__( + self, + pyfunc, + otypes: Any = ..., + doc: Any = ..., + excluded: Any = ..., + cache: Any = ..., + signature: Any = ..., + ) -> None: ... + def __call__(self, *args: Any, **kwargs: Any) -> Any: ... + +# Placeholders for Python-based functions +def asmatrix(data, dtype=...): ... +def asscalar(a): ... +def cumproduct(*args, **kwargs): ... +def histogram(a, bins=..., range=..., normed=..., weights=..., density=...): ... +def histogram_bin_edges(a, bins=..., range=..., weights=...): ... +def histogramdd(sample, bins=..., range=..., normed=..., weights=..., density=...): ... +def mat(data, dtype=...): ... +def max(a, axis=..., out=..., keepdims=..., initial=..., where=...): ... +def min(a, axis=..., out=..., keepdims=..., initial=..., where=...): ... +def product(*args, **kwargs): ... +def round(a, decimals=..., out=...): ... +def round_(a, decimals=..., out=...): ... +def show_config(): ... + +# Placeholders for C-based functions +# TODO: Sort out which parameters are positional-only +@overload +def arange(stop, dtype=..., *, like=...): ... +@overload +def arange(start, stop, step=..., dtype=..., *, like=...): ... +def busday_count( + begindates, + enddates, + weekmask=..., + holidays=..., + busdaycal=..., + out=..., +): ... +def busday_offset( + dates, + offsets, + roll=..., + weekmask=..., + holidays=..., + busdaycal=..., + out=..., +): ... +def can_cast(from_, to, casting=...): ... +def compare_chararrays(a, b, cmp_op, rstrip): ... +def concatenate(__a, axis=..., out=..., dtype=..., casting=...): ... +def copyto(dst, src, casting=..., where=...): ... +def datetime_as_string(arr, unit=..., timezone=..., casting=...): ... +def datetime_data(__dtype): ... +def dot(a, b, out=...): ... +def frombuffer(buffer, dtype=..., count=..., offset=..., *, like=...): ... +def fromfile( + file, dtype=..., count=..., sep=..., offset=..., *, like=... +): ... +def fromiter(iter, dtype, count=..., *, like=...): ... +def frompyfunc(func, nin, nout, * identity): ... +def fromstring(string, dtype=..., count=..., sep=..., *, like=...): ... +def geterrobj(): ... +def inner(a, b): ... +def is_busday( + dates, weekmask=..., holidays=..., busdaycal=..., out=... +): ... +def lexsort(keys, axis=...): ... +def may_share_memory(a, b, max_work=...): ... +def min_scalar_type(a): ... +def nested_iters(*args, **kwargs): ... # TODO: Sort out parameters +def promote_types(type1, type2): ... +def putmask(a, mask, values): ... +def result_type(*arrays_and_dtypes): ... +def seterrobj(errobj): ... +def shares_memory(a, b, max_work=...): ... +def vdot(a, b): ... +@overload +def where(__condition): ... +@overload +def where(__condition, __x, __y): ... _NdArraySubClass = TypeVar("_NdArraySubClass", bound=ndarray) -_DTypeScalar = TypeVar("_DTypeScalar", bound=generic) -_ByteOrder = Literal["S", "<", ">", "=", "|", "L", "B", "N", "I"] +_DTypeScalar_co = TypeVar("_DTypeScalar_co", covariant=True, bound=generic) +_ByteOrder = L["S", "<", ">", "=", "|", "L", "B", "N", "I"] -class dtype(Generic[_DTypeScalar]): +class dtype(Generic[_DTypeScalar_co]): names: Optional[Tuple[str, ...]] # Overload for subclass of generic @overload def __new__( cls, - dtype: Type[_DTypeScalar], + dtype: Type[_DTypeScalar_co], align: bool = ..., copy: bool = ..., - ) -> dtype[_DTypeScalar]: ... + ) -> dtype[_DTypeScalar_co]: ... # Overloads for string aliases, Python types, and some assorted # other special cases. Order is sometimes important because of the # subtype relationships @@ -564,285 +924,123 @@ class dtype(Generic[_DTypeScalar]): # # so we have to make sure the overloads for the narrowest type is # first. + # Builtin types @overload - def __new__( - cls, - dtype: Union[ - Type[bool], - Literal[ - "?", - "=?", - "?", - "bool", - "bool_", - ], - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[bool_]: ... + def __new__(cls, dtype: Type[bool], align: bool = ..., copy: bool = ...) -> dtype[bool_]: ... @overload - def __new__( - cls, - dtype: Literal[ - "uint8", - "u1", - "=u1", - "u1", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[uint8]: ... + def __new__(cls, dtype: Type[int], align: bool = ..., copy: bool = ...) -> dtype[int_]: ... @overload - def __new__( - cls, - dtype: Literal[ - "uint16", - "u2", - "=u2", - "u2", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[uint16]: ... + def __new__(cls, dtype: Optional[Type[float]], align: bool = ..., copy: bool = ...) -> dtype[float_]: ... @overload - def __new__( - cls, - dtype: Literal[ - "uint32", - "u4", - "=u4", - "u4", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[uint32]: ... + def __new__(cls, dtype: Type[complex], align: bool = ..., copy: bool = ...) -> dtype[complex_]: ... @overload - def __new__( - cls, - dtype: Literal[ - "uint64", - "u8", - "=u8", - "u8", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[uint64]: ... + def __new__(cls, dtype: Type[str], align: bool = ..., copy: bool = ...) -> dtype[str_]: ... @overload - def __new__( - cls, - dtype: Literal[ - "int8", - "i1", - "=i1", - "i1", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[int8]: ... + def __new__(cls, dtype: Type[bytes], align: bool = ..., copy: bool = ...) -> dtype[bytes_]: ... + + # `unsignedinteger` string-based representations @overload - def __new__( - cls, - dtype: Literal[ - "int16", - "i2", - "=i2", - "i2", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[int16]: ... + def __new__(cls, dtype: _UInt8Codes, align: bool = ..., copy: bool = ...) -> dtype[uint8]: ... @overload - def __new__( - cls, - dtype: Literal[ - "int32", - "i4", - "=i4", - "i4", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[int32]: ... + def __new__(cls, dtype: _UInt16Codes, align: bool = ..., copy: bool = ...) -> dtype[uint16]: ... @overload - def __new__( - cls, - dtype: Literal[ - "int64", - "i8", - "=i8", - "i8", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[int64]: ... - # "int"/int resolve to int_, which is system dependent and as of - # now untyped. Long-term we'll do something fancier here. + def __new__(cls, dtype: _UInt32Codes, align: bool = ..., copy: bool = ...) -> dtype[uint32]: ... @overload - def __new__( - cls, - dtype: Union[Type[int], Literal["int"]], - align: bool = ..., - copy: bool = ..., - ) -> dtype: ... + def __new__(cls, dtype: _UInt64Codes, align: bool = ..., copy: bool = ...) -> dtype[uint64]: ... @overload - def __new__( - cls, - dtype: Literal[ - "float16", - "f4", - "=f4", - "f4", - "e", - "=e", - "e", - "half", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[float16]: ... + def __new__(cls, dtype: _UByteCodes, align: bool = ..., copy: bool = ...) -> dtype[ubyte]: ... @overload - def __new__( - cls, - dtype: Literal[ - "float32", - "f4", - "=f4", - "f4", - "f", - "=f", - "f", - "single", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[float32]: ... + def __new__(cls, dtype: _UShortCodes, align: bool = ..., copy: bool = ...) -> dtype[ushort]: ... @overload - def __new__( - cls, - dtype: Union[ - None, - Type[float], - Literal[ - "float64", - "f8", - "=f8", - "f8", - "d", - "d", - "float", - "double", - "float_", - ], - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[float64]: ... + def __new__(cls, dtype: _UIntCCodes, align: bool = ..., copy: bool = ...) -> dtype[uintc]: ... @overload - def __new__( - cls, - dtype: Literal[ - "complex64", - "c8", - "=c8", - "c8", - "F", - "=F", - "F", - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[complex64]: ... + def __new__(cls, dtype: _UIntPCodes, align: bool = ..., copy: bool = ...) -> dtype[uintp]: ... @overload - def __new__( - cls, - dtype: Union[ - Type[complex], - Literal[ - "complex128", - "c16", - "=c16", - "c16", - "D", - "=D", - "D", - ], - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[complex128]: ... + def __new__(cls, dtype: _UIntCodes, align: bool = ..., copy: bool = ...) -> dtype[uint]: ... @overload - def __new__( - cls, - dtype: Union[ - Type[bytes], - Literal[ - "S", - "=S", - "S", - "bytes", - "bytes_", - "bytes0", - ], - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[bytes_]: ... + def __new__(cls, dtype: _ULongLongCodes, align: bool = ..., copy: bool = ...) -> dtype[ulonglong]: ... + + # `signedinteger` string-based representations @overload - def __new__( - cls, - dtype: Union[ - Type[str], - Literal[ - "U", - "=U", - # U intentionally not included; they are not - # the same dtype and which one dtype("U") translates - # to is platform-dependent. - "str", - "str_", - "str0", - ], - ], - align: bool = ..., - copy: bool = ..., - ) -> dtype[str_]: ... + def __new__(cls, dtype: _Int8Codes, align: bool = ..., copy: bool = ...) -> dtype[int8]: ... + @overload + def __new__(cls, dtype: _Int16Codes, align: bool = ..., copy: bool = ...) -> dtype[int16]: ... + @overload + def __new__(cls, dtype: _Int32Codes, align: bool = ..., copy: bool = ...) -> dtype[int32]: ... + @overload + def __new__(cls, dtype: _Int64Codes, align: bool = ..., copy: bool = ...) -> dtype[int64]: ... + @overload + def __new__(cls, dtype: _ByteCodes, align: bool = ..., copy: bool = ...) -> dtype[byte]: ... + @overload + def __new__(cls, dtype: _ShortCodes, align: bool = ..., copy: bool = ...) -> dtype[short]: ... + @overload + def __new__(cls, dtype: _IntCCodes, align: bool = ..., copy: bool = ...) -> dtype[intc]: ... + @overload + def __new__(cls, dtype: _IntPCodes, align: bool = ..., copy: bool = ...) -> dtype[intp]: ... + @overload + def __new__(cls, dtype: _IntCodes, align: bool = ..., copy: bool = ...) -> dtype[int_]: ... + @overload + def __new__(cls, dtype: _LongLongCodes, align: bool = ..., copy: bool = ...) -> dtype[longlong]: ... + + # `floating` string-based representations + @overload + def __new__(cls, dtype: _Float16Codes, align: bool = ..., copy: bool = ...) -> dtype[float16]: ... + @overload + def __new__(cls, dtype: _Float32Codes, align: bool = ..., copy: bool = ...) -> dtype[float32]: ... + @overload + def __new__(cls, dtype: _Float64Codes, align: bool = ..., copy: bool = ...) -> dtype[float64]: ... + @overload + def __new__(cls, dtype: _HalfCodes, align: bool = ..., copy: bool = ...) -> dtype[half]: ... + @overload + def __new__(cls, dtype: _SingleCodes, align: bool = ..., copy: bool = ...) -> dtype[single]: ... + @overload + def __new__(cls, dtype: _DoubleCodes, align: bool = ..., copy: bool = ...) -> dtype[double]: ... + @overload + def __new__(cls, dtype: _LongDoubleCodes, align: bool = ..., copy: bool = ...) -> dtype[longdouble]: ... + + # `complexfloating` string-based representations + @overload + def __new__(cls, dtype: _Complex64Codes, align: bool = ..., copy: bool = ...) -> dtype[complex64]: ... + @overload + def __new__(cls, dtype: _Complex128Codes, align: bool = ..., copy: bool = ...) -> dtype[complex128]: ... + @overload + def __new__(cls, dtype: _CSingleCodes, align: bool = ..., copy: bool = ...) -> dtype[csingle]: ... + @overload + def __new__(cls, dtype: _CDoubleCodes, align: bool = ..., copy: bool = ...) -> dtype[cdouble]: ... + @overload + def __new__(cls, dtype: _CLongDoubleCodes, align: bool = ..., copy: bool = ...) -> dtype[clongdouble]: ... + + # Miscellaneous string-based representations + @overload + def __new__(cls, dtype: _BoolCodes, align: bool = ..., copy: bool = ...) -> dtype[bool_]: ... + @overload + def __new__(cls, dtype: _TD64Codes, align: bool = ..., copy: bool = ...) -> dtype[timedelta64]: ... + @overload + def __new__(cls, dtype: _DT64Codes, align: bool = ..., copy: bool = ...) -> dtype[datetime64]: ... + @overload + def __new__(cls, dtype: _StrCodes, align: bool = ..., copy: bool = ...) -> dtype[str_]: ... + @overload + def __new__(cls, dtype: _BytesCodes, align: bool = ..., copy: bool = ...) -> dtype[bytes_]: ... + @overload + def __new__(cls, dtype: _VoidCodes, align: bool = ..., copy: bool = ...) -> dtype[void]: ... + @overload + def __new__(cls, dtype: _ObjectCodes, align: bool = ..., copy: bool = ...) -> dtype[object_]: ... + # dtype of a dtype is the same dtype @overload def __new__( cls, - dtype: dtype[_DTypeScalar], + dtype: dtype[_DTypeScalar_co], align: bool = ..., copy: bool = ..., - ) -> dtype[_DTypeScalar]: ... - # TODO: handle _SupportsDType better + ) -> dtype[_DTypeScalar_co]: ... @overload def __new__( cls, - dtype: _SupportsDType, + dtype: _SupportsDType[dtype[_DTypeScalar_co]], align: bool = ..., copy: bool = ..., - ) -> dtype[Any]: ... + ) -> dtype[_DTypeScalar_co]: ... # Handle strings that can't be expressed as literals; i.e. s1, s2, ... @overload def __new__( @@ -859,8 +1057,24 @@ class dtype(Generic[_DTypeScalar]): align: bool = ..., copy: bool = ..., ) -> dtype[void]: ... - def __eq__(self, other: DTypeLike) -> bool: ... - def __ne__(self, other: DTypeLike) -> bool: ... + + @overload + def __getitem__(self: dtype[void], key: List[str]) -> dtype[void]: ... + @overload + def __getitem__(self: dtype[void], key: Union[str, int]) -> dtype[Any]: ... + + # NOTE: In the future 1-based multiplications will also yield `void` dtypes + @overload + def __mul__(self, value: L[0]) -> None: ... # type: ignore[misc] + @overload + def __mul__(self: _DType, value: L[1]) -> _DType: ... + @overload + def __mul__(self, value: int) -> dtype[void]: ... + + # NOTE: `__rmul__` seems to be broken when used in combination with + # literals as of mypy 0.800. Set the return-type to `Any` for now. + def __rmul__(self, value: int) -> Any: ... + def __gt__(self, other: DTypeLike) -> bool: ... def __ge__(self, other: DTypeLike) -> bool: ... def __lt__(self, other: DTypeLike) -> bool: ... @@ -868,7 +1082,7 @@ class dtype(Generic[_DTypeScalar]): @property def alignment(self) -> int: ... @property - def base(self) -> dtype: ... + def base(self: _DType) -> _DType: ... @property def byteorder(self) -> str: ... @property @@ -878,7 +1092,7 @@ class dtype(Generic[_DTypeScalar]): @property def fields( self, - ) -> Optional[Mapping[str, Union[Tuple[dtype, int], Tuple[dtype, int, Any]]]]: ... + ) -> Optional[Mapping[str, Union[Tuple[dtype[Any], int], Tuple[dtype[Any], int, Any]]]]: ... @property def flags(self) -> int: ... @property @@ -898,22 +1112,22 @@ class dtype(Generic[_DTypeScalar]): @property def name(self) -> str: ... @property + def names(self) -> Optional[Tuple[str, ...]]: ... + @property def num(self) -> int: ... @property def shape(self) -> _Shape: ... @property def ndim(self) -> int: ... @property - def subdtype(self) -> Optional[Tuple[dtype, _Shape]]: ... - def newbyteorder(self, __new_order: _ByteOrder = ...) -> dtype: ... + def subdtype(self: _DType) -> Optional[Tuple[_DType, _Shape]]: ... + def newbyteorder(self: _DType, __new_order: _ByteOrder = ...) -> _DType: ... # Leave str and type for end to avoid having to use `builtins.str` # everywhere. See https://github.com/python/mypy/issues/3775 @property def str(self) -> builtins.str: ... @property - def type(self) -> Type[generic]: ... - -_DType = dtype # to avoid name conflicts with ndarray.dtype + def type(self) -> Type[_DTypeScalar_co]: ... class _flagsobj: aligned: bool @@ -955,43 +1169,39 @@ _ArrayLikeInt = Union[ _FlatIterSelf = TypeVar("_FlatIterSelf", bound=flatiter) -class flatiter(Generic[_ArraySelf]): +class flatiter(Generic[_NdArraySubClass]): @property - def base(self) -> _ArraySelf: ... + def base(self) -> _NdArraySubClass: ... @property def coords(self) -> _Shape: ... @property def index(self) -> int: ... - def copy(self) -> _ArraySelf: ... + def copy(self) -> _NdArraySubClass: ... def __iter__(self: _FlatIterSelf) -> _FlatIterSelf: ... - def __next__(self) -> generic: ... + def __next__(self: flatiter[ndarray[Any, dtype[_ScalarType]]]) -> _ScalarType: ... def __len__(self) -> int: ... @overload - def __getitem__(self, key: Union[int, integer]) -> generic: ... + def __getitem__( + self: flatiter[ndarray[Any, dtype[_ScalarType]]], + key: Union[int, integer], + ) -> _ScalarType: ... @overload def __getitem__( self, key: Union[_ArrayLikeInt, slice, ellipsis], - ) -> _ArraySelf: ... - def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... - -_OrderKACF = Optional[Literal["K", "A", "C", "F"]] -_OrderACF = Optional[Literal["A", "C", "F"]] -_OrderCF = Optional[Literal["C", "F"]] - -_ModeKind = Literal["raise", "wrap", "clip"] -_PartitionKind = Literal["introselect"] -_SortKind = Literal["quicksort", "mergesort", "heapsort", "stable"] -_SortSide = Literal["left", "right"] - -_ArrayLikeBool = Union[_BoolLike, Sequence[_BoolLike], ndarray] -_ArrayLikeIntOrBool = Union[ - _IntLike, - _BoolLike, - ndarray, - Sequence[_IntLike], - Sequence[_BoolLike], - Sequence[Sequence[Any]], # TODO: wait for support for recursive types -] + ) -> _NdArraySubClass: ... + @overload + def __array__(self: flatiter[ndarray[Any, _DType]], __dtype: None = ...) -> ndarray[Any, _DType]: ... + @overload + def __array__(self, __dtype: _DType) -> ndarray[Any, _DType]: ... + +_OrderKACF = Optional[L["K", "A", "C", "F"]] +_OrderACF = Optional[L["A", "C", "F"]] +_OrderCF = Optional[L["C", "F"]] + +_ModeKind = L["raise", "wrap", "clip"] +_PartitionKind = L["introselect"] +_SortKind = L["quicksort", "mergesort", "heapsort", "stable"] +_SortSide = L["left", "right"] _ArraySelf = TypeVar("_ArraySelf", bound=_ArrayOrScalarCommon) @@ -1006,19 +1216,14 @@ class _ArrayOrScalarCommon: def itemsize(self) -> int: ... @property def nbytes(self) -> int: ... - def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... def __bool__(self) -> bool: ... def __bytes__(self) -> bytes: ... def __str__(self) -> str: ... def __repr__(self) -> str: ... def __copy__(self: _ArraySelf) -> _ArraySelf: ... def __deepcopy__(self: _ArraySelf, __memo: Optional[dict] = ...) -> _ArraySelf: ... - def __lt__(self, other): ... - def __le__(self, other): ... def __eq__(self, other): ... def __ne__(self, other): ... - def __gt__(self, other): ... - def __ge__(self, other): ... def astype( self: _ArraySelf, dtype: DTypeLike, @@ -1030,24 +1235,14 @@ class _ArrayOrScalarCommon: def copy(self: _ArraySelf, order: _OrderKACF = ...) -> _ArraySelf: ... def dump(self, file: str) -> None: ... def dumps(self) -> bytes: ... - def flatten(self, order: _OrderKACF = ...) -> ndarray: ... def getfield( self: _ArraySelf, dtype: DTypeLike, offset: int = ... ) -> _ArraySelf: ... - def ravel(self, order: _OrderKACF = ...) -> ndarray: ... - @overload - def reshape( - self, __shape: Sequence[int], *, order: _OrderACF = ... - ) -> ndarray: ... - @overload - def reshape( - self, *shape: int, order: _OrderACF = ... - ) -> ndarray: ... def tobytes(self, order: _OrderKACF = ...) -> bytes: ... # NOTE: `tostring()` is deprecated and therefore excluded # def tostring(self, order=...): ... def tofile( - self, fid: Union[IO[bytes], str], sep: str = ..., format: str = ... + self, fid: Union[IO[bytes], str, bytes, os.PathLike[Any]], sep: str = ..., format: str = ... ) -> None: ... # generics and 0d arrays return builtin scalars def tolist(self) -> Any: ... @@ -1071,14 +1266,21 @@ class _ArrayOrScalarCommon: def __array_wrap__(array, context=...): ... def __setstate__(self, __state): ... # a `bool_` is returned when `keepdims=True` and `self` is a 0d array + @overload def all( - self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ... + self, + axis: None = ..., + out: None = ..., + keepdims: L[False] = ..., ) -> bool_: ... @overload def all( - self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ... - ) -> Union[bool_, ndarray]: ... + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + ) -> Any: ... @overload def all( self, @@ -1086,14 +1288,21 @@ class _ArrayOrScalarCommon: out: _NdArraySubClass = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... + @overload def any( - self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ... + self, + axis: None = ..., + out: None = ..., + keepdims: L[False] = ..., ) -> bool_: ... @overload def any( - self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ... - ) -> Union[bool_, ndarray]: ... + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + ) -> Any: ... @overload def any( self, @@ -1101,48 +1310,75 @@ class _ArrayOrScalarCommon: out: _NdArraySubClass = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... + @overload - def argmax(self, axis: None = ..., out: None = ...) -> signedinteger: ... + def argmax( + self, + axis: None = ..., + out: None = ..., + ) -> intp: ... @overload def argmax( - self, axis: _ShapeLike = ..., out: None = ... - ) -> Union[signedinteger, ndarray]: ... + self, + axis: _ShapeLike = ..., + out: None = ..., + ) -> Any: ... @overload def argmax( - self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... + @overload - def argmin(self, axis: None = ..., out: None = ...) -> signedinteger: ... + def argmin( + self, + axis: None = ..., + out: None = ..., + ) -> intp: ... @overload def argmin( - self, axis: _ShapeLike = ..., out: None = ... - ) -> Union[signedinteger, ndarray]: ... + self, + axis: _ShapeLike = ..., + out: None = ..., + ) -> Any: ... @overload def argmin( - self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ... + self, + axis: Optional[_ShapeLike] = ..., + out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... + def argsort( self, - axis: Optional[int] = ..., + axis: Optional[SupportsIndex] = ..., kind: Optional[_SortKind] = ..., order: Union[None, str, Sequence[str]] = ..., ) -> ndarray: ... + @overload def choose( - self, choices: ArrayLike, out: None = ..., mode: _ModeKind = ..., + self, + choices: ArrayLike, + out: None = ..., + mode: _ModeKind = ..., ) -> ndarray: ... @overload def choose( - self, choices: ArrayLike, out: _NdArraySubClass = ..., mode: _ModeKind = ..., - ) -> _NdArraySubClass: ... - @overload + self, + choices: ArrayLike, + out: _NdArraySubClass = ..., + mode: _ModeKind = ..., + ) -> _NdArraySubClass: ... + + @overload def clip( self, min: ArrayLike = ..., max: Optional[ArrayLike] = ..., out: None = ..., **kwargs: Any, - ) -> Union[number, ndarray]: ... + ) -> ndarray: ... @overload def clip( self, @@ -1150,7 +1386,7 @@ class _ArrayOrScalarCommon: max: ArrayLike = ..., out: None = ..., **kwargs: Any, - ) -> Union[number, ndarray]: ... + ) -> ndarray: ... @overload def clip( self, @@ -1167,73 +1403,75 @@ class _ArrayOrScalarCommon: out: _NdArraySubClass = ..., **kwargs: Any, ) -> _NdArraySubClass: ... + @overload def compress( - self, a: ArrayLike, axis: Optional[int] = ..., out: None = ..., + self, + a: ArrayLike, + axis: Optional[SupportsIndex] = ..., + out: None = ..., ) -> ndarray: ... @overload def compress( - self, a: ArrayLike, axis: Optional[int] = ..., out: _NdArraySubClass = ..., + self, + a: ArrayLike, + axis: Optional[SupportsIndex] = ..., + out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... + def conj(self: _ArraySelf) -> _ArraySelf: ... + def conjugate(self: _ArraySelf) -> _ArraySelf: ... + @overload def cumprod( - self, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: None = ..., + self, + axis: Optional[SupportsIndex] = ..., + dtype: DTypeLike = ..., + out: None = ..., ) -> ndarray: ... @overload def cumprod( self, - axis: Optional[int] = ..., + axis: Optional[SupportsIndex] = ..., dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... + @overload def cumsum( - self, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: None = ..., + self, + axis: Optional[SupportsIndex] = ..., + dtype: DTypeLike = ..., + out: None = ..., ) -> ndarray: ... @overload def cumsum( self, - axis: Optional[int] = ..., + axis: Optional[SupportsIndex] = ..., dtype: DTypeLike = ..., out: _NdArraySubClass = ..., ) -> _NdArraySubClass: ... - @overload - def max( - self, - axis: None = ..., - out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> number: ... + @overload def max( self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> Union[number, ndarray]: ... + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., + ) -> Any: ... @overload def max( self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., ) -> _NdArraySubClass: ... - @overload - def mean( - self, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., - ) -> number: ... + @overload def mean( self, @@ -1241,7 +1479,7 @@ class _ArrayOrScalarCommon: dtype: DTypeLike = ..., out: None = ..., keepdims: bool = ..., - ) -> Union[number, ndarray]: ... + ) -> Any: ... @overload def mean( self, @@ -1250,44 +1488,31 @@ class _ArrayOrScalarCommon: out: _NdArraySubClass = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... - @overload - def min( - self, - axis: None = ..., - out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> number: ... + @overload def min( self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> Union[number, ndarray]: ... + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., + ) -> Any: ... @overload def min( self, axis: Optional[_ShapeLike] = ..., out: _NdArraySubClass = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., ) -> _NdArraySubClass: ... - def newbyteorder(self: _ArraySelf, __new_order: _ByteOrder = ...) -> _ArraySelf: ... - @overload - def prod( - self, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> number: ... + + def newbyteorder( + self: _ArraySelf, + __new_order: _ByteOrder = ..., + ) -> _ArraySelf: ... + @overload def prod( self, @@ -1295,9 +1520,9 @@ class _ArrayOrScalarCommon: dtype: DTypeLike = ..., out: None = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> Union[number, ndarray]: ... + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., + ) -> Any: ... @overload def prod( self, @@ -1305,17 +1530,17 @@ class _ArrayOrScalarCommon: dtype: DTypeLike = ..., out: _NdArraySubClass = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., ) -> _NdArraySubClass: ... + @overload def ptp( - self, axis: None = ..., out: None = ..., keepdims: Literal[False] = ..., - ) -> number: ... - @overload - def ptp( - self, axis: Optional[_ShapeLike] = ..., out: None = ..., keepdims: bool = ..., - ) -> Union[number, ndarray]: ... + self, + axis: Optional[_ShapeLike] = ..., + out: None = ..., + keepdims: bool = ..., + ) -> Any: ... @overload def ptp( self, @@ -1323,24 +1548,20 @@ class _ArrayOrScalarCommon: out: _NdArraySubClass = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... - def repeat( - self, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... - ) -> ndarray: ... - @overload - def round(self: _ArraySelf, decimals: int = ..., out: None = ...) -> _ArraySelf: ... + @overload def round( - self, decimals: int = ..., out: _NdArraySubClass = ... - ) -> _NdArraySubClass: ... + self: _ArraySelf, + decimals: SupportsIndex = ..., + out: None = ..., + ) -> _ArraySelf: ... @overload - def std( + def round( self, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., - ) -> number: ... + decimals: SupportsIndex = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + @overload def std( self, @@ -1349,7 +1570,7 @@ class _ArrayOrScalarCommon: out: None = ..., ddof: int = ..., keepdims: bool = ..., - ) -> Union[number, ndarray]: ... + ) -> Any: ... @overload def std( self, @@ -1359,272 +1580,1237 @@ class _ArrayOrScalarCommon: ddof: int = ..., keepdims: bool = ..., ) -> _NdArraySubClass: ... + @overload def sum( self, - axis: None = ..., + axis: Optional[_ShapeLike] = ..., dtype: DTypeLike = ..., out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> number: ... + keepdims: bool = ..., + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., + ) -> Any: ... @overload def sum( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + keepdims: bool = ..., + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., + ) -> _NdArraySubClass: ... + + @overload + def var( self, axis: Optional[_ShapeLike] = ..., dtype: DTypeLike = ..., out: None = ..., + ddof: int = ..., + keepdims: bool = ..., + ) -> Any: ... + @overload + def var( + self, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ddof: int = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> Union[number, ndarray]: ... + ) -> _NdArraySubClass: ... + +_DType = TypeVar("_DType", bound=dtype[Any]) +_DType_co = TypeVar("_DType_co", covariant=True, bound=dtype[Any]) + +# TODO: Set the `bound` to something more suitable once we +# have proper shape support +_ShapeType = TypeVar("_ShapeType", bound=Any) +_NumberType = TypeVar("_NumberType", bound=number[Any]) +_BufferType = Union[ndarray, bytes, bytearray, memoryview] + +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_2Tuple = Tuple[_T, _T] +_Casting = L["no", "equiv", "safe", "same_kind", "unsafe"] + +_ArrayUInt_co = NDArray[Union[bool_, unsignedinteger[Any]]] +_ArrayInt_co = NDArray[Union[bool_, integer[Any]]] +_ArrayFloat_co = NDArray[Union[bool_, integer[Any], floating[Any]]] +_ArrayComplex_co = NDArray[Union[bool_, integer[Any], floating[Any], complexfloating[Any, Any]]] +_ArrayNumber_co = NDArray[Union[bool_, number[Any]]] +_ArrayTD64_co = NDArray[Union[bool_, integer[Any], timedelta64]] + +class _SupportsItem(Protocol[_T_co]): + def item(self, __args: Any) -> _T_co: ... + +class ndarray(_ArrayOrScalarCommon, Generic[_ShapeType, _DType_co]): + @property + def base(self) -> Optional[ndarray]: ... + @property + def ndim(self) -> int: ... + @property + def size(self) -> int: ... + @property + def real(self: _ArraySelf) -> _ArraySelf: ... + @real.setter + def real(self, value: ArrayLike) -> None: ... + @property + def imag(self: _ArraySelf) -> _ArraySelf: ... + @imag.setter + def imag(self, value: ArrayLike) -> None: ... + def __new__( + cls: Type[_ArraySelf], + shape: _ShapeLike, + dtype: DTypeLike = ..., + buffer: _BufferType = ..., + offset: int = ..., + strides: _ShapeLike = ..., + order: _OrderKACF = ..., + ) -> _ArraySelf: ... + @overload + def __array__(self, __dtype: None = ...) -> ndarray[Any, _DType_co]: ... + @overload + def __array__(self, __dtype: _DType) -> ndarray[Any, _DType]: ... + @property + def ctypes(self) -> _ctypes[int]: ... + @property + def shape(self) -> _Shape: ... + @shape.setter + def shape(self, value: _ShapeLike) -> None: ... + @property + def strides(self) -> _Shape: ... + @strides.setter + def strides(self, value: _ShapeLike) -> None: ... + def byteswap(self: _ArraySelf, inplace: bool = ...) -> _ArraySelf: ... + def fill(self, value: Any) -> None: ... + @property + def flat(self: _NdArraySubClass) -> flatiter[_NdArraySubClass]: ... + + # Use the same output type as that of the underlying `generic` + @overload + def item( + self: ndarray[Any, dtype[_SupportsItem[_T]]], # type: ignore[type-var] + *args: SupportsIndex, + ) -> _T: ... + @overload + def item( + self: ndarray[Any, dtype[_SupportsItem[_T]]], # type: ignore[type-var] + __args: Tuple[SupportsIndex, ...], + ) -> _T: ... + + @overload + def itemset(self, __value: Any) -> None: ... + @overload + def itemset(self, __item: _ShapeLike, __value: Any) -> None: ... + + @overload + def resize(self, __new_shape: _ShapeLike, *, refcheck: bool = ...) -> None: ... + @overload + def resize(self, *new_shape: SupportsIndex, refcheck: bool = ...) -> None: ... + + def setflags( + self, write: bool = ..., align: bool = ..., uic: bool = ... + ) -> None: ... + + def squeeze( + self, + axis: Union[SupportsIndex, Tuple[SupportsIndex, ...]] = ..., + ) -> ndarray[Any, _DType_co]: ... + + def swapaxes( + self, + axis1: SupportsIndex, + axis2: SupportsIndex, + ) -> ndarray[Any, _DType_co]: ... + + @overload + def transpose(self: _ArraySelf, __axes: _ShapeLike) -> _ArraySelf: ... + @overload + def transpose(self: _ArraySelf, *axes: SupportsIndex) -> _ArraySelf: ... + + def argpartition( + self, + kth: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> ndarray[Any, dtype[intp]]: ... + + def diagonal( + self, + offset: SupportsIndex = ..., + axis1: SupportsIndex = ..., + axis2: SupportsIndex = ..., + ) -> ndarray[Any, _DType_co]: ... + + # 1D + 1D returns a scalar; + # all other with at least 1 non-0D array return an ndarray. + @overload + def dot(self, b: _ScalarLike_co, out: None = ...) -> ndarray: ... + @overload + def dot(self, b: ArrayLike, out: None = ...) -> Any: ... # type: ignore[misc] + @overload + def dot(self, b: ArrayLike, out: _NdArraySubClass) -> _NdArraySubClass: ... + + # `nonzero()` is deprecated for 0d arrays/generics + def nonzero(self) -> Tuple[ndarray[Any, dtype[intp]], ...]: ... + + def partition( + self, + kth: _ArrayLikeInt_co, + axis: SupportsIndex = ..., + kind: _PartitionKind = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + + # `put` is technically available to `generic`, + # but is pointless as `generic`s are immutable + def put( + self, + ind: _ArrayLikeInt_co, + v: ArrayLike, + mode: _ModeKind = ..., + ) -> None: ... + + @overload + def searchsorted( # type: ignore[misc] + self, # >= 1D array + v: _ScalarLike_co, # 0D array-like + side: _SortSide = ..., + sorter: Optional[_ArrayLikeInt_co] = ..., + ) -> intp: ... + @overload + def searchsorted( + self, # >= 1D array + v: ArrayLike, + side: _SortSide = ..., + sorter: Optional[_ArrayLikeInt_co] = ..., + ) -> ndarray[Any, dtype[intp]]: ... + + def setfield( + self, + val: ArrayLike, + dtype: DTypeLike, + offset: SupportsIndex = ..., + ) -> None: ... + + def sort( + self, + axis: SupportsIndex = ..., + kind: Optional[_SortKind] = ..., + order: Union[None, str, Sequence[str]] = ..., + ) -> None: ... + + @overload + def trace( + self, # >= 2D array + offset: SupportsIndex = ..., + axis1: SupportsIndex = ..., + axis2: SupportsIndex = ..., + dtype: DTypeLike = ..., + out: None = ..., + ) -> Any: ... + @overload + def trace( + self, # >= 2D array + offset: SupportsIndex = ..., + axis1: SupportsIndex = ..., + axis2: SupportsIndex = ..., + dtype: DTypeLike = ..., + out: _NdArraySubClass = ..., + ) -> _NdArraySubClass: ... + + @overload + def take( # type: ignore[misc] + self: ndarray[Any, dtype[_ScalarType]], + indices: _IntLike_co, + axis: Optional[SupportsIndex] = ..., + out: None = ..., + mode: _ModeKind = ..., + ) -> _ScalarType: ... + @overload + def take( # type: ignore[misc] + self, + indices: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + out: None = ..., + mode: _ModeKind = ..., + ) -> ndarray[Any, _DType_co]: ... + @overload + def take( + self, + indices: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + out: _NdArraySubClass = ..., + mode: _ModeKind = ..., + ) -> _NdArraySubClass: ... + + def repeat( + self, + repeats: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + ) -> ndarray[Any, _DType_co]: ... + + def flatten( + self, + order: _OrderKACF = ..., + ) -> ndarray[Any, _DType_co]: ... + + def ravel( + self, + order: _OrderKACF = ..., + ) -> ndarray[Any, _DType_co]: ... + + @overload + def reshape( + self, __shape: _ShapeLike, *, order: _OrderACF = ... + ) -> ndarray[Any, _DType_co]: ... + @overload + def reshape( + self, *shape: SupportsIndex, order: _OrderACF = ... + ) -> ndarray[Any, _DType_co]: ... + + # Dispatch to the underlying `generic` via protocols + def __int__( + self: ndarray[Any, dtype[SupportsInt]], # type: ignore[type-var] + ) -> int: ... + + def __float__( + self: ndarray[Any, dtype[SupportsFloat]], # type: ignore[type-var] + ) -> float: ... + + def __complex__( + self: ndarray[Any, dtype[SupportsComplex]], # type: ignore[type-var] + ) -> complex: ... + + def __index__( + self: ndarray[Any, dtype[SupportsIndex]], # type: ignore[type-var] + ) -> int: ... + + def __len__(self) -> int: ... + def __setitem__(self, key, value): ... + def __iter__(self) -> Any: ... + def __contains__(self, key) -> bool: ... + + # The last overload is for catching recursive objects whose + # nesting is too deep. + # The first overload is for catching `bytes` (as they are a subtype of + # `Sequence[int]`) and `str`. As `str` is a recusive sequence of + # strings, it will pass through the final overload otherwise + + @overload + def __lt__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __lt__(self: _ArrayNumber_co, other: _ArrayLikeNumber_co) -> NDArray[bool_]: ... + @overload + def __lt__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[bool_]: ... + @overload + def __lt__(self: NDArray[datetime64], other: _ArrayLikeDT64_co) -> NDArray[bool_]: ... + @overload + def __lt__(self: NDArray[object_], other: Any) -> NDArray[bool_]: ... + @overload + def __lt__(self: NDArray[Any], other: _ArrayLikeObject_co) -> NDArray[bool_]: ... + @overload + def __lt__( + self: NDArray[Union[number[Any], datetime64, timedelta64, bool_]], + other: _RecursiveSequence, + ) -> NDArray[bool_]: ... + + @overload + def __le__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __le__(self: _ArrayNumber_co, other: _ArrayLikeNumber_co) -> NDArray[bool_]: ... + @overload + def __le__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[bool_]: ... + @overload + def __le__(self: NDArray[datetime64], other: _ArrayLikeDT64_co) -> NDArray[bool_]: ... + @overload + def __le__(self: NDArray[object_], other: Any) -> NDArray[bool_]: ... + @overload + def __le__(self: NDArray[Any], other: _ArrayLikeObject_co) -> NDArray[bool_]: ... + @overload + def __le__( + self: NDArray[Union[number[Any], datetime64, timedelta64, bool_]], + other: _RecursiveSequence, + ) -> NDArray[bool_]: ... + + @overload + def __gt__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __gt__(self: _ArrayNumber_co, other: _ArrayLikeNumber_co) -> NDArray[bool_]: ... + @overload + def __gt__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[bool_]: ... + @overload + def __gt__(self: NDArray[datetime64], other: _ArrayLikeDT64_co) -> NDArray[bool_]: ... + @overload + def __gt__(self: NDArray[object_], other: Any) -> NDArray[bool_]: ... + @overload + def __gt__(self: NDArray[Any], other: _ArrayLikeObject_co) -> NDArray[bool_]: ... + @overload + def __gt__( + self: NDArray[Union[number[Any], datetime64, timedelta64, bool_]], + other: _RecursiveSequence, + ) -> NDArray[bool_]: ... + + @overload + def __ge__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __ge__(self: _ArrayNumber_co, other: _ArrayLikeNumber_co) -> NDArray[bool_]: ... + @overload + def __ge__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[bool_]: ... + @overload + def __ge__(self: NDArray[datetime64], other: _ArrayLikeDT64_co) -> NDArray[bool_]: ... + @overload + def __ge__(self: NDArray[object_], other: Any) -> NDArray[bool_]: ... + @overload + def __ge__(self: NDArray[Any], other: _ArrayLikeObject_co) -> NDArray[bool_]: ... + @overload + def __ge__( + self: NDArray[Union[number[Any], datetime64, timedelta64, bool_]], + other: _RecursiveSequence, + ) -> NDArray[bool_]: ... + + # Unary ops + @overload + def __abs__(self: NDArray[bool_]) -> NDArray[bool_]: ... + @overload + def __abs__(self: NDArray[complexfloating[_NBit1, _NBit1]]) -> NDArray[floating[_NBit1]]: ... + @overload + def __abs__(self: NDArray[_NumberType]) -> NDArray[_NumberType]: ... + @overload + def __abs__(self: NDArray[timedelta64]) -> NDArray[timedelta64]: ... + @overload + def __abs__(self: NDArray[object_]) -> Any: ... + + @overload + def __invert__(self: NDArray[bool_]) -> NDArray[bool_]: ... + @overload + def __invert__(self: NDArray[_IntType]) -> NDArray[_IntType]: ... + @overload + def __invert__(self: NDArray[object_]) -> Any: ... + + @overload + def __pos__(self: NDArray[_NumberType]) -> NDArray[_NumberType]: ... + @overload + def __pos__(self: NDArray[timedelta64]) -> NDArray[timedelta64]: ... + @overload + def __pos__(self: NDArray[object_]) -> Any: ... + + @overload + def __neg__(self: NDArray[_NumberType]) -> NDArray[_NumberType]: ... + @overload + def __neg__(self: NDArray[timedelta64]) -> NDArray[timedelta64]: ... + @overload + def __neg__(self: NDArray[object_]) -> Any: ... + + # Binary ops + # NOTE: `ndarray` does not implement `__imatmul__` + @overload + def __matmul__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __matmul__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __matmul__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __matmul__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __matmul__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __matmul__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + @overload + def __matmul__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __matmul__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __matmul__( + self: _ArrayNumber_co, + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rmatmul__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rmatmul__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __rmatmul__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rmatmul__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rmatmul__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rmatmul__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + @overload + def __rmatmul__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rmatmul__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rmatmul__( + self: _ArrayNumber_co, + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __mod__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __mod__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __mod__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __mod__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __mod__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __mod__(self: _ArrayTD64_co, other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[timedelta64]: ... + @overload + def __mod__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __mod__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __mod__( + self: NDArray[Union[bool_, integer[Any], floating[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rmod__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rmod__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __rmod__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rmod__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rmod__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rmod__(self: _ArrayTD64_co, other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[timedelta64]: ... + @overload + def __rmod__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rmod__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rmod__( + self: NDArray[Union[bool_, integer[Any], floating[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __divmod__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __divmod__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> _2Tuple[NDArray[int8]]: ... # type: ignore[misc] + @overload + def __divmod__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> _2Tuple[NDArray[unsignedinteger[Any]]]: ... # type: ignore[misc] + @overload + def __divmod__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> _2Tuple[NDArray[signedinteger[Any]]]: ... # type: ignore[misc] + @overload + def __divmod__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> _2Tuple[NDArray[floating[Any]]]: ... # type: ignore[misc] + @overload + def __divmod__(self: _ArrayTD64_co, other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> Tuple[NDArray[int64], NDArray[timedelta64]]: ... + @overload + def __divmod__( + self: NDArray[Union[bool_, integer[Any], floating[Any], timedelta64]], + other: _RecursiveSequence, + ) -> _2Tuple[Any]: ... + + @overload + def __rdivmod__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rdivmod__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> _2Tuple[NDArray[int8]]: ... # type: ignore[misc] + @overload + def __rdivmod__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> _2Tuple[NDArray[unsignedinteger[Any]]]: ... # type: ignore[misc] + @overload + def __rdivmod__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> _2Tuple[NDArray[signedinteger[Any]]]: ... # type: ignore[misc] + @overload + def __rdivmod__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> _2Tuple[NDArray[floating[Any]]]: ... # type: ignore[misc] + @overload + def __rdivmod__(self: _ArrayTD64_co, other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> Tuple[NDArray[int64], NDArray[timedelta64]]: ... + @overload + def __rdivmod__( + self: NDArray[Union[bool_, integer[Any], floating[Any], timedelta64]], + other: _RecursiveSequence, + ) -> _2Tuple[Any]: ... + + @overload + def __add__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __add__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __add__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __add__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __add__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __add__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __add__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... # type: ignore[misc] + @overload + def __add__(self: _ArrayTD64_co, other: _ArrayLikeDT64_co) -> NDArray[datetime64]: ... + @overload + def __add__(self: NDArray[datetime64], other: _ArrayLikeTD64_co) -> NDArray[datetime64]: ... + @overload + def __add__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __add__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __add__( + self: NDArray[Union[bool_, number[Any], timedelta64, datetime64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __radd__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __radd__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __radd__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __radd__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __radd__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __radd__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __radd__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... # type: ignore[misc] + @overload + def __radd__(self: _ArrayTD64_co, other: _ArrayLikeDT64_co) -> NDArray[datetime64]: ... + @overload + def __radd__(self: NDArray[datetime64], other: _ArrayLikeTD64_co) -> NDArray[datetime64]: ... + @overload + def __radd__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __radd__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __radd__( + self: NDArray[Union[bool_, number[Any], timedelta64, datetime64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __sub__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __sub__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NoReturn: ... + @overload + def __sub__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __sub__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __sub__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __sub__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __sub__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... # type: ignore[misc] + @overload + def __sub__(self: NDArray[datetime64], other: _ArrayLikeTD64_co) -> NDArray[datetime64]: ... + @overload + def __sub__(self: NDArray[datetime64], other: _ArrayLikeDT64_co) -> NDArray[timedelta64]: ... + @overload + def __sub__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __sub__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __sub__( + self: NDArray[Union[bool_, number[Any], timedelta64, datetime64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rsub__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rsub__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NoReturn: ... + @overload + def __rsub__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rsub__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rsub__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rsub__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __rsub__(self: _ArrayTD64_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... # type: ignore[misc] + @overload + def __rsub__(self: _ArrayTD64_co, other: _ArrayLikeDT64_co) -> NDArray[datetime64]: ... # type: ignore[misc] + @overload + def __rsub__(self: NDArray[datetime64], other: _ArrayLikeDT64_co) -> NDArray[timedelta64]: ... + @overload + def __rsub__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rsub__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rsub__( + self: NDArray[Union[bool_, number[Any], timedelta64, datetime64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __mul__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __mul__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __mul__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __mul__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __mul__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __mul__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __mul__(self: _ArrayTD64_co, other: _ArrayLikeFloat_co) -> NDArray[timedelta64]: ... + @overload + def __mul__(self: _ArrayFloat_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... + @overload + def __mul__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __mul__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __mul__( + self: NDArray[Union[bool_, number[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rmul__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rmul__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __rmul__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rmul__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rmul__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rmul__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __rmul__(self: _ArrayTD64_co, other: _ArrayLikeFloat_co) -> NDArray[timedelta64]: ... + @overload + def __rmul__(self: _ArrayFloat_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... + @overload + def __rmul__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rmul__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rmul__( + self: NDArray[Union[bool_, number[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __floordiv__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __floordiv__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __floordiv__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __floordiv__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __floordiv__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __floordiv__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __floordiv__(self: NDArray[timedelta64], other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[int64]: ... + @overload + def __floordiv__(self: NDArray[timedelta64], other: _ArrayLikeBool_co) -> NoReturn: ... + @overload + def __floordiv__(self: NDArray[timedelta64], other: _ArrayLikeFloat_co) -> NDArray[timedelta64]: ... + @overload + def __floordiv__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __floordiv__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __floordiv__( + self: NDArray[Union[bool_, number[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rfloordiv__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rfloordiv__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __rfloordiv__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rfloordiv__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rfloordiv__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rfloordiv__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __rfloordiv__(self: NDArray[timedelta64], other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[int64]: ... + @overload + def __rfloordiv__(self: NDArray[bool_], other: _ArrayLikeTD64_co) -> NoReturn: ... + @overload + def __rfloordiv__(self: _ArrayFloat_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... + @overload + def __rfloordiv__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rfloordiv__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rfloordiv__( + self: NDArray[Union[bool_, number[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __pow__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __pow__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __pow__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __pow__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __pow__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __pow__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + @overload + def __pow__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __pow__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __pow__( + self: NDArray[Union[bool_, number[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rpow__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rpow__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __rpow__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rpow__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rpow__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rpow__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... + @overload + def __rpow__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rpow__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rpow__( + self: NDArray[Union[bool_, number[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __truediv__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __truediv__(self: _ArrayInt_co, other: _ArrayInt_co) -> NDArray[float64]: ... # type: ignore[misc] + @overload + def __truediv__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __truediv__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __truediv__(self: NDArray[timedelta64], other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[float64]: ... + @overload + def __truediv__(self: NDArray[timedelta64], other: _ArrayLikeBool_co) -> NoReturn: ... + @overload + def __truediv__(self: NDArray[timedelta64], other: _ArrayLikeFloat_co) -> NDArray[timedelta64]: ... + @overload + def __truediv__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __truediv__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __truediv__( + self: NDArray[Union[bool_, number[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rtruediv__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rtruediv__(self: _ArrayInt_co, other: _ArrayInt_co) -> NDArray[float64]: ... # type: ignore[misc] + @overload + def __rtruediv__(self: _ArrayFloat_co, other: _ArrayLikeFloat_co) -> NDArray[floating[Any]]: ... # type: ignore[misc] + @overload + def __rtruediv__(self: _ArrayComplex_co, other: _ArrayLikeComplex_co) -> NDArray[complexfloating[Any, Any]]: ... # type: ignore[misc] + @overload + def __rtruediv__(self: NDArray[timedelta64], other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[float64]: ... + @overload + def __rtruediv__(self: NDArray[bool_], other: _ArrayLikeTD64_co) -> NoReturn: ... + @overload + def __rtruediv__(self: _ArrayFloat_co, other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... + @overload + def __rtruediv__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rtruediv__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rtruediv__( + self: NDArray[Union[bool_, number[Any], timedelta64]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __lshift__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __lshift__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __lshift__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __lshift__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __lshift__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __lshift__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __lshift__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rlshift__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rlshift__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __rlshift__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rlshift__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __rlshift__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rlshift__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rlshift__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rshift__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rshift__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __rshift__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rshift__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __rshift__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rshift__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rshift__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rrshift__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rrshift__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[int8]: ... # type: ignore[misc] + @overload + def __rrshift__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rrshift__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __rrshift__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rrshift__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rrshift__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __and__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __and__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __and__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __and__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __and__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __and__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __and__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rand__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rand__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __rand__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rand__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __rand__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rand__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rand__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __xor__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __xor__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __xor__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __xor__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __xor__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __xor__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __xor__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __rxor__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __rxor__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __rxor__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __rxor__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __rxor__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __rxor__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __rxor__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __or__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __or__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __or__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __or__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __or__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __or__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __or__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + @overload + def __ror__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __ror__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... # type: ignore[misc] + @overload + def __ror__(self: _ArrayUInt_co, other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[Any]]: ... # type: ignore[misc] + @overload + def __ror__(self: _ArrayInt_co, other: _ArrayLikeInt_co) -> NDArray[signedinteger[Any]]: ... + @overload + def __ror__(self: NDArray[object_], other: Any) -> Any: ... + @overload + def __ror__(self: NDArray[Any], other: _ArrayLikeObject_co) -> Any: ... + @overload + def __ror__( + self: NDArray[Union[bool_, integer[Any]]], + other: _RecursiveSequence, + ) -> Any: ... + + # `np.generic` does not support inplace operations + @overload # type: ignore[misc] + def __iadd__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __iadd__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... + @overload + def __iadd__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __iadd__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __iadd__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... + @overload + def __iadd__(self: NDArray[complexfloating[_NBit1, _NBit1]], other: _ArrayLikeComplex_co) -> NDArray[complexfloating[_NBit1, _NBit1]]: ... + @overload + def __iadd__(self: NDArray[timedelta64], other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... + @overload + def __iadd__(self: NDArray[datetime64], other: _ArrayLikeTD64_co) -> NDArray[datetime64]: ... + @overload + def __iadd__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __iadd__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __isub__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __isub__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __isub__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __isub__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... + @overload + def __isub__(self: NDArray[complexfloating[_NBit1, _NBit1]], other: _ArrayLikeComplex_co) -> NDArray[complexfloating[_NBit1, _NBit1]]: ... + @overload + def __isub__(self: NDArray[timedelta64], other: _ArrayLikeTD64_co) -> NDArray[timedelta64]: ... + @overload + def __isub__(self: NDArray[datetime64], other: _ArrayLikeTD64_co) -> NDArray[datetime64]: ... + @overload + def __isub__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __isub__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __imul__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __imul__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... + @overload + def __imul__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __imul__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __imul__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... + @overload + def __imul__(self: NDArray[complexfloating[_NBit1, _NBit1]], other: _ArrayLikeComplex_co) -> NDArray[complexfloating[_NBit1, _NBit1]]: ... + @overload + def __imul__(self: NDArray[timedelta64], other: _ArrayLikeFloat_co) -> NDArray[timedelta64]: ... + @overload + def __imul__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __imul__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __itruediv__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __itruediv__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... + @overload + def __itruediv__(self: NDArray[complexfloating[_NBit1, _NBit1]], other: _ArrayLikeComplex_co) -> NDArray[complexfloating[_NBit1, _NBit1]]: ... + @overload + def __itruediv__(self: NDArray[timedelta64], other: _ArrayLikeBool_co) -> NoReturn: ... + @overload + def __itruediv__(self: NDArray[timedelta64], other: _ArrayLikeInt_co) -> NDArray[timedelta64]: ... + @overload + def __itruediv__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __itruediv__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __ifloordiv__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __ifloordiv__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __ifloordiv__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __ifloordiv__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... + @overload + def __ifloordiv__(self: NDArray[complexfloating[_NBit1, _NBit1]], other: _ArrayLikeComplex_co) -> NDArray[complexfloating[_NBit1, _NBit1]]: ... + @overload + def __ifloordiv__(self: NDArray[timedelta64], other: _ArrayLikeBool_co) -> NoReturn: ... + @overload + def __ifloordiv__(self: NDArray[timedelta64], other: _ArrayLikeInt_co) -> NDArray[timedelta64]: ... + @overload + def __ifloordiv__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __ifloordiv__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __ipow__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __ipow__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __ipow__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __ipow__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... + @overload + def __ipow__(self: NDArray[complexfloating[_NBit1, _NBit1]], other: _ArrayLikeComplex_co) -> NDArray[complexfloating[_NBit1, _NBit1]]: ... + @overload + def __ipow__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __ipow__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __imod__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __imod__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __imod__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __imod__(self: NDArray[floating[_NBit1]], other: _ArrayLikeFloat_co) -> NDArray[floating[_NBit1]]: ... @overload - def sum( - self, - axis: Optional[_ShapeLike] = ..., - dtype: DTypeLike = ..., - out: _NdArraySubClass = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., - ) -> _NdArraySubClass: ... + def __imod__(self: NDArray[timedelta64], other: _NestedSequence[_SupportsArray[dtype[timedelta64]]]) -> NDArray[timedelta64]: ... @overload - def take( - self, - indices: Union[_IntLike, _BoolLike], - axis: Optional[int] = ..., - out: None = ..., - mode: _ModeKind = ..., - ) -> generic: ... + def __imod__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... @overload - def take( - self, - indices: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - out: None = ..., - mode: _ModeKind = ..., - ) -> ndarray: ... + def __imod__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __ilshift__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... @overload - def take( - self, - indices: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - out: _NdArraySubClass = ..., - mode: _ModeKind = ..., - ) -> _NdArraySubClass: ... + def __ilshift__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... @overload - def var( - self, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., - ) -> number: ... + def __ilshift__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... @overload - def var( - self, - axis: Optional[_ShapeLike] = ..., - dtype: DTypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: bool = ..., - ) -> Union[number, ndarray]: ... + def __ilshift__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... @overload - def var( - self, - axis: Optional[_ShapeLike] = ..., - dtype: DTypeLike = ..., - out: _NdArraySubClass = ..., - ddof: int = ..., - keepdims: bool = ..., - ) -> _NdArraySubClass: ... + def __ilshift__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... -_BufferType = Union[ndarray, bytes, bytearray, memoryview] -_Casting = Literal["no", "equiv", "safe", "same_kind", "unsafe"] + @overload # type: ignore[misc] + def __irshift__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... + @overload + def __irshift__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __irshift__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __irshift__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __irshift__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... -class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): - @property - def base(self) -> Optional[ndarray]: ... - @property - def ndim(self) -> int: ... - @property - def size(self) -> int: ... - @property - def real(self: _ArraySelf) -> _ArraySelf: ... - @real.setter - def real(self, value: ArrayLike) -> None: ... - @property - def imag(self: _ArraySelf) -> _ArraySelf: ... - @imag.setter - def imag(self, value: ArrayLike) -> None: ... - def __new__( - cls: Type[_ArraySelf], - shape: Sequence[int], - dtype: DTypeLike = ..., - buffer: _BufferType = ..., - offset: int = ..., - strides: _ShapeLike = ..., - order: _OrderKACF = ..., - ) -> _ArraySelf: ... - @property - def dtype(self) -> _DType: ... - @property - def ctypes(self) -> _ctypes: ... - @property - def shape(self) -> _Shape: ... - @shape.setter - def shape(self, value: _ShapeLike): ... - @property - def strides(self) -> _Shape: ... - @strides.setter - def strides(self, value: _ShapeLike): ... - def byteswap(self: _ArraySelf, inplace: bool = ...) -> _ArraySelf: ... - def fill(self, value: Any) -> None: ... - @property - def flat(self: _ArraySelf) -> flatiter[_ArraySelf]: ... + @overload # type: ignore[misc] + def __iand__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... @overload - def item(self, *args: int) -> Any: ... + def __iand__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... @overload - def item(self, __args: Tuple[int, ...]) -> Any: ... + def __iand__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... @overload - def itemset(self, __value: Any) -> None: ... + def __iand__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... @overload - def itemset(self, __item: _ShapeLike, __value: Any) -> None: ... + def __iand__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... @overload - def resize(self, __new_shape: Sequence[int], *, refcheck: bool = ...) -> None: ... + def __iand__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __ixor__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... @overload - def resize(self, *new_shape: int, refcheck: bool = ...) -> None: ... - def setflags( - self, write: bool = ..., align: bool = ..., uic: bool = ... - ) -> None: ... - def squeeze( - self: _ArraySelf, axis: Union[int, Tuple[int, ...]] = ... - ) -> _ArraySelf: ... - def swapaxes(self: _ArraySelf, axis1: int, axis2: int) -> _ArraySelf: ... + def __ixor__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... @overload - def transpose(self: _ArraySelf, __axes: Sequence[int]) -> _ArraySelf: ... + def __ixor__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... @overload - def transpose(self: _ArraySelf, *axes: int) -> _ArraySelf: ... - def argpartition( - self, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., - ) -> ndarray: ... - def diagonal( - self: _ArraySelf, offset: int = ..., axis1: int = ..., axis2: int = ... - ) -> _ArraySelf: ... + def __ixor__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... @overload - def dot(self, b: ArrayLike, out: None = ...) -> Union[number, ndarray]: ... + def __ixor__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... @overload - def dot(self, b: ArrayLike, out: _NdArraySubClass = ...) -> _NdArraySubClass: ... - # `nonzero()` is deprecated for 0d arrays/generics - def nonzero(self) -> Tuple[ndarray, ...]: ... - def partition( - self, - kth: _ArrayLikeIntOrBool, - axis: int = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., - ) -> None: ... - # `put` is technically available to `generic`, - # but is pointless as `generic`s are immutable - def put( - self, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _ModeKind = ... - ) -> None: ... - def searchsorted( - self, # >= 1D array - v: ArrayLike, - side: _SortSide = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array - ) -> ndarray: ... - def setfield( - self, val: ArrayLike, dtype: DTypeLike, offset: int = ... - ) -> None: ... - def sort( - self, - axis: int = ..., - kind: Optional[_SortKind] = ..., - order: Union[None, str, Sequence[str]] = ..., - ) -> None: ... + def __ixor__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + @overload # type: ignore[misc] + def __ior__(self: NDArray[Any], other: _NestedSequence[Union[str, bytes]]) -> NoReturn: ... @overload - def trace( - self, # >= 2D array - offset: int = ..., - axis1: int = ..., - axis2: int = ..., - dtype: DTypeLike = ..., - out: None = ..., - ) -> Union[number, ndarray]: ... + def __ior__(self: NDArray[bool_], other: _ArrayLikeBool_co) -> NDArray[bool_]: ... @overload - def trace( - self, # >= 2D array - offset: int = ..., - axis1: int = ..., - axis2: int = ..., - dtype: DTypeLike = ..., - out: _NdArraySubClass = ..., - ) -> _NdArraySubClass: ... - # Many of these special methods are irrelevant currently, since protocols - # aren't supported yet. That said, I'm adding them for completeness. - # https://docs.python.org/3/reference/datamodel.html - def __int__(self) -> int: ... - def __float__(self) -> float: ... - def __complex__(self) -> complex: ... - def __len__(self) -> int: ... - def __setitem__(self, key, value): ... - def __iter__(self) -> Any: ... - def __contains__(self, key) -> bool: ... - def __index__(self) -> int: ... - def __matmul__(self, other: ArrayLike) -> Any: ... - # NOTE: `ndarray` does not implement `__imatmul__` - def __rmatmul__(self, other: ArrayLike) -> Any: ... - def __neg__(self: _ArraySelf) -> Any: ... - def __pos__(self: _ArraySelf) -> Any: ... - def __abs__(self: _ArraySelf) -> Any: ... - def __mod__(self, other: ArrayLike) -> Any: ... - def __rmod__(self, other: ArrayLike) -> Any: ... - def __divmod__(self, other: ArrayLike) -> Tuple[Any, Any]: ... - def __rdivmod__(self, other: ArrayLike) -> Tuple[Any, Any]: ... - def __add__(self, other: ArrayLike) -> Any: ... - def __radd__(self, other: ArrayLike) -> Any: ... - def __sub__(self, other: ArrayLike) -> Any: ... - def __rsub__(self, other: ArrayLike) -> Any: ... - def __mul__(self, other: ArrayLike) -> Any: ... - def __rmul__(self, other: ArrayLike) -> Any: ... - def __floordiv__(self, other: ArrayLike) -> Any: ... - def __rfloordiv__(self, other: ArrayLike) -> Any: ... - def __pow__(self, other: ArrayLike) -> Any: ... - def __rpow__(self, other: ArrayLike) -> Any: ... - def __truediv__(self, other: ArrayLike) -> Any: ... - def __rtruediv__(self, other: ArrayLike) -> Any: ... - def __invert__(self: _ArraySelf) -> Any: ... - def __lshift__(self, other: ArrayLike) -> Any: ... - def __rlshift__(self, other: ArrayLike) -> Any: ... - def __rshift__(self, other: ArrayLike) -> Any: ... - def __rrshift__(self, other: ArrayLike) -> Any: ... - def __and__(self, other: ArrayLike) -> Any: ... - def __rand__(self, other: ArrayLike) -> Any: ... - def __xor__(self, other: ArrayLike) -> Any: ... - def __rxor__(self, other: ArrayLike) -> Any: ... - def __or__(self, other: ArrayLike) -> Any: ... - def __ror__(self, other: ArrayLike) -> Any: ... - # `np.generic` does not support inplace operations - def __iadd__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __isub__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __imul__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __itruediv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __ifloordiv__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __ipow__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __imod__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __ilshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __irshift__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __iand__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __ixor__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... - def __ior__(self: _ArraySelf, other: ArrayLike) -> _ArraySelf: ... + def __ior__(self: NDArray[unsignedinteger[_NBit1]], other: _ArrayLikeUInt_co) -> NDArray[unsignedinteger[_NBit1]]: ... + @overload + def __ior__(self: NDArray[signedinteger[_NBit1]], other: _ArrayLikeInt_co) -> NDArray[signedinteger[_NBit1]]: ... + @overload + def __ior__(self: NDArray[object_], other: Any) -> NDArray[object_]: ... + @overload + def __ior__(self: NDArray[_ScalarType], other: _RecursiveSequence) -> NDArray[_ScalarType]: ... + + # Keep `dtype` at the bottom to avoid name conflicts with `np.dtype` + @property + def dtype(self) -> _DType_co: ... # NOTE: while `np.generic` is not technically an instance of `ABCMeta`, # the `@abstractmethod` decorator is herein used to (forcefully) deny @@ -1635,37 +2821,93 @@ class ndarray(_ArrayOrScalarCommon, Iterable, Sized, Container): # See https://github.com/numpy/numpy-stubs/pull/80 for more details. _ScalarType = TypeVar("_ScalarType", bound=generic) -_NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) -_NBit_co2 = TypeVar("_NBit_co2", covariant=True, bound=NBitBase) +_NBit1 = TypeVar("_NBit1", bound=NBitBase) +_NBit2 = TypeVar("_NBit2", bound=NBitBase) class generic(_ArrayOrScalarCommon): @abstractmethod def __init__(self, *args: Any, **kwargs: Any) -> None: ... + @overload + def __array__(self: _ScalarType, __dtype: None = ...) -> ndarray[Any, dtype[_ScalarType]]: ... + @overload + def __array__(self, __dtype: _DType) -> ndarray[Any, _DType]: ... @property def base(self) -> None: ... @property - def dtype(self: _ScalarType) -> _DType[_ScalarType]: ... + def ndim(self) -> L[0]: ... @property - def ndim(self) -> Literal[0]: ... - @property - def size(self) -> Literal[1]: ... + def size(self) -> L[1]: ... @property def shape(self) -> Tuple[()]: ... @property def strides(self) -> Tuple[()]: ... - def byteswap(self: _ScalarType, inplace: Literal[False] = ...) -> _ScalarType: ... + def byteswap(self: _ScalarType, inplace: L[False] = ...) -> _ScalarType: ... @property - def flat(self) -> flatiter[ndarray]: ... + def flat(self: _ScalarType) -> flatiter[ndarray[Any, dtype[_ScalarType]]]: ... def item( - self: _ScalarType, - __args: Union[Literal[0], Tuple[()], Tuple[Literal[0]]] = ..., + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., ) -> Any: ... + + @overload + def take( # type: ignore[misc] + self: _ScalarType, + indices: _IntLike_co, + axis: Optional[SupportsIndex] = ..., + out: None = ..., + mode: _ModeKind = ..., + ) -> _ScalarType: ... + @overload + def take( # type: ignore[misc] + self: _ScalarType, + indices: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + out: None = ..., + mode: _ModeKind = ..., + ) -> ndarray[Any, dtype[_ScalarType]]: ... + @overload + def take( + self, + indices: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + out: _NdArraySubClass = ..., + mode: _ModeKind = ..., + ) -> _NdArraySubClass: ... + + def repeat( + self: _ScalarType, + repeats: _ArrayLikeInt_co, + axis: Optional[SupportsIndex] = ..., + ) -> ndarray[Any, dtype[_ScalarType]]: ... + + def flatten( + self: _ScalarType, + order: _OrderKACF = ..., + ) -> ndarray[Any, dtype[_ScalarType]]: ... + + def ravel( + self: _ScalarType, + order: _OrderKACF = ..., + ) -> ndarray[Any, dtype[_ScalarType]]: ... + + @overload + def reshape( + self: _ScalarType, __shape: _ShapeLike, *, order: _OrderACF = ... + ) -> ndarray[Any, dtype[_ScalarType]]: ... + @overload + def reshape( + self: _ScalarType, *shape: SupportsIndex, order: _OrderACF = ... + ) -> ndarray[Any, dtype[_ScalarType]]: ... + def squeeze( - self: _ScalarType, axis: Union[Literal[0], Tuple[()]] = ... + self: _ScalarType, axis: Union[L[0], Tuple[()]] = ... ) -> _ScalarType: ... def transpose(self: _ScalarType, __axes: Tuple[()] = ...) -> _ScalarType: ... + # Keep `dtype` at the bottom to avoid name conflicts with `np.dtype` + @property + def dtype(self: _ScalarType) -> dtype[_ScalarType]: ... -class number(generic, Generic[_NBit_co]): # type: ignore +class number(generic, Generic[_NBit1]): # type: ignore @property def real(self: _ArraySelf) -> _ArraySelf: ... @property @@ -1689,9 +2931,18 @@ class number(generic, Generic[_NBit_co]): # type: ignore __rpow__: _NumberOp __truediv__: _NumberOp __rtruediv__: _NumberOp + __lt__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + __le__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + __gt__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + __ge__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] class bool_(generic): def __init__(self, __value: object = ...) -> None: ... + def item( + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., + ) -> bool: ... + def tolist(self) -> bool: ... @property def real(self: _ArraySelf) -> _ArraySelf: ... @property @@ -1727,6 +2978,12 @@ class bool_(generic): __rmod__: _BoolMod __divmod__: _BoolDivMod __rdivmod__: _BoolDivMod + __lt__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + __le__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + __gt__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + __ge__: _ComparisonOp[_NumberLike_co, _ArrayLikeNumber_co] + +bool8 = bool_ class object_(generic): def __init__(self, __value: object = ...) -> None: ... @@ -1734,109 +2991,163 @@ class object_(generic): def real(self: _ArraySelf) -> _ArraySelf: ... @property def imag(self: _ArraySelf) -> _ArraySelf: ... + # The 3 protocols below may or may not raise, + # depending on the underlying object + def __int__(self) -> int: ... + def __float__(self) -> float: ... + def __complex__(self) -> complex: ... + +object0 = object_ + +# The `datetime64` constructors requires an object with the three attributes below, +# and thus supports datetime duck typing +class _DatetimeScalar(Protocol): + @property + def day(self) -> int: ... + @property + def month(self) -> int: ... + @property + def year(self) -> int: ... +# TODO: `item`/`tolist` returns either `dt.date`, `dt.datetime` or `int` +# depending on the unit class datetime64(generic): @overload def __init__( self, - __value: Union[None, datetime64, _CharLike, dt.datetime] = ..., - __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., + __value: Union[None, datetime64, _CharLike_co, _DatetimeScalar] = ..., + __format: Union[_CharLike_co, Tuple[_CharLike_co, _IntLike_co]] = ..., ) -> None: ... @overload def __init__( self, __value: int, - __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] + __format: Union[_CharLike_co, Tuple[_CharLike_co, _IntLike_co]] ) -> None: ... - def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... - def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + def __add__(self, other: _TD64Like_co) -> datetime64: ... + def __radd__(self, other: _TD64Like_co) -> datetime64: ... @overload def __sub__(self, other: datetime64) -> timedelta64: ... @overload - def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> datetime64: ... + def __sub__(self, other: _TD64Like_co) -> datetime64: ... def __rsub__(self, other: datetime64) -> timedelta64: ... + __lt__: _ComparisonOp[datetime64, _ArrayLikeDT64_co] + __le__: _ComparisonOp[datetime64, _ArrayLikeDT64_co] + __gt__: _ComparisonOp[datetime64, _ArrayLikeDT64_co] + __ge__: _ComparisonOp[datetime64, _ArrayLikeDT64_co] # Support for `__index__` was added in python 3.8 (bpo-20092) if sys.version_info >= (3, 8): - _IntValue = Union[SupportsInt, _CharLike, SupportsIndex] - _FloatValue = Union[None, _CharLike, SupportsFloat, SupportsIndex] - _ComplexValue = Union[None, _CharLike, SupportsFloat, SupportsComplex, SupportsIndex] + _IntValue = Union[SupportsInt, _CharLike_co, SupportsIndex] + _FloatValue = Union[None, _CharLike_co, SupportsFloat, SupportsIndex] + _ComplexValue = Union[ + None, + _CharLike_co, + SupportsFloat, + SupportsComplex, + SupportsIndex, + complex, # `complex` is not a subtype of `SupportsComplex` + ] else: - _IntValue = Union[SupportsInt, _CharLike] - _FloatValue = Union[None, _CharLike, SupportsFloat] - _ComplexValue = Union[None, _CharLike, SupportsFloat, SupportsComplex] - -class integer(number[_NBit_co]): # type: ignore + _IntValue = Union[SupportsInt, _CharLike_co] + _FloatValue = Union[None, _CharLike_co, SupportsFloat] + _ComplexValue = Union[ + None, + _CharLike_co, + SupportsFloat, + SupportsComplex, + complex, + ] + +class integer(number[_NBit1]): # type: ignore # NOTE: `__index__` is technically defined in the bottom-most # sub-classes (`int64`, `uint32`, etc) + def item( + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., + ) -> int: ... + def tolist(self) -> int: ... def __index__(self) -> int: ... - __truediv__: _IntTrueDiv[_NBit_co] - __rtruediv__: _IntTrueDiv[_NBit_co] - def __mod__(self, value: Union[_IntLike, integer]) -> integer: ... - def __rmod__(self, value: Union[_IntLike, integer]) -> integer: ... + __truediv__: _IntTrueDiv[_NBit1] + __rtruediv__: _IntTrueDiv[_NBit1] + def __mod__(self, value: _IntLike_co) -> integer: ... + def __rmod__(self, value: _IntLike_co) -> integer: ... def __invert__(self: _IntType) -> _IntType: ... # Ensure that objects annotated as `integer` support bit-wise operations - def __lshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __rlshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __rshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __rrshift__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __and__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __rand__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __or__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __ror__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __xor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - def __rxor__(self, other: Union[_IntLike, _BoolLike]) -> integer: ... - -class signedinteger(integer[_NBit_co]): + def __lshift__(self, other: _IntLike_co) -> integer: ... + def __rlshift__(self, other: _IntLike_co) -> integer: ... + def __rshift__(self, other: _IntLike_co) -> integer: ... + def __rrshift__(self, other: _IntLike_co) -> integer: ... + def __and__(self, other: _IntLike_co) -> integer: ... + def __rand__(self, other: _IntLike_co) -> integer: ... + def __or__(self, other: _IntLike_co) -> integer: ... + def __ror__(self, other: _IntLike_co) -> integer: ... + def __xor__(self, other: _IntLike_co) -> integer: ... + def __rxor__(self, other: _IntLike_co) -> integer: ... + +class signedinteger(integer[_NBit1]): def __init__(self, __value: _IntValue = ...) -> None: ... - __add__: _SignedIntOp[_NBit_co] - __radd__: _SignedIntOp[_NBit_co] - __sub__: _SignedIntOp[_NBit_co] - __rsub__: _SignedIntOp[_NBit_co] - __mul__: _SignedIntOp[_NBit_co] - __rmul__: _SignedIntOp[_NBit_co] - __floordiv__: _SignedIntOp[_NBit_co] - __rfloordiv__: _SignedIntOp[_NBit_co] - __pow__: _SignedIntOp[_NBit_co] - __rpow__: _SignedIntOp[_NBit_co] - __lshift__: _SignedIntBitOp[_NBit_co] - __rlshift__: _SignedIntBitOp[_NBit_co] - __rshift__: _SignedIntBitOp[_NBit_co] - __rrshift__: _SignedIntBitOp[_NBit_co] - __and__: _SignedIntBitOp[_NBit_co] - __rand__: _SignedIntBitOp[_NBit_co] - __xor__: _SignedIntBitOp[_NBit_co] - __rxor__: _SignedIntBitOp[_NBit_co] - __or__: _SignedIntBitOp[_NBit_co] - __ror__: _SignedIntBitOp[_NBit_co] - __mod__: _SignedIntMod[_NBit_co] - __rmod__: _SignedIntMod[_NBit_co] - __divmod__: _SignedIntDivMod[_NBit_co] - __rdivmod__: _SignedIntDivMod[_NBit_co] + __add__: _SignedIntOp[_NBit1] + __radd__: _SignedIntOp[_NBit1] + __sub__: _SignedIntOp[_NBit1] + __rsub__: _SignedIntOp[_NBit1] + __mul__: _SignedIntOp[_NBit1] + __rmul__: _SignedIntOp[_NBit1] + __floordiv__: _SignedIntOp[_NBit1] + __rfloordiv__: _SignedIntOp[_NBit1] + __pow__: _SignedIntOp[_NBit1] + __rpow__: _SignedIntOp[_NBit1] + __lshift__: _SignedIntBitOp[_NBit1] + __rlshift__: _SignedIntBitOp[_NBit1] + __rshift__: _SignedIntBitOp[_NBit1] + __rrshift__: _SignedIntBitOp[_NBit1] + __and__: _SignedIntBitOp[_NBit1] + __rand__: _SignedIntBitOp[_NBit1] + __xor__: _SignedIntBitOp[_NBit1] + __rxor__: _SignedIntBitOp[_NBit1] + __or__: _SignedIntBitOp[_NBit1] + __ror__: _SignedIntBitOp[_NBit1] + __mod__: _SignedIntMod[_NBit1] + __rmod__: _SignedIntMod[_NBit1] + __divmod__: _SignedIntDivMod[_NBit1] + __rdivmod__: _SignedIntDivMod[_NBit1] int8 = signedinteger[_8Bit] int16 = signedinteger[_16Bit] int32 = signedinteger[_32Bit] int64 = signedinteger[_64Bit] +byte = signedinteger[_NBitByte] +short = signedinteger[_NBitShort] +intc = signedinteger[_NBitIntC] +intp = signedinteger[_NBitIntP] +int0 = signedinteger[_NBitIntP] +int_ = signedinteger[_NBitInt] +longlong = signedinteger[_NBitLongLong] + +# TODO: `item`/`tolist` returns either `dt.timedelta` or `int` +# depending on the unit class timedelta64(generic): def __init__( self, - __value: Union[None, int, _CharLike, dt.timedelta, timedelta64] = ..., - __format: Union[_CharLike, Tuple[_CharLike, _IntLike]] = ..., + __value: Union[None, int, _CharLike_co, dt.timedelta, timedelta64] = ..., + __format: Union[_CharLike_co, Tuple[_CharLike_co, _IntLike_co]] = ..., ) -> None: ... + + # NOTE: Only a limited number of units support conversion + # to builtin scalar types: `Y`, `M`, `ns`, `ps`, `fs`, `as` def __int__(self) -> int: ... def __float__(self) -> float: ... def __complex__(self) -> complex: ... def __neg__(self: _ArraySelf) -> _ArraySelf: ... def __pos__(self: _ArraySelf) -> _ArraySelf: ... def __abs__(self: _ArraySelf) -> _ArraySelf: ... - def __add__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... - def __radd__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... - def __sub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... - def __rsub__(self, other: Union[timedelta64, _IntLike, _BoolLike]) -> timedelta64: ... - def __mul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... - def __rmul__(self, other: Union[_FloatLike, _BoolLike]) -> timedelta64: ... + def __add__(self, other: _TD64Like_co) -> timedelta64: ... + def __radd__(self, other: _TD64Like_co) -> timedelta64: ... + def __sub__(self, other: _TD64Like_co) -> timedelta64: ... + def __rsub__(self, other: _TD64Like_co) -> timedelta64: ... + def __mul__(self, other: _FloatLike_co) -> timedelta64: ... + def __rmul__(self, other: _FloatLike_co) -> timedelta64: ... __truediv__: _TD64Div[float64] __floordiv__: _TD64Div[int64] def __rtruediv__(self, other: timedelta64) -> float64: ... @@ -1845,99 +3156,140 @@ class timedelta64(generic): def __rmod__(self, other: timedelta64) -> timedelta64: ... def __divmod__(self, other: timedelta64) -> Tuple[int64, timedelta64]: ... def __rdivmod__(self, other: timedelta64) -> Tuple[int64, timedelta64]: ... + __lt__: _ComparisonOp[_TD64Like_co, _ArrayLikeTD64_co] + __le__: _ComparisonOp[_TD64Like_co, _ArrayLikeTD64_co] + __gt__: _ComparisonOp[_TD64Like_co, _ArrayLikeTD64_co] + __ge__: _ComparisonOp[_TD64Like_co, _ArrayLikeTD64_co] -class unsignedinteger(integer[_NBit_co]): +class unsignedinteger(integer[_NBit1]): # NOTE: `uint64 + signedinteger -> float64` def __init__(self, __value: _IntValue = ...) -> None: ... - __add__: _UnsignedIntOp[_NBit_co] - __radd__: _UnsignedIntOp[_NBit_co] - __sub__: _UnsignedIntOp[_NBit_co] - __rsub__: _UnsignedIntOp[_NBit_co] - __mul__: _UnsignedIntOp[_NBit_co] - __rmul__: _UnsignedIntOp[_NBit_co] - __floordiv__: _UnsignedIntOp[_NBit_co] - __rfloordiv__: _UnsignedIntOp[_NBit_co] - __pow__: _UnsignedIntOp[_NBit_co] - __rpow__: _UnsignedIntOp[_NBit_co] - __lshift__: _UnsignedIntBitOp[_NBit_co] - __rlshift__: _UnsignedIntBitOp[_NBit_co] - __rshift__: _UnsignedIntBitOp[_NBit_co] - __rrshift__: _UnsignedIntBitOp[_NBit_co] - __and__: _UnsignedIntBitOp[_NBit_co] - __rand__: _UnsignedIntBitOp[_NBit_co] - __xor__: _UnsignedIntBitOp[_NBit_co] - __rxor__: _UnsignedIntBitOp[_NBit_co] - __or__: _UnsignedIntBitOp[_NBit_co] - __ror__: _UnsignedIntBitOp[_NBit_co] - __mod__: _UnsignedIntMod[_NBit_co] - __rmod__: _UnsignedIntMod[_NBit_co] - __divmod__: _UnsignedIntDivMod[_NBit_co] - __rdivmod__: _UnsignedIntDivMod[_NBit_co] + __add__: _UnsignedIntOp[_NBit1] + __radd__: _UnsignedIntOp[_NBit1] + __sub__: _UnsignedIntOp[_NBit1] + __rsub__: _UnsignedIntOp[_NBit1] + __mul__: _UnsignedIntOp[_NBit1] + __rmul__: _UnsignedIntOp[_NBit1] + __floordiv__: _UnsignedIntOp[_NBit1] + __rfloordiv__: _UnsignedIntOp[_NBit1] + __pow__: _UnsignedIntOp[_NBit1] + __rpow__: _UnsignedIntOp[_NBit1] + __lshift__: _UnsignedIntBitOp[_NBit1] + __rlshift__: _UnsignedIntBitOp[_NBit1] + __rshift__: _UnsignedIntBitOp[_NBit1] + __rrshift__: _UnsignedIntBitOp[_NBit1] + __and__: _UnsignedIntBitOp[_NBit1] + __rand__: _UnsignedIntBitOp[_NBit1] + __xor__: _UnsignedIntBitOp[_NBit1] + __rxor__: _UnsignedIntBitOp[_NBit1] + __or__: _UnsignedIntBitOp[_NBit1] + __ror__: _UnsignedIntBitOp[_NBit1] + __mod__: _UnsignedIntMod[_NBit1] + __rmod__: _UnsignedIntMod[_NBit1] + __divmod__: _UnsignedIntDivMod[_NBit1] + __rdivmod__: _UnsignedIntDivMod[_NBit1] uint8 = unsignedinteger[_8Bit] uint16 = unsignedinteger[_16Bit] uint32 = unsignedinteger[_32Bit] uint64 = unsignedinteger[_64Bit] -class inexact(number[_NBit_co]): ... # type: ignore +ubyte = unsignedinteger[_NBitByte] +ushort = unsignedinteger[_NBitShort] +uintc = unsignedinteger[_NBitIntC] +uintp = unsignedinteger[_NBitIntP] +uint0 = unsignedinteger[_NBitIntP] +uint = unsignedinteger[_NBitInt] +ulonglong = unsignedinteger[_NBitLongLong] + +class inexact(number[_NBit1]): ... # type: ignore _IntType = TypeVar("_IntType", bound=integer) _FloatType = TypeVar('_FloatType', bound=floating) -class floating(inexact[_NBit_co]): +class floating(inexact[_NBit1]): def __init__(self, __value: _FloatValue = ...) -> None: ... - __add__: _FloatOp[_NBit_co] - __radd__: _FloatOp[_NBit_co] - __sub__: _FloatOp[_NBit_co] - __rsub__: _FloatOp[_NBit_co] - __mul__: _FloatOp[_NBit_co] - __rmul__: _FloatOp[_NBit_co] - __truediv__: _FloatOp[_NBit_co] - __rtruediv__: _FloatOp[_NBit_co] - __floordiv__: _FloatOp[_NBit_co] - __rfloordiv__: _FloatOp[_NBit_co] - __pow__: _FloatOp[_NBit_co] - __rpow__: _FloatOp[_NBit_co] - __mod__: _FloatMod[_NBit_co] - __rmod__: _FloatMod[_NBit_co] - __divmod__: _FloatDivMod[_NBit_co] - __rdivmod__: _FloatDivMod[_NBit_co] + def item( + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., + ) -> float: ... + def tolist(self) -> float: ... + __add__: _FloatOp[_NBit1] + __radd__: _FloatOp[_NBit1] + __sub__: _FloatOp[_NBit1] + __rsub__: _FloatOp[_NBit1] + __mul__: _FloatOp[_NBit1] + __rmul__: _FloatOp[_NBit1] + __truediv__: _FloatOp[_NBit1] + __rtruediv__: _FloatOp[_NBit1] + __floordiv__: _FloatOp[_NBit1] + __rfloordiv__: _FloatOp[_NBit1] + __pow__: _FloatOp[_NBit1] + __rpow__: _FloatOp[_NBit1] + __mod__: _FloatMod[_NBit1] + __rmod__: _FloatMod[_NBit1] + __divmod__: _FloatDivMod[_NBit1] + __rdivmod__: _FloatDivMod[_NBit1] float16 = floating[_16Bit] float32 = floating[_32Bit] float64 = floating[_64Bit] +half = floating[_NBitHalf] +single = floating[_NBitSingle] +double = floating[_NBitDouble] +float_ = floating[_NBitDouble] +longdouble = floating[_NBitLongDouble] +longfloat = floating[_NBitLongDouble] + # The main reason for `complexfloating` having two typevars is cosmetic. # It is used to clarify why `complex128`s precision is `_64Bit`, the latter # describing the two 64 bit floats representing its real and imaginary component -class complexfloating(inexact[_NBit_co], Generic[_NBit_co, _NBit_co2]): +class complexfloating(inexact[_NBit1], Generic[_NBit1, _NBit2]): def __init__(self, __value: _ComplexValue = ...) -> None: ... - @property - def real(self) -> floating[_NBit_co]: ... # type: ignore[override] - @property - def imag(self) -> floating[_NBit_co2]: ... # type: ignore[override] - def __abs__(self) -> floating[_NBit_co]: ... # type: ignore[override] - __add__: _ComplexOp[_NBit_co] - __radd__: _ComplexOp[_NBit_co] - __sub__: _ComplexOp[_NBit_co] - __rsub__: _ComplexOp[_NBit_co] - __mul__: _ComplexOp[_NBit_co] - __rmul__: _ComplexOp[_NBit_co] - __truediv__: _ComplexOp[_NBit_co] - __rtruediv__: _ComplexOp[_NBit_co] - __floordiv__: _ComplexOp[_NBit_co] - __rfloordiv__: _ComplexOp[_NBit_co] - __pow__: _ComplexOp[_NBit_co] - __rpow__: _ComplexOp[_NBit_co] + def item( + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., + ) -> complex: ... + def tolist(self) -> complex: ... + @property + def real(self) -> floating[_NBit1]: ... # type: ignore[override] + @property + def imag(self) -> floating[_NBit2]: ... # type: ignore[override] + def __abs__(self) -> floating[_NBit1]: ... # type: ignore[override] + __add__: _ComplexOp[_NBit1] + __radd__: _ComplexOp[_NBit1] + __sub__: _ComplexOp[_NBit1] + __rsub__: _ComplexOp[_NBit1] + __mul__: _ComplexOp[_NBit1] + __rmul__: _ComplexOp[_NBit1] + __truediv__: _ComplexOp[_NBit1] + __rtruediv__: _ComplexOp[_NBit1] + __floordiv__: _ComplexOp[_NBit1] + __rfloordiv__: _ComplexOp[_NBit1] + __pow__: _ComplexOp[_NBit1] + __rpow__: _ComplexOp[_NBit1] complex64 = complexfloating[_32Bit, _32Bit] complex128 = complexfloating[_64Bit, _64Bit] +csingle = complexfloating[_NBitSingle, _NBitSingle] +singlecomplex = complexfloating[_NBitSingle, _NBitSingle] +cdouble = complexfloating[_NBitDouble, _NBitDouble] +complex_ = complexfloating[_NBitDouble, _NBitDouble] +cfloat = complexfloating[_NBitDouble, _NBitDouble] +clongdouble = complexfloating[_NBitLongDouble, _NBitLongDouble] +clongfloat = complexfloating[_NBitLongDouble, _NBitLongDouble] +longcomplex = complexfloating[_NBitLongDouble, _NBitLongDouble] + class flexible(generic): ... # type: ignore +# TODO: `item`/`tolist` returns either `bytes` or `tuple` +# depending on whether or not it's used as an opaque bytes sequence +# or a structure class void(flexible): - def __init__(self, __value: Union[_IntLike, _BoolLike, bytes]): ... + def __init__(self, __value: Union[_IntLike_co, bytes]) -> None: ... @property def real(self: _ArraySelf) -> _ArraySelf: ... @property @@ -1945,6 +3297,10 @@ class void(flexible): def setfield( self, val: ArrayLike, dtype: DTypeLike, offset: int = ... ) -> None: ... + def __getitem__(self, key: SupportsIndex) -> Any: ... + def __setitem__(self, key: SupportsIndex, value: ArrayLike) -> None: ... + +void0 = void class character(flexible): # type: ignore def __int__(self) -> int: ... @@ -1960,6 +3316,14 @@ class bytes_(character, bytes): def __init__( self, __value: str, encoding: str = ..., errors: str = ... ) -> None: ... + def item( + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., + ) -> bytes: ... + def tolist(self) -> bytes: ... + +string_ = bytes_ +bytes0 = bytes_ class str_(character, str): @overload @@ -1968,16 +3332,14 @@ class str_(character, str): def __init__( self, __value: bytes, encoding: str = ..., errors: str = ... ) -> None: ... + def item( + self, + __args: Union[L[0], Tuple[()], Tuple[L[0]]] = ..., + ) -> str: ... + def tolist(self) -> str: ... -unicode_ = str0 = str_ - -# TODO(alan): Platform dependent types -# longcomplex, longdouble, longfloat -# bytes, short, intc, intp, longlong -# half, single, double, longdouble -# uint_, int_, float_, complex_ -# float128, complex256 -# float96 +unicode_ = str_ +str0 = str_ def array( object: object, @@ -2004,8 +3366,6 @@ def empty( like: ArrayLike = ..., ) -> ndarray: ... -def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... - # # Constants # @@ -2057,32 +3417,15 @@ False_: Final[bool_] UFUNC_PYVALS_NAME: Final[str] +newaxis: None + +# See `npt._ufunc` for more concrete nin-/nout-specific stubs class ufunc: @property def __name__(self) -> str: ... - def __call__( - self, - *args: ArrayLike, - out: Optional[Union[ndarray, Tuple[ndarray, ...]]] = ..., - where: Optional[ndarray] = ..., - # The list should be a list of tuples of ints, but since we - # don't know the signature it would need to be - # Tuple[int, ...]. But, since List is invariant something like - # e.g. List[Tuple[int, int]] isn't a subtype of - # List[Tuple[int, ...]], so we can't type precisely here. - axes: List[Any] = ..., - axis: int = ..., - keepdims: bool = ..., - casting: _Casting = ..., - order: _OrderKACF = ..., - dtype: DTypeLike = ..., - subok: bool = ..., - signature: Union[str, Tuple[str]] = ..., - # In reality this should be a length of list 3 containing an - # int, an int, and a callable, but there's no way to express - # that. - extobj: List[Union[int, Callable]] = ..., - ) -> Any: ... + @property + def __doc__(self) -> str: ... + __call__: Callable[..., Any] @property def nin(self) -> int: ... @property @@ -2112,105 +3455,105 @@ class ufunc: # raise a ValueError ufuncs with that don't accept two input # arguments and return one output argument. Because of that we # can't type them very precisely. - @property - def reduce(self) -> Any: ... - @property - def accumulate(self) -> Any: ... - @property - def reduceat(self) -> Any: ... - @property - def outer(self) -> Any: ... + reduce: Any + accumulate: Any + reduce: Any + outer: Any # Similarly at won't be defined for ufuncs that return multiple # outputs, so we can't type it very precisely. - @property - def at(self) -> Any: ... - -absolute: ufunc -add: ufunc -arccos: ufunc -arccosh: ufunc -arcsin: ufunc -arcsinh: ufunc -arctan2: ufunc -arctan: ufunc -arctanh: ufunc -bitwise_and: ufunc -bitwise_or: ufunc -bitwise_xor: ufunc -cbrt: ufunc -ceil: ufunc -conjugate: ufunc -copysign: ufunc -cos: ufunc -cosh: ufunc -deg2rad: ufunc -degrees: ufunc -divmod: ufunc -equal: ufunc -exp2: ufunc -exp: ufunc -expm1: ufunc -fabs: ufunc -float_power: ufunc -floor: ufunc -floor_divide: ufunc -fmax: ufunc -fmin: ufunc -fmod: ufunc -frexp: ufunc -gcd: ufunc -greater: ufunc -greater_equal: ufunc -heaviside: ufunc -hypot: ufunc -invert: ufunc -isfinite: ufunc -isinf: ufunc -isnan: ufunc -isnat: ufunc -lcm: ufunc -ldexp: ufunc -left_shift: ufunc -less: ufunc -less_equal: ufunc -log10: ufunc -log1p: ufunc -log2: ufunc -log: ufunc -logaddexp2: ufunc -logaddexp: ufunc -logical_and: ufunc -logical_not: ufunc -logical_or: ufunc -logical_xor: ufunc -matmul: ufunc -maximum: ufunc -minimum: ufunc -modf: ufunc -multiply: ufunc -negative: ufunc -nextafter: ufunc -not_equal: ufunc -positive: ufunc -power: ufunc -rad2deg: ufunc -radians: ufunc -reciprocal: ufunc -remainder: ufunc -right_shift: ufunc -rint: ufunc -sign: ufunc -signbit: ufunc -sin: ufunc -sinh: ufunc -spacing: ufunc -sqrt: ufunc -square: ufunc -subtract: ufunc -tan: ufunc -tanh: ufunc -true_divide: ufunc -trunc: ufunc + at: Any + +# Parameters: `__name__`, `ntypes` and `identity` +absolute: _UFunc_Nin1_Nout1[L['absolute'], L[20], None] +add: _UFunc_Nin2_Nout1[L['add'], L[22], L[0]] +arccos: _UFunc_Nin1_Nout1[L['arccos'], L[8], None] +arccosh: _UFunc_Nin1_Nout1[L['arccosh'], L[8], None] +arcsin: _UFunc_Nin1_Nout1[L['arcsin'], L[8], None] +arcsinh: _UFunc_Nin1_Nout1[L['arcsinh'], L[8], None] +arctan2: _UFunc_Nin2_Nout1[L['arctan2'], L[5], None] +arctan: _UFunc_Nin1_Nout1[L['arctan'], L[8], None] +arctanh: _UFunc_Nin1_Nout1[L['arctanh'], L[8], None] +bitwise_and: _UFunc_Nin2_Nout1[L['bitwise_and'], L[12], L[-1]] +bitwise_not: _UFunc_Nin1_Nout1[L['invert'], L[12], None] +bitwise_or: _UFunc_Nin2_Nout1[L['bitwise_or'], L[12], L[0]] +bitwise_xor: _UFunc_Nin2_Nout1[L['bitwise_xor'], L[12], L[0]] +cbrt: _UFunc_Nin1_Nout1[L['cbrt'], L[5], None] +ceil: _UFunc_Nin1_Nout1[L['ceil'], L[7], None] +conj: _UFunc_Nin1_Nout1[L['conjugate'], L[18], None] +conjugate: _UFunc_Nin1_Nout1[L['conjugate'], L[18], None] +copysign: _UFunc_Nin2_Nout1[L['copysign'], L[4], None] +cos: _UFunc_Nin1_Nout1[L['cos'], L[9], None] +cosh: _UFunc_Nin1_Nout1[L['cosh'], L[8], None] +deg2rad: _UFunc_Nin1_Nout1[L['deg2rad'], L[5], None] +degrees: _UFunc_Nin1_Nout1[L['degrees'], L[5], None] +divide: _UFunc_Nin2_Nout1[L['true_divide'], L[11], None] +divmod: _UFunc_Nin2_Nout2[L['divmod'], L[15], None] +equal: _UFunc_Nin2_Nout1[L['equal'], L[23], None] +exp2: _UFunc_Nin1_Nout1[L['exp2'], L[8], None] +exp: _UFunc_Nin1_Nout1[L['exp'], L[10], None] +expm1: _UFunc_Nin1_Nout1[L['expm1'], L[8], None] +fabs: _UFunc_Nin1_Nout1[L['fabs'], L[5], None] +float_power: _UFunc_Nin2_Nout1[L['float_power'], L[4], None] +floor: _UFunc_Nin1_Nout1[L['floor'], L[7], None] +floor_divide: _UFunc_Nin2_Nout1[L['floor_divide'], L[21], None] +fmax: _UFunc_Nin2_Nout1[L['fmax'], L[21], None] +fmin: _UFunc_Nin2_Nout1[L['fmin'], L[21], None] +fmod: _UFunc_Nin2_Nout1[L['fmod'], L[15], None] +frexp: _UFunc_Nin1_Nout2[L['frexp'], L[4], None] +gcd: _UFunc_Nin2_Nout1[L['gcd'], L[11], L[0]] +greater: _UFunc_Nin2_Nout1[L['greater'], L[23], None] +greater_equal: _UFunc_Nin2_Nout1[L['greater_equal'], L[23], None] +heaviside: _UFunc_Nin2_Nout1[L['heaviside'], L[4], None] +hypot: _UFunc_Nin2_Nout1[L['hypot'], L[5], L[0]] +invert: _UFunc_Nin1_Nout1[L['invert'], L[12], None] +isfinite: _UFunc_Nin1_Nout1[L['isfinite'], L[20], None] +isinf: _UFunc_Nin1_Nout1[L['isinf'], L[20], None] +isnan: _UFunc_Nin1_Nout1[L['isnan'], L[20], None] +isnat: _UFunc_Nin1_Nout1[L['isnat'], L[2], None] +lcm: _UFunc_Nin2_Nout1[L['lcm'], L[11], None] +ldexp: _UFunc_Nin2_Nout1[L['ldexp'], L[8], None] +left_shift: _UFunc_Nin2_Nout1[L['left_shift'], L[11], None] +less: _UFunc_Nin2_Nout1[L['less'], L[23], None] +less_equal: _UFunc_Nin2_Nout1[L['less_equal'], L[23], None] +log10: _UFunc_Nin1_Nout1[L['log10'], L[8], None] +log1p: _UFunc_Nin1_Nout1[L['log1p'], L[8], None] +log2: _UFunc_Nin1_Nout1[L['log2'], L[8], None] +log: _UFunc_Nin1_Nout1[L['log'], L[10], None] +logaddexp2: _UFunc_Nin2_Nout1[L['logaddexp2'], L[4], float] +logaddexp: _UFunc_Nin2_Nout1[L['logaddexp'], L[4], float] +logical_and: _UFunc_Nin2_Nout1[L['logical_and'], L[20], L[True]] +logical_not: _UFunc_Nin1_Nout1[L['logical_not'], L[20], None] +logical_or: _UFunc_Nin2_Nout1[L['logical_or'], L[20], L[False]] +logical_xor: _UFunc_Nin2_Nout1[L['logical_xor'], L[19], L[False]] +matmul: _GUFunc_Nin2_Nout1[L['matmul'], L[19], None] +maximum: _UFunc_Nin2_Nout1[L['maximum'], L[21], None] +minimum: _UFunc_Nin2_Nout1[L['minimum'], L[21], None] +mod: _UFunc_Nin2_Nout1[L['remainder'], L[16], None] +modf: _UFunc_Nin1_Nout2[L['modf'], L[4], None] +multiply: _UFunc_Nin2_Nout1[L['multiply'], L[23], L[1]] +negative: _UFunc_Nin1_Nout1[L['negative'], L[19], None] +nextafter: _UFunc_Nin2_Nout1[L['nextafter'], L[4], None] +not_equal: _UFunc_Nin2_Nout1[L['not_equal'], L[23], None] +positive: _UFunc_Nin1_Nout1[L['positive'], L[19], None] +power: _UFunc_Nin2_Nout1[L['power'], L[18], None] +rad2deg: _UFunc_Nin1_Nout1[L['rad2deg'], L[5], None] +radians: _UFunc_Nin1_Nout1[L['radians'], L[5], None] +reciprocal: _UFunc_Nin1_Nout1[L['reciprocal'], L[18], None] +remainder: _UFunc_Nin2_Nout1[L['remainder'], L[16], None] +right_shift: _UFunc_Nin2_Nout1[L['right_shift'], L[11], None] +rint: _UFunc_Nin1_Nout1[L['rint'], L[10], None] +sign: _UFunc_Nin1_Nout1[L['sign'], L[19], None] +signbit: _UFunc_Nin1_Nout1[L['signbit'], L[4], None] +sin: _UFunc_Nin1_Nout1[L['sin'], L[9], None] +sinh: _UFunc_Nin1_Nout1[L['sinh'], L[8], None] +spacing: _UFunc_Nin1_Nout1[L['spacing'], L[4], None] +sqrt: _UFunc_Nin1_Nout1[L['sqrt'], L[10], None] +square: _UFunc_Nin1_Nout1[L['square'], L[18], None] +subtract: _UFunc_Nin2_Nout1[L['subtract'], L[21], None] +tan: _UFunc_Nin1_Nout1[L['tan'], L[8], None] +tanh: _UFunc_Nin1_Nout1[L['tanh'], L[8], None] +true_divide: _UFunc_Nin2_Nout1[L['true_divide'], L[11], None] +trunc: _UFunc_Nin1_Nout1[L['trunc'], L[7], None] abs = absolute @@ -2252,3 +3595,73 @@ class errstate(Generic[_CallType], ContextDecorator): __exc_value: Optional[BaseException], __traceback: Optional[TracebackType], ) -> None: ... + +class ndenumerate(Generic[_ScalarType]): + iter: flatiter[NDArray[_ScalarType]] + @overload + def __new__( + cls, arr: _NestedSequence[_SupportsArray[dtype[_ScalarType]]], + ) -> ndenumerate[_ScalarType]: ... + @overload + def __new__(cls, arr: _NestedSequence[str]) -> ndenumerate[str_]: ... + @overload + def __new__(cls, arr: _NestedSequence[bytes]) -> ndenumerate[bytes_]: ... + @overload + def __new__(cls, arr: _NestedSequence[bool]) -> ndenumerate[bool_]: ... + @overload + def __new__(cls, arr: _NestedSequence[int]) -> ndenumerate[int_]: ... + @overload + def __new__(cls, arr: _NestedSequence[float]) -> ndenumerate[float_]: ... + @overload + def __new__(cls, arr: _NestedSequence[complex]) -> ndenumerate[complex_]: ... + @overload + def __new__(cls, arr: _RecursiveSequence) -> ndenumerate[Any]: ... + def __next__(self: ndenumerate[_ScalarType]) -> Tuple[_Shape, _ScalarType]: ... + def __iter__(self: _T) -> _T: ... + +class ndindex: + def __init__(self, *shape: SupportsIndex) -> None: ... + def __iter__(self: _T) -> _T: ... + def __next__(self) -> _Shape: ... + +class DataSource: + def __init__( + self, + destpath: Union[None, str, os.PathLike[str]] = ..., + ) -> None: ... + def __del__(self) -> None: ... + def abspath(self, path: str) -> str: ... + def exists(self, path: str) -> bool: ... + + # Whether the file-object is opened in string or bytes mode (by default) + # depends on the file-extension of `path` + def open( + self, + path: str, + mode: str = ..., + encoding: Optional[str] = ..., + newline: Optional[str] = ..., + ) -> IO[Any]: ... + +# TODO: The type of each `__next__` and `iters` return-type depends +# on the length and dtype of `args`; we can't describe this behavior yet +# as we lack variadics (PEP 646). +class broadcast: + def __new__(cls, *args: ArrayLike) -> broadcast: ... + @property + def index(self) -> int: ... + @property + def iters(self) -> Tuple[flatiter[Any], ...]: ... + @property + def nd(self) -> int: ... + @property + def ndim(self) -> int: ... + @property + def numiter(self) -> int: ... + @property + def shape(self) -> _Shape: ... + @property + def size(self) -> int: ... + def __next__(self) -> Tuple[Any, ...]: ... + def __iter__(self: _T) -> _T: ... + def reset(self) -> None: ... diff --git a/numpy/_globals.py b/numpy/_globals.py index 9f44c7729668..0b715c870870 100644 --- a/numpy/_globals.py +++ b/numpy/_globals.py @@ -58,14 +58,26 @@ class _NoValueType: """Special keyword value. The instance of this class may be used as the default value assigned to a - deprecated keyword in order to check if it has been given a user defined - value. + keyword if no other obvious default (e.g., `None`) is suitable, + + Common reasons for using this keyword are: + + - A new keyword is added to a function, and that function forwards its + inputs to another function or method which can be defined outside of + NumPy. For example, ``np.std(x)`` calls ``x.std``, so when a ``keepdims`` + keyword was added that could only be forwarded if the user explicitly + specified ``keepdims``; downstream array libraries may not have added + the same keyword, so adding ``x.std(..., keepdims=keepdims)`` + unconditionally could have broken previously working code. + - A keyword is being deprecated, and a deprecation warning must only be + emitted when the keyword is used. + """ __instance = None def __new__(cls): # ensure that only one instance exists if not cls.__instance: - cls.__instance = super(_NoValueType, cls).__new__(cls) + cls.__instance = super().__new__(cls) return cls.__instance # needed for python 2 to preserve identity through a pickle diff --git a/numpy/_pytesttester.py b/numpy/_pytesttester.py index 813e069a4702..acfaa1ca54a1 100644 --- a/numpy/_pytesttester.py +++ b/numpy/_pytesttester.py @@ -35,25 +35,13 @@ def _show_numpy_info(): - from numpy.core._multiarray_umath import ( - __cpu_features__, __cpu_baseline__, __cpu_dispatch__ - ) import numpy as np print("NumPy version %s" % np.__version__) relaxed_strides = np.ones((10, 1), order="C").flags.f_contiguous print("NumPy relaxed strides checking option:", relaxed_strides) - - if len(__cpu_baseline__) == 0 and len(__cpu_dispatch__) == 0: - enabled_features = "nothing enabled" - else: - enabled_features = ' '.join(__cpu_baseline__) - for feature in __cpu_dispatch__: - if __cpu_features__[feature]: - enabled_features += " %s*" % feature - else: - enabled_features += " %s?" % feature - print("NumPy CPU features:", enabled_features) + info = np.lib.utils._opt_info() + print("NumPy CPU features: ", (info if info else 'nothing enabled')) diff --git a/numpy/_version.py b/numpy/_version.py new file mode 100644 index 000000000000..dcc0c5573521 --- /dev/null +++ b/numpy/_version.py @@ -0,0 +1,524 @@ +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "v" + cfg.parentdir_prefix = "numpy-" + cfg.versionfile_source = "numpy/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip().decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "main". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty=", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post0.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post0.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post0.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/numpy/char.pyi b/numpy/char.pyi index 0e7342c0bf3b..4904aa27a3e0 100644 --- a/numpy/char.pyi +++ b/numpy/char.pyi @@ -1,53 +1,59 @@ -from typing import Any +from typing import Any, List -equal: Any -not_equal: Any -greater_equal: Any -less_equal: Any -greater: Any -less: Any -str_len: Any -add: Any -multiply: Any -mod: Any -capitalize: Any -center: Any -count: Any -decode: Any -encode: Any -endswith: Any -expandtabs: Any -find: Any -index: Any -isalnum: Any -isalpha: Any -isdigit: Any -islower: Any -isspace: Any -istitle: Any -isupper: Any -join: Any -ljust: Any -lower: Any -lstrip: Any -partition: Any -replace: Any -rfind: Any -rindex: Any -rjust: Any -rpartition: Any -rsplit: Any -rstrip: Any -split: Any -splitlines: Any -startswith: Any -strip: Any -swapcase: Any -title: Any -translate: Any -upper: Any -zfill: Any -isnumeric: Any -isdecimal: Any -array: Any -asarray: Any +from numpy import ( + chararray as chararray, +) + +__all__: List[str] + +def equal(x1, x2): ... +def not_equal(x1, x2): ... +def greater_equal(x1, x2): ... +def less_equal(x1, x2): ... +def greater(x1, x2): ... +def less(x1, x2): ... +def str_len(a): ... +def add(x1, x2): ... +def multiply(a, i): ... +def mod(a, values): ... +def capitalize(a): ... +def center(a, width, fillchar=...): ... +def count(a, sub, start=..., end=...): ... +def decode(a, encoding=..., errors=...): ... +def encode(a, encoding=..., errors=...): ... +def endswith(a, suffix, start=..., end=...): ... +def expandtabs(a, tabsize=...): ... +def find(a, sub, start=..., end=...): ... +def index(a, sub, start=..., end=...): ... +def isalnum(a): ... +def isalpha(a): ... +def isdigit(a): ... +def islower(a): ... +def isspace(a): ... +def istitle(a): ... +def isupper(a): ... +def join(sep, seq): ... +def ljust(a, width, fillchar=...): ... +def lower(a): ... +def lstrip(a, chars=...): ... +def partition(a, sep): ... +def replace(a, old, new, count=...): ... +def rfind(a, sub, start=..., end=...): ... +def rindex(a, sub, start=..., end=...): ... +def rjust(a, width, fillchar=...): ... +def rpartition(a, sep): ... +def rsplit(a, sep=..., maxsplit=...): ... +def rstrip(a, chars=...): ... +def split(a, sep=..., maxsplit=...): ... +def splitlines(a, keepends=...): ... +def startswith(a, prefix, start=..., end=...): ... +def strip(a, chars=...): ... +def swapcase(a): ... +def title(a): ... +def translate(a, table, deletechars=...): ... +def upper(a): ... +def zfill(a, width): ... +def isnumeric(a): ... +def isdecimal(a): ... +def array(obj, itemsize=..., copy=..., unicode=..., order=...): ... +def asarray(obj, itemsize=..., unicode=..., order=...): ... diff --git a/numpy/compat/py3k.py b/numpy/compat/py3k.py index f36aaca170ff..e1e236d92306 100644 --- a/numpy/compat/py3k.py +++ b/numpy/compat/py3k.py @@ -94,6 +94,9 @@ class contextlib_nullcontext: cm = optional_cm if condition else nullcontext() with cm: # Perform operation, using optional_cm if condition is True + + .. note:: + Prefer using `contextlib.nullcontext` instead of this context manager. """ def __init__(self, enter_result=None): diff --git a/numpy/core/__init__.py b/numpy/core/__init__.py index e8d3a381b602..dad9293e1a19 100644 --- a/numpy/core/__init__.py +++ b/numpy/core/__init__.py @@ -75,7 +75,7 @@ from .fromnumeric import * from . import defchararray as char from . import records as rec -from .records import * +from .records import record, recarray, format_parser from .memmap import * from .defchararray import chararray from . import function_base @@ -106,7 +106,7 @@ __all__ = ['char', 'rec', 'memmap'] __all__ += numeric.__all__ __all__ += fromnumeric.__all__ -__all__ += rec.__all__ +__all__ += ['record', 'recarray', 'format_parser'] __all__ += ['chararray'] __all__ += function_base.__all__ __all__ += machar.__all__ @@ -125,6 +125,7 @@ def _ufunc_reconstruct(module, name): mod = __import__(module, fromlist=[name]) return getattr(mod, name) + def _ufunc_reduce(func): # Report the `__name__`. pickle will try to find the module. Note that # pickle supports for this `__name__` to be a `__qualname__`. It may @@ -134,12 +135,31 @@ def _ufunc_reduce(func): return func.__name__ +def _DType_reconstruct(scalar_type): + # This is a work-around to pickle type(np.dtype(np.float64)), etc. + # and it should eventually be replaced with a better solution, e.g. when + # DTypes become HeapTypes. + return type(dtype(scalar_type)) + + +def _DType_reduce(DType): + # To pickle a DType without having to add top-level names, pickle the + # scalar type for now (and assume that reconstruction will be possible). + if DType is dtype: + return "dtype" # must pickle `np.dtype` as a singleton. + scalar_type = DType.type # pickle the scalar type for reconstruction + return _DType_reconstruct, (scalar_type,) + + import copyreg copyreg.pickle(ufunc, _ufunc_reduce) -# Unclutter namespace (must keep _ufunc_reconstruct for unpickling) +copyreg.pickle(type(dtype), _DType_reduce, _DType_reconstruct) + +# Unclutter namespace (must keep _*_reconstruct for unpickling) del copyreg del _ufunc_reduce +del _DType_reduce from numpy._pytesttester import PytestTester test = PytestTester(__name__) diff --git a/numpy/core/__init__.pyi b/numpy/core/__init__.pyi index e69de29bb2d1..4c7a42bf3db4 100644 --- a/numpy/core/__init__.pyi +++ b/numpy/core/__init__.pyi @@ -0,0 +1,2 @@ +# NOTE: The `np.core` namespace is deliberately kept empty due to it +# being private (despite the lack of leading underscore) diff --git a/numpy/core/_add_newdocs.py b/numpy/core/_add_newdocs.py index c9968f122942..b8f0ee907b4b 100644 --- a/numpy/core/_add_newdocs.py +++ b/numpy/core/_add_newdocs.py @@ -377,7 +377,7 @@ ... while not it.finished: ... it[0] = lamdaexpr(*it[1:]) ... it.iternext() - ... return it.operands[0] + ... return it.operands[0] >>> a = np.arange(5) >>> b = np.ones(5) @@ -821,7 +821,7 @@ ===== ========= =================================================== When ``copy=False`` and a copy is made for other reasons, the result is - the same as if ``copy=True``, with some exceptions for `A`, see the + the same as if ``copy=True``, with some exceptions for 'A', see the Notes section. The default order is 'K'. subok : bool, optional If True, then sub-classes will be passed-through, otherwise @@ -904,6 +904,242 @@ array_function_like_doc, )) +add_newdoc('numpy.core.multiarray', 'asarray', + """ + asarray(a, dtype=None, order=None, *, like=None) + + Convert the input to an array. + + Parameters + ---------- + a : array_like + Input data, in any form that can be converted to an array. This + includes lists, lists of tuples, tuples, tuples of tuples, tuples + of lists and ndarrays. + dtype : data-type, optional + By default, the data-type is inferred from the input data. + order : {'C', 'F', 'A', 'K'}, optional + Memory layout. 'A' and 'K' depend on the order of input array a. + 'C' row-major (C-style), + 'F' column-major (Fortran-style) memory representation. + 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise + 'K' (keep) preserve input order + Defaults to 'C'. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 + + Returns + ------- + out : ndarray + Array interpretation of `a`. No copy is performed if the input + is already an ndarray with matching dtype and order. If `a` is a + subclass of ndarray, a base class ndarray is returned. + + See Also + -------- + asanyarray : Similar function which passes through subclasses. + ascontiguousarray : Convert input to a contiguous array. + asfarray : Convert input to a floating point ndarray. + asfortranarray : Convert input to an ndarray with column-major + memory order. + asarray_chkfinite : Similar function which checks input for NaNs and Infs. + fromiter : Create an array from an iterator. + fromfunction : Construct an array by executing a function on grid + positions. + + Examples + -------- + Convert a list into an array: + + >>> a = [1, 2] + >>> np.asarray(a) + array([1, 2]) + + Existing arrays are not copied: + + >>> a = np.array([1, 2]) + >>> np.asarray(a) is a + True + + If `dtype` is set, array is copied only if dtype does not match: + + >>> a = np.array([1, 2], dtype=np.float32) + >>> np.asarray(a, dtype=np.float32) is a + True + >>> np.asarray(a, dtype=np.float64) is a + False + + Contrary to `asanyarray`, ndarray subclasses are not passed through: + + >>> issubclass(np.recarray, np.ndarray) + True + >>> a = np.array([(1.0, 2), (3.0, 4)], dtype='f4,i4').view(np.recarray) + >>> np.asarray(a) is a + False + >>> np.asanyarray(a) is a + True + + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) + +add_newdoc('numpy.core.multiarray', 'asanyarray', + """ + asanyarray(a, dtype=None, order=None, *, like=None) + + Convert the input to an ndarray, but pass ndarray subclasses through. + + Parameters + ---------- + a : array_like + Input data, in any form that can be converted to an array. This + includes scalars, lists, lists of tuples, tuples, tuples of tuples, + tuples of lists, and ndarrays. + dtype : data-type, optional + By default, the data-type is inferred from the input data. + order : {'C', 'F', 'A', 'K'}, optional + Memory layout. 'A' and 'K' depend on the order of input array a. + 'C' row-major (C-style), + 'F' column-major (Fortran-style) memory representation. + 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise + 'K' (keep) preserve input order + Defaults to 'C'. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 + + Returns + ------- + out : ndarray or an ndarray subclass + Array interpretation of `a`. If `a` is an ndarray or a subclass + of ndarray, it is returned as-is and no copy is performed. + + See Also + -------- + asarray : Similar function which always returns ndarrays. + ascontiguousarray : Convert input to a contiguous array. + asfarray : Convert input to a floating point ndarray. + asfortranarray : Convert input to an ndarray with column-major + memory order. + asarray_chkfinite : Similar function which checks input for NaNs and + Infs. + fromiter : Create an array from an iterator. + fromfunction : Construct an array by executing a function on grid + positions. + + Examples + -------- + Convert a list into an array: + + >>> a = [1, 2] + >>> np.asanyarray(a) + array([1, 2]) + + Instances of `ndarray` subclasses are passed through as-is: + + >>> a = np.array([(1.0, 2), (3.0, 4)], dtype='f4,i4').view(np.recarray) + >>> np.asanyarray(a) is a + True + + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) + +add_newdoc('numpy.core.multiarray', 'ascontiguousarray', + """ + ascontiguousarray(a, dtype=None, *, like=None) + + Return a contiguous array (ndim >= 1) in memory (C order). + + Parameters + ---------- + a : array_like + Input array. + dtype : str or dtype object, optional + Data-type of returned array. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 + + Returns + ------- + out : ndarray + Contiguous array of same shape and content as `a`, with type `dtype` + if specified. + + See Also + -------- + asfortranarray : Convert input to an ndarray with column-major + memory order. + require : Return an ndarray that satisfies requirements. + ndarray.flags : Information about the memory layout of the array. + + Examples + -------- + >>> x = np.arange(6).reshape(2,3) + >>> np.ascontiguousarray(x, dtype=np.float32) + array([[0., 1., 2.], + [3., 4., 5.]], dtype=float32) + >>> x.flags['C_CONTIGUOUS'] + True + + Note: This function returns an array with at least one-dimension (1-d) + so it will not preserve 0-d arrays. + + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) + +add_newdoc('numpy.core.multiarray', 'asfortranarray', + """ + asfortranarray(a, dtype=None, *, like=None) + + Return an array (ndim >= 1) laid out in Fortran order in memory. + + Parameters + ---------- + a : array_like + Input array. + dtype : str or dtype object, optional + By default, the data-type is inferred from the input data. + ${ARRAY_FUNCTION_LIKE} + + .. versionadded:: 1.20.0 + + Returns + ------- + out : ndarray + The input `a` in Fortran, or column-major, order. + + See Also + -------- + ascontiguousarray : Convert input to a contiguous (C order) array. + asanyarray : Convert input to an ndarray with either row or + column-major memory order. + require : Return an ndarray that satisfies requirements. + ndarray.flags : Information about the memory layout of the array. + + Examples + -------- + >>> x = np.arange(6).reshape(2,3) + >>> y = np.asfortranarray(x) + >>> x.flags['F_CONTIGUOUS'] + False + >>> y.flags['F_CONTIGUOUS'] + True + + Note: This function returns an array with at least one-dimension (1-d) + so it will not preserve 0-d arrays. + + """.replace( + "${ARRAY_FUNCTION_LIKE}", + array_function_like_doc, + )) + add_newdoc('numpy.core.multiarray', 'empty', """ empty(shape, dtype=float, order='C', *, like=None) @@ -1147,13 +1383,13 @@ add_newdoc('numpy.core.multiarray', 'fromiter', """ - fromiter(iterable, dtype, count=-1, *, like=None) + fromiter(iter, dtype, count=-1, *, like=None) Create a new 1-dimensional array from an iterable object. Parameters ---------- - iterable : iterable object + iter : iterable object An iterable object providing data for the array. dtype : data-type The data-type of the returned array. @@ -1940,6 +2176,8 @@ empty : Create an array, but leave its allocated memory unchanged (i.e., it contains "garbage"). dtype : Create a data-type. + numpy.typing.NDArray : A :term:`generic ` version + of ndarray. Notes ----- @@ -2570,7 +2808,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('all', """ - a.all(axis=None, out=None, keepdims=False) + a.all(axis=None, out=None, keepdims=False, *, where=True) Returns True if all elements evaluate to True. @@ -2585,7 +2823,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('any', """ - a.any(axis=None, out=None, keepdims=False) + a.any(axis=None, out=None, keepdims=False, *, where=True) Returns True if any of the elements of `a` evaluate to True. @@ -2877,14 +3115,20 @@ 'F' means F-order, 'A' means 'F' if `a` is Fortran contiguous, 'C' otherwise. 'K' means match the layout of `a` as closely as possible. (Note that this function and :func:`numpy.copy` are very - similar, but have different default values for their order= - arguments.) + similar but have different default values for their order= + arguments, and this function always passes sub-classes through.) See also -------- - numpy.copy + numpy.copy : Similar function with different default behavior numpy.copyto + Notes + ----- + This function is the preferred method for creating an array copy. The + function :func:`numpy.copy` is similar, but it defaults to using order 'K', + and will not pass sub-classes through by default. + Examples -------- >>> x = np.array([[1,2,3],[4,5,6]], order='F') @@ -3242,7 +3486,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('mean', """ - a.mean(axis=None, dtype=None, out=None, keepdims=False) + a.mean(axis=None, dtype=None, out=None, keepdims=False, *, where=True) Returns the average of the array elements along given axis. @@ -3694,7 +3938,7 @@ actual implementation will vary with datatype. The 'mergesort' option is retained for backwards compatibility. - .. versionchanged:: 1.15.0. + .. versionchanged:: 1.15.0 The 'stable' option was added. order : str or list of str, optional @@ -3813,7 +4057,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('std', """ - a.std(axis=None, dtype=None, out=None, ddof=0, keepdims=False) + a.std(axis=None, dtype=None, out=None, ddof=0, keepdims=False, *, where=True) Returns the standard deviation of the array elements along given axis. @@ -4076,6 +4320,7 @@ See Also -------- + transpose : Equivalent function ndarray.T : Array property returning the array transposed. ndarray.reshape : Give a new shape to an array without changing its data. @@ -4100,7 +4345,7 @@ add_newdoc('numpy.core.multiarray', 'ndarray', ('var', """ - a.var(axis=None, dtype=None, out=None, ddof=0, keepdims=False) + a.var(axis=None, dtype=None, out=None, ddof=0, keepdims=False, *, where=True) Returns the variance of the array elements, along given axis. @@ -5070,7 +5315,7 @@ r = empty(len(A),len(B)) for i in range(len(A)): for j in range(len(B)): - r[i,j] = op(A[i], B[j]) # op = ufunc in question + r[i,j] = op(A[i], B[j]) # op = ufunc in question Parameters ---------- @@ -5080,6 +5325,7 @@ Second array kwargs : any Arguments to pass on to the ufunc. Typically `dtype` or `out`. + See `ufunc` for a comprehensive overview of all available arguments. Returns ------- @@ -5571,7 +5817,7 @@ >>> (arr + arr).dtype.metadata mappingproxy({'key': 'value'}) - But if the arrays have different dtype metadata, the metadata may be + But if the arrays have different dtype metadata, the metadata may be dropped: >>> dt2 = np.dtype(float, metadata={"key2": "value2"}) diff --git a/numpy/core/_add_newdocs_scalars.py b/numpy/core/_add_newdocs_scalars.py index b9b151224e61..602b1db6e64a 100644 --- a/numpy/core/_add_newdocs_scalars.py +++ b/numpy/core/_add_newdocs_scalars.py @@ -6,6 +6,7 @@ from numpy.core import dtype from numpy.core import numerictypes as _numerictypes from numpy.core.function_base import add_newdoc +import platform ############################################################################## # @@ -49,6 +50,8 @@ def type_aliases_gen(): ]) + + def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): # note: `:field: value` is rST syntax which renders as field lists. o = getattr(_numerictypes, obj) @@ -56,7 +59,7 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): character_code = dtype(o).char canonical_name_doc = "" if obj == o.__name__ else ":Canonical name: `numpy.{}`\n ".format(obj) alias_doc = ''.join(":Alias: `numpy.{}`\n ".format(alias) for alias in fixed_aliases) - alias_doc += ''.join(":Alias on this platform: `numpy.{}`: {}.\n ".format(alias, doc) + alias_doc += ''.join(":Alias on this platform ({} {}): `numpy.{}`: {}.\n ".format(platform.system(), platform.machine(), alias, doc) for (alias_type, alias, doc) in possible_aliases if alias_type is o) docstring = """ {doc} @@ -215,10 +218,15 @@ def add_newdoc_for_scalar_type(obj, fixed_aliases, doc): add_newdoc_for_scalar_type('datetime64', [], """ - A datetime stored as a 64-bit integer, counting from ``1970-01-01T00:00:00``. + If created from a 64-bit integer, it represents an offset from + ``1970-01-01T00:00:00``. + If created from string, the string can be in ISO 8601 date + or datetime format. >>> np.datetime64(10, 'Y') numpy.datetime64('1980') + >>> np.datetime64('1980', 'Y') + numpy.datetime64('1980') >>> np.datetime64(10, 'D') numpy.datetime64('1970-01-11') diff --git a/numpy/core/_asarray.py b/numpy/core/_asarray.py index a406308f3dd3..ecb4e7c39d0c 100644 --- a/numpy/core/_asarray.py +++ b/numpy/core/_asarray.py @@ -8,283 +8,12 @@ set_array_function_like_doc, set_module, ) -from .multiarray import array +from .multiarray import array, asanyarray -__all__ = [ - "asarray", "asanyarray", "ascontiguousarray", "asfortranarray", "require", -] +__all__ = ["require"] -def _asarray_dispatcher(a, dtype=None, order=None, *, like=None): - return (like,) - - -@set_array_function_like_doc -@set_module('numpy') -def asarray(a, dtype=None, order=None, *, like=None): - """Convert the input to an array. - - Parameters - ---------- - a : array_like - Input data, in any form that can be converted to an array. This - includes lists, lists of tuples, tuples, tuples of tuples, tuples - of lists and ndarrays. - dtype : data-type, optional - By default, the data-type is inferred from the input data. - order : {'C', 'F', 'A', 'K'}, optional - Memory layout. 'A' and 'K' depend on the order of input array a. - 'C' row-major (C-style), - 'F' column-major (Fortran-style) memory representation. - 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise - 'K' (keep) preserve input order - Defaults to 'C'. - ${ARRAY_FUNCTION_LIKE} - - .. versionadded:: 1.20.0 - - Returns - ------- - out : ndarray - Array interpretation of `a`. No copy is performed if the input - is already an ndarray with matching dtype and order. If `a` is a - subclass of ndarray, a base class ndarray is returned. - - See Also - -------- - asanyarray : Similar function which passes through subclasses. - ascontiguousarray : Convert input to a contiguous array. - asfarray : Convert input to a floating point ndarray. - asfortranarray : Convert input to an ndarray with column-major - memory order. - asarray_chkfinite : Similar function which checks input for NaNs and Infs. - fromiter : Create an array from an iterator. - fromfunction : Construct an array by executing a function on grid - positions. - - Examples - -------- - Convert a list into an array: - - >>> a = [1, 2] - >>> np.asarray(a) - array([1, 2]) - - Existing arrays are not copied: - - >>> a = np.array([1, 2]) - >>> np.asarray(a) is a - True - - If `dtype` is set, array is copied only if dtype does not match: - - >>> a = np.array([1, 2], dtype=np.float32) - >>> np.asarray(a, dtype=np.float32) is a - True - >>> np.asarray(a, dtype=np.float64) is a - False - - Contrary to `asanyarray`, ndarray subclasses are not passed through: - - >>> issubclass(np.recarray, np.ndarray) - True - >>> a = np.array([(1.0, 2), (3.0, 4)], dtype='f4,i4').view(np.recarray) - >>> np.asarray(a) is a - False - >>> np.asanyarray(a) is a - True - - """ - if like is not None: - return _asarray_with_like(a, dtype=dtype, order=order, like=like) - - return array(a, dtype, copy=False, order=order) - - -_asarray_with_like = array_function_dispatch( - _asarray_dispatcher -)(asarray) - - -@set_array_function_like_doc -@set_module('numpy') -def asanyarray(a, dtype=None, order=None, *, like=None): - """Convert the input to an ndarray, but pass ndarray subclasses through. - - Parameters - ---------- - a : array_like - Input data, in any form that can be converted to an array. This - includes scalars, lists, lists of tuples, tuples, tuples of tuples, - tuples of lists, and ndarrays. - dtype : data-type, optional - By default, the data-type is inferred from the input data. - order : {'C', 'F', 'A', 'K'}, optional - Memory layout. 'A' and 'K' depend on the order of input array a. - 'C' row-major (C-style), - 'F' column-major (Fortran-style) memory representation. - 'A' (any) means 'F' if `a` is Fortran contiguous, 'C' otherwise - 'K' (keep) preserve input order - Defaults to 'C'. - ${ARRAY_FUNCTION_LIKE} - - .. versionadded:: 1.20.0 - - Returns - ------- - out : ndarray or an ndarray subclass - Array interpretation of `a`. If `a` is an ndarray or a subclass - of ndarray, it is returned as-is and no copy is performed. - - See Also - -------- - asarray : Similar function which always returns ndarrays. - ascontiguousarray : Convert input to a contiguous array. - asfarray : Convert input to a floating point ndarray. - asfortranarray : Convert input to an ndarray with column-major - memory order. - asarray_chkfinite : Similar function which checks input for NaNs and - Infs. - fromiter : Create an array from an iterator. - fromfunction : Construct an array by executing a function on grid - positions. - - Examples - -------- - Convert a list into an array: - - >>> a = [1, 2] - >>> np.asanyarray(a) - array([1, 2]) - - Instances of `ndarray` subclasses are passed through as-is: - - >>> a = np.array([(1.0, 2), (3.0, 4)], dtype='f4,i4').view(np.recarray) - >>> np.asanyarray(a) is a - True - - """ - if like is not None: - return _asanyarray_with_like(a, dtype=dtype, order=order, like=like) - - return array(a, dtype, copy=False, order=order, subok=True) - - -_asanyarray_with_like = array_function_dispatch( - _asarray_dispatcher -)(asanyarray) - - -def _asarray_contiguous_fortran_dispatcher(a, dtype=None, *, like=None): - return (like,) - - -@set_array_function_like_doc -@set_module('numpy') -def ascontiguousarray(a, dtype=None, *, like=None): - """ - Return a contiguous array (ndim >= 1) in memory (C order). - - Parameters - ---------- - a : array_like - Input array. - dtype : str or dtype object, optional - Data-type of returned array. - ${ARRAY_FUNCTION_LIKE} - - .. versionadded:: 1.20.0 - - Returns - ------- - out : ndarray - Contiguous array of same shape and content as `a`, with type `dtype` - if specified. - - See Also - -------- - asfortranarray : Convert input to an ndarray with column-major - memory order. - require : Return an ndarray that satisfies requirements. - ndarray.flags : Information about the memory layout of the array. - - Examples - -------- - >>> x = np.arange(6).reshape(2,3) - >>> np.ascontiguousarray(x, dtype=np.float32) - array([[0., 1., 2.], - [3., 4., 5.]], dtype=float32) - >>> x.flags['C_CONTIGUOUS'] - True - - Note: This function returns an array with at least one-dimension (1-d) - so it will not preserve 0-d arrays. - - """ - if like is not None: - return _ascontiguousarray_with_like(a, dtype=dtype, like=like) - - return array(a, dtype, copy=False, order='C', ndmin=1) - - -_ascontiguousarray_with_like = array_function_dispatch( - _asarray_contiguous_fortran_dispatcher -)(ascontiguousarray) - - -@set_array_function_like_doc -@set_module('numpy') -def asfortranarray(a, dtype=None, *, like=None): - """ - Return an array (ndim >= 1) laid out in Fortran order in memory. - - Parameters - ---------- - a : array_like - Input array. - dtype : str or dtype object, optional - By default, the data-type is inferred from the input data. - ${ARRAY_FUNCTION_LIKE} - - .. versionadded:: 1.20.0 - - Returns - ------- - out : ndarray - The input `a` in Fortran, or column-major, order. - - See Also - -------- - ascontiguousarray : Convert input to a contiguous (C order) array. - asanyarray : Convert input to an ndarray with either row or - column-major memory order. - require : Return an ndarray that satisfies requirements. - ndarray.flags : Information about the memory layout of the array. - - Examples - -------- - >>> x = np.arange(6).reshape(2,3) - >>> y = np.asfortranarray(x) - >>> x.flags['F_CONTIGUOUS'] - False - >>> y.flags['F_CONTIGUOUS'] - True - - Note: This function returns an array with at least one-dimension (1-d) - so it will not preserve 0-d arrays. - - """ - if like is not None: - return _asfortranarray_with_like(a, dtype=dtype, like=like) - - return array(a, dtype, copy=False, order='F', ndmin=1) - - -_asfortranarray_with_like = array_function_dispatch( - _asarray_contiguous_fortran_dispatcher -)(asfortranarray) - def _require_dispatcher(a, dtype=None, requirements=None, *, like=None): return (like,) diff --git a/numpy/core/_asarray.pyi b/numpy/core/_asarray.pyi index 8c200ba22ae2..ee21fc0f1492 100644 --- a/numpy/core/_asarray.pyi +++ b/numpy/core/_asarray.pyi @@ -11,6 +11,10 @@ else: _ArrayType = TypeVar("_ArrayType", bound=ndarray) +# TODO: The following functions are now defined in C, so should be defined +# in a (not yet existing) `multiarray.pyi`. +# (with the exception of `require`) + def asarray( a: object, dtype: DTypeLike = ..., diff --git a/numpy/core/_exceptions.py b/numpy/core/_exceptions.py index 5e17ed3b21e0..77aa2f6e1926 100644 --- a/numpy/core/_exceptions.py +++ b/numpy/core/_exceptions.py @@ -135,7 +135,7 @@ def __init__(self, axis, ndim=None, msg_prefix=None): if msg_prefix is not None: msg = "{}: {}".format(msg_prefix, msg) - super(AxisError, self).__init__(msg) + super().__init__(msg) @_display_as_base diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 449926f586ac..3b0c464674b6 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -8,6 +8,7 @@ import re import sys import platform +import warnings from .multiarray import dtype, array, ndarray try: @@ -350,11 +351,45 @@ def _as_parameter_(self): """ return self.data_as(ctypes.c_void_p) - # kept for compatibility - get_data = data.fget - get_shape = shape.fget - get_strides = strides.fget - get_as_parameter = _as_parameter_.fget + # Numpy 1.21.0, 2021-05-18 + + def get_data(self): + """Deprecated getter for the `_ctypes.data` property. + + .. deprecated:: 1.21 + """ + warnings.warn('"get_data" is deprecated. Use "data" instead', + DeprecationWarning, stacklevel=2) + return self.data + + def get_shape(self): + """Deprecated getter for the `_ctypes.shape` property. + + .. deprecated:: 1.21 + """ + warnings.warn('"get_shape" is deprecated. Use "shape" instead', + DeprecationWarning, stacklevel=2) + return self.shape + + def get_strides(self): + """Deprecated getter for the `_ctypes.strides` property. + + .. deprecated:: 1.21 + """ + warnings.warn('"get_strides" is deprecated. Use "strides" instead', + DeprecationWarning, stacklevel=2) + return self.strides + + def get_as_parameter(self): + """Deprecated getter for the `_ctypes._as_parameter_` property. + + .. deprecated:: 1.21 + """ + warnings.warn( + '"get_as_parameter" is deprecated. Use "_as_parameter_" instead', + DeprecationWarning, stacklevel=2, + ) + return self._as_parameter_ def _newnames(datatype, order): @@ -808,11 +843,13 @@ def _ufunc_doc_signature_formatter(ufunc): ", order='K'" ", dtype=None" ", subok=True" - "[, signature" - ", extobj]" ) + + # NOTE: gufuncs may or may not support the `axis` parameter if ufunc.signature is None: - kwargs = ", where=True" + kwargs + kwargs = f", where=True{kwargs}[, signature, extobj]" + else: + kwargs += "[, signature, extobj, axes, axis]" # join all the parts together return '{name}({in_args}{out_args}, *{kwargs})'.format( diff --git a/numpy/core/_internal.pyi b/numpy/core/_internal.pyi index 1b3889e51cfe..1ef1c9fa1564 100644 --- a/numpy/core/_internal.pyi +++ b/numpy/core/_internal.pyi @@ -1,18 +1,35 @@ -from typing import Any +from typing import Any, TypeVar, Type, overload, Optional, Generic +import ctypes as ct -# TODO: add better annotations when ctypes is stubbed out +from numpy import ndarray -class _ctypes: +_CastT = TypeVar("_CastT", bound=ct._CanCastTo) # Copied from `ctypes.cast` +_CT = TypeVar("_CT", bound=ct._CData) +_PT = TypeVar("_PT", bound=Optional[int]) + +# TODO: Let the likes of `shape_as` and `strides_as` return `None` +# for 0D arrays once we've got shape-support + +class _ctypes(Generic[_PT]): + @overload + def __new__(cls, array: ndarray[Any, Any], ptr: None = ...) -> _ctypes[None]: ... + @overload + def __new__(cls, array: ndarray[Any, Any], ptr: _PT) -> _ctypes[_PT]: ... + + # NOTE: In practice `shape` and `strides` return one of the concrete + # platform dependant array-types (`c_int`, `c_long` or `c_longlong`) + # corresponding to C's `int_ptr_t`, as determined by `_getintp_ctype` + # TODO: Hook this in to the mypy plugin so that a more appropiate + # `ctypes._SimpleCData[int]` sub-type can be returned @property - def data(self) -> int: ... + def data(self) -> _PT: ... @property - def shape(self) -> Any: ... + def shape(self) -> ct.Array[ct.c_int64]: ... @property - def strides(self) -> Any: ... - def data_as(self, obj: Any) -> Any: ... - def shape_as(self, obj: Any) -> Any: ... - def strides_as(self, obj: Any) -> Any: ... - def get_data(self) -> int: ... - def get_shape(self) -> Any: ... - def get_strides(self) -> Any: ... - def get_as_parameter(self) -> Any: ... + def strides(self) -> ct.Array[ct.c_int64]: ... + @property + def _as_parameter_(self) -> ct.c_void_p: ... + + def data_as(self, obj: Type[_CastT]) -> _CastT: ... + def shape_as(self, obj: Type[_CT]) -> ct.Array[_CT]: ... + def strides_as(self, obj: Type[_CT]) -> ct.Array[_CT]: ... diff --git a/numpy/core/_methods.py b/numpy/core/_methods.py index 75fd32ec8d66..e475b94dfb4e 100644 --- a/numpy/core/_methods.py +++ b/numpy/core/_methods.py @@ -4,14 +4,15 @@ """ import warnings +from contextlib import nullcontext from numpy.core import multiarray as mu from numpy.core import umath as um -from numpy.core._asarray import asanyarray +from numpy.core.multiarray import asanyarray from numpy.core import numerictypes as nt from numpy.core import _exceptions from numpy._globals import _NoValue -from numpy.compat import pickle, os_fspath, contextlib_nullcontext +from numpy.compat import pickle, os_fspath # save those O(100) nanoseconds! umr_maximum = um.maximum.reduce @@ -51,9 +52,15 @@ def _prod(a, axis=None, dtype=None, out=None, keepdims=False, return umr_prod(a, axis, dtype, out, keepdims, initial, where) def _any(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True): + # Parsing keyword arguments is currently fairly slow, so avoid it for now + if where is True: + return umr_any(a, axis, dtype, out, keepdims) return umr_any(a, axis, dtype, out, keepdims, where=where) def _all(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True): + # Parsing keyword arguments is currently fairly slow, so avoid it for now + if where is True: + return umr_all(a, axis, dtype, out, keepdims) return umr_all(a, axis, dtype, out, keepdims, where=where) def _count_reduce_items(arr, axis, keepdims=False, where=True): @@ -158,7 +165,7 @@ def _mean(a, axis=None, dtype=None, out=None, keepdims=False, *, where=True): is_float16_result = False rcount = _count_reduce_items(arr, axis, keepdims=keepdims, where=where) - if umr_any(rcount == 0, axis=None): + if rcount == 0 if where is True else umr_any(rcount == 0, axis=None): warnings.warn("Mean of empty slice.", RuntimeWarning, stacklevel=2) # Cast bool, unsigned int, and int to float64 by default @@ -191,7 +198,7 @@ def _var(a, axis=None, dtype=None, out=None, ddof=0, keepdims=False, *, rcount = _count_reduce_items(arr, axis, keepdims=keepdims, where=where) # Make this warning show up on top. - if umr_any(ddof >= rcount, axis=None): + if ddof >= rcount if where is True else umr_any(ddof >= rcount, axis=None): warnings.warn("Degrees of freedom <= 0 for slice", RuntimeWarning, stacklevel=2) @@ -273,7 +280,7 @@ def _ptp(a, axis=None, out=None, keepdims=False): def _dump(self, file, protocol=2): if hasattr(file, 'write'): - ctx = contextlib_nullcontext(file) + ctx = nullcontext(file) else: ctx = open(os_fspath(file), "wb") with ctx as f: diff --git a/numpy/core/_type_aliases.py b/numpy/core/_type_aliases.py index de90fd818686..67addef483f6 100644 --- a/numpy/core/_type_aliases.py +++ b/numpy/core/_type_aliases.py @@ -46,7 +46,8 @@ def _bits_of(obj): info = next(v for v in _concrete_typeinfo.values() if v.type is obj) except StopIteration: if obj in _abstract_types.values(): - raise ValueError("Cannot count the bits of an abstract type") + msg = "Cannot count the bits of an abstract type" + raise ValueError(msg) from None # some third-party type - make a best-guess return dtype(obj).itemsize * 8 diff --git a/numpy/core/_ufunc_config.py b/numpy/core/_ufunc_config.py index 454d911cf575..b40e7445ec5b 100644 --- a/numpy/core/_ufunc_config.py +++ b/numpy/core/_ufunc_config.py @@ -98,10 +98,9 @@ def seterr(all=None, divide=None, over=None, under=None, invalid=None): File "", line 1, in FloatingPointError: overflow encountered in short_scalars - >>> from collections import OrderedDict >>> old_settings = np.seterr(all='print') - >>> OrderedDict(np.geterr()) - OrderedDict([('divide', 'print'), ('over', 'print'), ('under', 'print'), ('invalid', 'print')]) + >>> np.geterr() + {'divide': 'print', 'over': 'print', 'under': 'print', 'invalid': 'print'} >>> np.int16(32000) * np.int16(3) 30464 @@ -153,15 +152,14 @@ def geterr(): Examples -------- - >>> from collections import OrderedDict - >>> sorted(np.geterr().items()) - [('divide', 'warn'), ('invalid', 'warn'), ('over', 'warn'), ('under', 'ignore')] + >>> np.geterr() + {'divide': 'warn', 'over': 'warn', 'under': 'ignore', 'invalid': 'warn'} >>> np.arange(3.) / np.arange(3.) array([nan, 1., 1.]) >>> oldsettings = np.seterr(all='warn', over='raise') - >>> OrderedDict(sorted(np.geterr().items())) - OrderedDict([('divide', 'warn'), ('invalid', 'warn'), ('over', 'raise'), ('under', 'warn')]) + >>> np.geterr() + {'divide': 'warn', 'over': 'raise', 'under': 'warn', 'invalid': 'warn'} >>> np.arange(3.) / np.arange(3.) array([nan, 1., 1.]) @@ -270,7 +268,6 @@ def seterrcall(func): >>> saved_handler = np.seterrcall(err_handler) >>> save_err = np.seterr(all='call') - >>> from collections import OrderedDict >>> np.array([1, 2, 3]) / 0.0 Floating point error (divide by zero), with flag 1 @@ -278,8 +275,8 @@ def seterrcall(func): >>> np.seterrcall(saved_handler) - >>> OrderedDict(sorted(np.seterr(**save_err).items())) - OrderedDict([('divide', 'call'), ('invalid', 'call'), ('over', 'call'), ('under', 'call')]) + >>> np.seterr(**save_err) + {'divide': 'call', 'over': 'call', 'under': 'call', 'invalid': 'call'} Log error message: @@ -298,8 +295,8 @@ def seterrcall(func): >>> np.seterrcall(saved_handler) - >>> OrderedDict(sorted(np.seterr(**save_err).items())) - OrderedDict([('divide', 'log'), ('invalid', 'log'), ('over', 'log'), ('under', 'log')]) + >>> np.seterr(**save_err) + {'divide': 'log', 'over': 'log', 'under': 'log', 'invalid': 'log'} """ if func is not None and not isinstance(func, collections.abc.Callable): @@ -402,7 +399,6 @@ class errstate(contextlib.ContextDecorator): Examples -------- - >>> from collections import OrderedDict >>> olderr = np.seterr(all='ignore') # Set error handling to known state. >>> np.arange(3) / 0. @@ -421,8 +417,8 @@ class errstate(contextlib.ContextDecorator): Outside the context the error handling behavior has not changed: - >>> OrderedDict(sorted(np.geterr().items())) - OrderedDict([('divide', 'ignore'), ('invalid', 'ignore'), ('over', 'ignore'), ('under', 'ignore')]) + >>> np.geterr() + {'divide': 'ignore', 'over': 'ignore', 'under': 'ignore', 'invalid': 'ignore'} """ diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index ad15304194a8..f16bcfd39e57 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -41,6 +41,7 @@ from .numerictypes import (longlong, intc, int_, float_, complex_, bool_, flexible) from .overrides import array_function_dispatch, set_module +import operator import warnings import contextlib @@ -78,6 +79,7 @@ def _make_options_dict(precision=None, threshold=None, edgeitems=None, if legacy not in [None, False, '1.13']: warnings.warn("legacy printing option can currently only be '1.13' or " "`False`", stacklevel=3) + if threshold is not None: # forbid the bad threshold arg suggested by stack overflow, gh-12351 if not isinstance(threshold, numbers.Number): @@ -85,6 +87,14 @@ def _make_options_dict(precision=None, threshold=None, edgeitems=None, if np.isnan(threshold): raise ValueError("threshold must be non-NAN, try " "sys.maxsize for untruncated representation") + + if precision is not None: + # forbid the bad precision arg as suggested by issue #18254 + try: + options['precision'] = operator.index(precision) + except TypeError as e: + raise TypeError('precision must be an integer') from e + return options @@ -146,7 +156,6 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, - 'longcomplexfloat' : composed of two 128-bit floats - 'numpystr' : types `numpy.string_` and `numpy.unicode_` - 'object' : `np.object_` arrays - - 'str' : all other strings Other keys that can be used to set a group of types at once are: @@ -154,7 +163,7 @@ def set_printoptions(precision=None, threshold=None, edgeitems=None, - 'int_kind' : sets 'int' - 'float_kind' : sets 'float' and 'longfloat' - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' - - 'str_kind' : sets 'str' and 'numpystr' + - 'str_kind' : sets 'numpystr' floatmode : str, optional Controls the interpretation of the `precision` option for floating-point types. Can take the following values @@ -375,8 +384,7 @@ def _get_formatdict(data, *, precision, floatmode, suppress, sign, legacy, 'timedelta': lambda: TimedeltaFormat(data), 'object': lambda: _object_format, 'void': lambda: str_format, - 'numpystr': lambda: repr_format, - 'str': lambda: str} + 'numpystr': lambda: repr_format} # we need to wrap values in `formatter` in a lambda, so that the interface # is the same as the above values. @@ -398,8 +406,7 @@ def indirect(x): for key in ['complexfloat', 'longcomplexfloat']: formatdict[key] = indirect(formatter['complex_kind']) if 'str_kind' in fkeys: - for key in ['numpystr', 'str']: - formatdict[key] = indirect(formatter['str_kind']) + formatdict['numpystr'] = indirect(formatter['str_kind']) for key in formatdict.keys(): if key in fkeys: formatdict[key] = indirect(formatter[key]) @@ -524,7 +531,7 @@ def array2string(a, max_line_width=None, precision=None, Parameters ---------- - a : array_like + a : ndarray Input array. max_line_width : int, optional Inserts newlines if text is longer than `max_line_width`. @@ -541,7 +548,7 @@ def array2string(a, max_line_width=None, precision=None, separator : str, optional Inserted between elements. prefix : str, optional - suffix: str, optional + suffix : str, optional The length of the prefix and suffix strings are used to respectively align and wrap the output. An array is typically printed as:: @@ -572,7 +579,6 @@ def array2string(a, max_line_width=None, precision=None, - 'longcomplexfloat' : composed of two 128-bit floats - 'void' : type `numpy.void` - 'numpystr' : types `numpy.string_` and `numpy.unicode_` - - 'str' : all other strings Other keys that can be used to set a group of types at once are: @@ -580,7 +586,7 @@ def array2string(a, max_line_width=None, precision=None, - 'int_kind' : sets 'int' - 'float_kind' : sets 'float' and 'longfloat' - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' - - 'str_kind' : sets 'str' and 'numpystr' + - 'str_kind' : sets 'numpystr' threshold : int, optional Total number of array elements which trigger summarization rather than full repr. @@ -908,6 +914,7 @@ def fillFormat(self, data): self.trim = '.' self.exp_size = -1 self.unique = True + self.min_digits = None elif self.exp_format: trim, unique = '.', True if self.floatmode == 'fixed' or self._legacy == '1.13': @@ -921,6 +928,8 @@ def fillFormat(self, data): self.trim = 'k' self.precision = max(len(s) for s in frac_part) + self.min_digits = self.precision + self.unique = unique # for back-compat with np 1.13, use 2 spaces & sign and full prec if self._legacy == '1.13': @@ -930,10 +939,7 @@ def fillFormat(self, data): self.pad_left = max(len(s) for s in int_part) # pad_right is only needed for nan length calculation self.pad_right = self.exp_size + 2 + self.precision - - self.unique = False else: - # first pass printing to determine sizes trim, unique = '.', True if self.floatmode == 'fixed': trim, unique = 'k', False @@ -949,14 +955,14 @@ def fillFormat(self, data): self.pad_left = max(len(s) for s in int_part) self.pad_right = max(len(s) for s in frac_part) self.exp_size = -1 + self.unique = unique if self.floatmode in ['fixed', 'maxprec_equal']: - self.precision = self.pad_right - self.unique = False + self.precision = self.min_digits = self.pad_right self.trim = 'k' else: - self.unique = True self.trim = '.' + self.min_digits = 0 if self._legacy != '1.13': # account for sign = ' ' by adding one to pad_left @@ -985,6 +991,7 @@ def __call__(self, x): if self.exp_format: return dragon4_scientific(x, precision=self.precision, + min_digits=self.min_digits, unique=self.unique, trim=self.trim, sign=self.sign == '+', @@ -993,6 +1000,7 @@ def __call__(self, x): else: return dragon4_positional(x, precision=self.precision, + min_digits=self.min_digits, unique=self.unique, fractional=True, trim=self.trim, @@ -1003,7 +1011,8 @@ def __call__(self, x): @set_module('numpy') def format_float_scientific(x, precision=None, unique=True, trim='k', - sign=False, pad_left=None, exp_digits=None): + sign=False, pad_left=None, exp_digits=None, + min_digits=None): """ Format a floating-point scalar as a decimal string in scientific notation. @@ -1021,11 +1030,12 @@ def format_float_scientific(x, precision=None, unique=True, trim='k', If `True`, use a digit-generation strategy which gives the shortest representation which uniquely identifies the floating-point number from other values of the same type, by judicious rounding. If `precision` - was omitted, print all necessary digits, otherwise digit generation is - cut off after `precision` digits and the remaining value is rounded. + is given fewer digits than necessary can be printed. If `min_digits` + is given more can be printed, in which cases the last digit is rounded + with unbiased rounding. If `False`, digits are generated as if printing an infinite-precision value and stopping after `precision` digits, rounding the remaining - value. + value with unbiased rounding trim : one of 'k', '.', '0', '-', optional Controls post-processing trimming of trailing digits, as follows: @@ -1042,7 +1052,13 @@ def format_float_scientific(x, precision=None, unique=True, trim='k', exp_digits : non-negative integer, optional Pad the exponent with zeros until it contains at least this many digits. If omitted, the exponent will be at least 2 digits. + min_digits : non-negative integer or None, optional + Minimum number of digits to print. This only has an effect for + `unique=True`. In that case more digits than necessary to uniquely + identify the value may be printed and rounded unbiased. + -- versionadded:: 1.21.0 + Returns ------- rep : string @@ -1065,15 +1081,18 @@ def format_float_scientific(x, precision=None, unique=True, trim='k', precision = _none_or_positive_arg(precision, 'precision') pad_left = _none_or_positive_arg(pad_left, 'pad_left') exp_digits = _none_or_positive_arg(exp_digits, 'exp_digits') + min_digits = _none_or_positive_arg(min_digits, 'min_digits') + if min_digits > 0 and precision > 0 and min_digits > precision: + raise ValueError("min_digits must be less than or equal to precision") return dragon4_scientific(x, precision=precision, unique=unique, trim=trim, sign=sign, pad_left=pad_left, - exp_digits=exp_digits) + exp_digits=exp_digits, min_digits=min_digits) @set_module('numpy') def format_float_positional(x, precision=None, unique=True, fractional=True, trim='k', sign=False, - pad_left=None, pad_right=None): + pad_left=None, pad_right=None, min_digits=None): """ Format a floating-point scalar as a decimal string in positional notation. @@ -1091,16 +1110,19 @@ def format_float_positional(x, precision=None, unique=True, If `True`, use a digit-generation strategy which gives the shortest representation which uniquely identifies the floating-point number from other values of the same type, by judicious rounding. If `precision` - was omitted, print out all necessary digits, otherwise digit generation - is cut off after `precision` digits and the remaining value is rounded. + is given fewer digits than necessary can be printed, or if `min_digits` + is given more can be printed, in which cases the last digit is rounded + with unbiased rounding. If `False`, digits are generated as if printing an infinite-precision value and stopping after `precision` digits, rounding the remaining - value. + value with unbiased rounding fractional : boolean, optional - If `True`, the cutoff of `precision` digits refers to the total number - of digits after the decimal point, including leading zeros. - If `False`, `precision` refers to the total number of significant - digits, before or after the decimal point, ignoring leading zeros. + If `True`, the cutoffs of `precision` and `min_digits` refer to the + total number of digits after the decimal point, including leading + zeros. + If `False`, `precision` and `min_digits` refer to the total number of + significant digits, before or after the decimal point, ignoring leading + zeros. trim : one of 'k', '.', '0', '-', optional Controls post-processing trimming of trailing digits, as follows: @@ -1117,6 +1139,12 @@ def format_float_positional(x, precision=None, unique=True, pad_right : non-negative integer, optional Pad the right side of the string with whitespace until at least that many characters are to the right of the decimal point. + min_digits : non-negative integer or None, optional + Minimum number of digits to print. Only has an effect if `unique=True` + in which case additional digits past those necessary to uniquely + identify the value may be printed, rounding the last additional digit. + + -- versionadded:: 1.21.0 Returns ------- @@ -1141,10 +1169,16 @@ def format_float_positional(x, precision=None, unique=True, precision = _none_or_positive_arg(precision, 'precision') pad_left = _none_or_positive_arg(pad_left, 'pad_left') pad_right = _none_or_positive_arg(pad_right, 'pad_right') + min_digits = _none_or_positive_arg(min_digits, 'min_digits') + if not fractional and precision == 0: + raise ValueError("precision must be greater than 0 if " + "fractional=False") + if min_digits > 0 and precision > 0 and min_digits > precision: + raise ValueError("min_digits must be less than or equal to precision") return dragon4_positional(x, precision=precision, unique=unique, fractional=fractional, trim=trim, sign=sign, pad_left=pad_left, - pad_right=pad_right) + pad_right=pad_right, min_digits=min_digits) class IntegerFormat: @@ -1247,12 +1281,12 @@ def __init__(self, x, unit=None, timezone=None, casting='same_kind', self.legacy = legacy # must be called after the above are configured - super(DatetimeFormat, self).__init__(x) + super().__init__(x) def __call__(self, x): if self.legacy == '1.13': return self._format_non_nat(x) - return super(DatetimeFormat, self).__call__(x) + return super().__call__(x) def _format_non_nat(self, x): return "'%s'" % datetime_as_string(x, diff --git a/numpy/core/arrayprint.pyi b/numpy/core/arrayprint.pyi new file mode 100644 index 000000000000..ac2b6f5a8abb --- /dev/null +++ b/numpy/core/arrayprint.pyi @@ -0,0 +1,147 @@ +import sys +from types import TracebackType +from typing import Any, Optional, Callable, Union, Type + +# Using a private class is by no means ideal, but it is simply a consquence +# of a `contextlib.context` returning an instance of aformentioned class +from contextlib import _GeneratorContextManager + +from numpy import ( + ndarray, + generic, + bool_, + integer, + timedelta64, + datetime64, + floating, + complexfloating, + void, + str_, + bytes_, + longdouble, + clongdouble, +) +from numpy.typing import ArrayLike, _CharLike_co, _FloatLike_co + +if sys.version_info > (3, 8): + from typing import Literal, TypedDict, SupportsIndex +else: + from typing_extensions import Literal, TypedDict, SupportsIndex + +_FloatMode = Literal["fixed", "unique", "maxprec", "maxprec_equal"] + +class _FormatDict(TypedDict, total=False): + bool: Callable[[bool_], str] + int: Callable[[integer[Any]], str] + timedelta: Callable[[timedelta64], str] + datetime: Callable[[datetime64], str] + float: Callable[[floating[Any]], str] + longfloat: Callable[[longdouble], str] + complexfloat: Callable[[complexfloating[Any, Any]], str] + longcomplexfloat: Callable[[clongdouble], str] + void: Callable[[void], str] + numpystr: Callable[[_CharLike_co], str] + object: Callable[[object], str] + all: Callable[[object], str] + int_kind: Callable[[integer[Any]], str] + float_kind: Callable[[floating[Any]], str] + complex_kind: Callable[[complexfloating[Any, Any]], str] + str_kind: Callable[[_CharLike_co], str] + +class _FormatOptions(TypedDict): + precision: int + threshold: int + edgeitems: int + linewidth: int + suppress: bool + nanstr: str + infstr: str + formatter: Optional[_FormatDict] + sign: Literal["-", "+", " "] + floatmode: _FloatMode + legacy: Literal[False, "1.13"] + +def set_printoptions( + precision: Optional[SupportsIndex] = ..., + threshold: Optional[int] = ..., + edgeitems: Optional[int] = ..., + linewidth: Optional[int] = ..., + suppress: Optional[bool] = ..., + nanstr: Optional[str] = ..., + infstr: Optional[str] = ..., + formatter: Optional[_FormatDict] = ..., + sign: Optional[Literal["-", "+", " "]] = ..., + floatmode: Optional[_FloatMode] = ..., + *, + legacy: Optional[Literal[False, "1.13"]] = ... +) -> None: ... +def get_printoptions() -> _FormatOptions: ... +def array2string( + a: ndarray[Any, Any], + max_line_width: Optional[int] = ..., + precision: Optional[SupportsIndex] = ..., + suppress_small: Optional[bool] = ..., + separator: str = ..., + prefix: str = ..., + # NOTE: With the `style` argument being deprecated, + # all arguments between `formatter` and `suffix` are de facto + # keyworld-only arguments + *, + formatter: Optional[_FormatDict] = ..., + threshold: Optional[int] = ..., + edgeitems: Optional[int] = ..., + sign: Optional[Literal["-", "+", " "]] = ..., + floatmode: Optional[_FloatMode] = ..., + suffix: str = ..., + legacy: Optional[Literal[False, "1.13"]] = ..., +) -> str: ... +def format_float_scientific( + x: _FloatLike_co, + precision: Optional[int] = ..., + unique: bool = ..., + trim: Literal["k", ".", "0", "-"] = ..., + sign: bool = ..., + pad_left: Optional[int] = ..., + exp_digits: Optional[int] = ..., + min_digits: Optional[int] = ..., +) -> str: ... +def format_float_positional( + x: _FloatLike_co, + precision: Optional[int] = ..., + unique: bool = ..., + fractional: bool = ..., + trim: Literal["k", ".", "0", "-"] = ..., + sign: bool = ..., + pad_left: Optional[int] = ..., + pad_right: Optional[int] = ..., + min_digits: Optional[int] = ..., +) -> str: ... +def array_repr( + arr: ndarray[Any, Any], + max_line_width: Optional[int] = ..., + precision: Optional[SupportsIndex] = ..., + suppress_small: Optional[bool] = ..., +) -> str: ... +def array_str( + a: ndarray[Any, Any], + max_line_width: Optional[int] = ..., + precision: Optional[SupportsIndex] = ..., + suppress_small: Optional[bool] = ..., +) -> str: ... +def set_string_function( + f: Optional[Callable[[ndarray[Any, Any]], str]], repr: bool = ... +) -> None: ... +def printoptions( + precision: Optional[SupportsIndex] = ..., + threshold: Optional[int] = ..., + edgeitems: Optional[int] = ..., + linewidth: Optional[int] = ..., + suppress: Optional[bool] = ..., + nanstr: Optional[str] = ..., + infstr: Optional[str] = ..., + formatter: Optional[_FormatDict] = ..., + sign: Optional[Literal["-", "+", " "]] = ..., + floatmode: Optional[_FloatMode] = ..., + *, + legacy: Optional[Literal[False, "1.13"]] = ... +) -> _GeneratorContextManager[_FormatOptions]: ... diff --git a/numpy/core/code_generators/cversions.txt b/numpy/core/code_generators/cversions.txt index 2d3a653913a3..09c1c31e03ef 100644 --- a/numpy/core/code_generators/cversions.txt +++ b/numpy/core/code_generators/cversions.txt @@ -55,4 +55,5 @@ # Version 14 (NumPy 1.20) # DType related API additions. # A new field was added to the end of PyArrayObject_fields. +# Version 14 (NumPy 1.21) No change. 0x0000000e = 17a0f366e55ec05e5c5c149123478452 diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index ca6a228284a4..c2458c2b5d80 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -31,6 +31,7 @@ join('multiarray', 'arraytypes.c.src'), join('multiarray', 'buffer.c'), join('multiarray', 'calculation.c'), + join('multiarray', 'common_dtype.c'), join('multiarray', 'conversion_utils.c'), join('multiarray', 'convert.c'), join('multiarray', 'convert_datatype.c'), @@ -175,7 +176,7 @@ def split_arguments(argstr): def finish_arg(): if current_argument: argstr = ''.join(current_argument).strip() - m = re.match(r'(.*(\s+|[*]))(\w+)$', argstr) + m = re.match(r'(.*(\s+|\*))(\w+)$', argstr) if m: typename = m.group(1).strip() name = m.group(3) @@ -281,9 +282,11 @@ def find_functions(filename, tag='API'): state = SCANNING else: function_args.append(line) - except Exception: - print(filename, lineno + 1) + except ParseError: raise + except Exception as e: + msg = "see chained exception for details" + raise ParseError(filename, lineno + 1, msg) from e fo.close() return functions diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index cb1147b93fba..9e94f9cccc47 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -45,14 +45,20 @@ class TypeDescription: astype : dict or None, optional If astype['x'] is 'y', uses PyUFunc_x_x_As_y_y/PyUFunc_xx_x_As_yy_y instead of PyUFunc_x_x/PyUFunc_xx_x. + cfunc_alias : str or none, optional + Appended to inner loop C function name, e.g., FLOAT_{cfunc_alias}. See make_arrays. + NOTE: it doesn't support 'astype' simd: list Available SIMD ufunc loops, dispatched at runtime in specified order Currently only supported for simples types (see make_arrays) - dispatch: list - Available SIMD ufunc loops, dispatched at runtime in specified order - Currently only supported for simples types (see make_arrays) + dispatch: str or None, optional + Dispatch-able source name without its extension '.dispatch.c' that + contains the definition of ufunc, dispatched at runtime depending on the + specified targets of the dispatch-able source. + NOTE: it doesn't support 'astype' """ - def __init__(self, type, f=None, in_=None, out=None, astype=None, simd=None, dispatch=None): + def __init__(self, type, f=None, in_=None, out=None, astype=None, cfunc_alias=None, + simd=None, dispatch=None): self.type = type self.func_data = f if astype is None: @@ -64,6 +70,7 @@ def __init__(self, type, f=None, in_=None, out=None, astype=None, simd=None, dis if out is not None: out = out.replace('P', type) self.out = out + self.cfunc_alias = cfunc_alias self.simd = simd self.dispatch = dispatch @@ -90,7 +97,8 @@ def build_func_data(types, f): func_data = [_fdata_map.get(t, '%s') % (f,) for t in types] return func_data -def TD(types, f=None, astype=None, in_=None, out=None, simd=None, dispatch=None): +def TD(types, f=None, astype=None, in_=None, out=None, cfunc_alias=None, + simd=None, dispatch=None): if f is not None: if isinstance(f, str): func_data = build_func_data(types, f) @@ -119,13 +127,15 @@ def TD(types, f=None, astype=None, in_=None, out=None, simd=None, dispatch=None) simdt = [k for k, v in simd if t in v] else: simdt = [] + # [(dispatch file name without extension '.dispatch.c*', list of types)] if dispatch: - dispt = [k for k, v in dispatch if t in v] + dispt = ([k for k, v in dispatch if t in v]+[None])[0] else: - dispt = [] + dispt = None tds.append(TypeDescription( - t, f=fd, in_=i, out=o, astype=astype, simd=simdt, dispatch=dispt + t, f=fd, in_=i, out=o, astype=astype, cfunc_alias=cfunc_alias, + simd=simdt, dispatch=dispt )) return tds @@ -235,6 +245,8 @@ def english_upper(s): O = 'O' P = 'P' ints = 'bBhHiIlLqQ' +sints = 'bhilq' +uints = 'BHILQ' times = 'Mm' timedeltaonly = 'm' intsO = ints + O @@ -280,7 +292,7 @@ def english_upper(s): Ufunc(2, 1, Zero, docstrings.get('numpy.core.umath.add'), 'PyUFunc_AdditionTypeResolver', - TD(notimes_or_obj, simd=[('avx512f', cmplxvec),('avx2', ints)]), + TD(notimes_or_obj, simd=[('avx2', ints)], dispatch=[('loops_arithm_fp', 'fdFD')]), [TypeDescription('M', FullTypeDescr, 'Mm', 'M'), TypeDescription('m', FullTypeDescr, 'mm', 'm'), TypeDescription('M', FullTypeDescr, 'mM', 'M'), @@ -291,7 +303,7 @@ def english_upper(s): Ufunc(2, 1, None, # Zero is only a unit to the right, not the left docstrings.get('numpy.core.umath.subtract'), 'PyUFunc_SubtractionTypeResolver', - TD(ints + inexact, simd=[('avx512f', cmplxvec),('avx2', ints)]), + TD(ints + inexact, simd=[('avx2', ints)], dispatch=[('loops_arithm_fp', 'fdFD')]), [TypeDescription('M', FullTypeDescr, 'Mm', 'M'), TypeDescription('m', FullTypeDescr, 'mm', 'm'), TypeDescription('M', FullTypeDescr, 'MM', 'm'), @@ -302,7 +314,7 @@ def english_upper(s): Ufunc(2, 1, One, docstrings.get('numpy.core.umath.multiply'), 'PyUFunc_MultiplicationTypeResolver', - TD(notimes_or_obj, simd=[('avx512f', cmplxvec),('avx2', ints)]), + TD(notimes_or_obj, simd=[('avx2', ints)], dispatch=[('loops_arithm_fp', 'fdFD')]), [TypeDescription('m', FullTypeDescr, 'mq', 'm'), TypeDescription('m', FullTypeDescr, 'qm', 'm'), TypeDescription('m', FullTypeDescr, 'md', 'm'), @@ -315,7 +327,9 @@ def english_upper(s): Ufunc(2, 1, None, # One is only a unit to the right, not the left docstrings.get('numpy.core.umath.floor_divide'), 'PyUFunc_DivisionTypeResolver', - TD(intfltcmplx), + TD(ints, cfunc_alias='divide', + dispatch=[('loops_arithmetic', 'bBhHiIlLqQ')]), + TD(flts + cmplx), [TypeDescription('m', FullTypeDescr, 'mq', 'm'), TypeDescription('m', FullTypeDescr, 'md', 'm'), TypeDescription('m', FullTypeDescr, 'mm', 'q'), @@ -326,10 +340,10 @@ def english_upper(s): Ufunc(2, 1, None, # One is only a unit to the right, not the left docstrings.get('numpy.core.umath.true_divide'), 'PyUFunc_TrueDivisionTypeResolver', - TD(flts+cmplx), - [TypeDescription('m', FullTypeDescr, 'mq', 'm'), - TypeDescription('m', FullTypeDescr, 'md', 'm'), - TypeDescription('m', FullTypeDescr, 'mm', 'd'), + TD(flts+cmplx, cfunc_alias='divide', dispatch=[('loops_arithm_fp', 'fd')]), + [TypeDescription('m', FullTypeDescr, 'mq', 'm', cfunc_alias='divide'), + TypeDescription('m', FullTypeDescr, 'md', 'm', cfunc_alias='divide'), + TypeDescription('m', FullTypeDescr, 'mm', 'd', cfunc_alias='divide'), ], TD(O, f='PyNumber_TrueDivide'), ), @@ -666,7 +680,7 @@ def english_upper(s): docstrings.get('numpy.core.umath.cos'), None, TD('e', f='cos', astype={'e':'f'}), - TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('f', dispatch=[('loops_trigonometric', 'f')]), TD('fdg' + cmplx, f='cos'), TD(P, f='cos'), ), @@ -675,7 +689,7 @@ def english_upper(s): docstrings.get('numpy.core.umath.sin'), None, TD('e', f='sin', astype={'e':'f'}), - TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), + TD('f', dispatch=[('loops_trigonometric', 'f')]), TD('fdg' + cmplx, f='sin'), TD(P, f='sin'), ), @@ -712,8 +726,7 @@ def english_upper(s): docstrings.get('numpy.core.umath.exp'), None, TD('e', f='exp', astype={'e':'f'}), - TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), - TD('d', simd=[('avx512f', 'd')]), + TD('fd', dispatch=[('loops_exponent_log', 'fd')]), TD('fdg' + cmplx, f='exp'), TD(P, f='exp'), ), @@ -736,8 +749,7 @@ def english_upper(s): docstrings.get('numpy.core.umath.log'), None, TD('e', f='log', astype={'e':'f'}), - TD('f', simd=[('fma', 'f'), ('avx512f', 'f')]), - TD('d', simd=[('avx512f', 'd')]), + TD('fd', dispatch=[('loops_exponent_log', 'fd')]), TD('fdg' + cmplx, f='log'), TD(P, f='log'), ), @@ -910,10 +922,10 @@ def english_upper(s): docstrings.get('numpy.core.umath.ldexp'), None, [TypeDescription('e', None, 'ei', 'e'), - TypeDescription('f', None, 'fi', 'f', simd=['avx512_skx']), + TypeDescription('f', None, 'fi', 'f', dispatch='loops_exponent_log'), TypeDescription('e', FuncNameSuffix('long'), 'el', 'e'), TypeDescription('f', FuncNameSuffix('long'), 'fl', 'f'), - TypeDescription('d', None, 'di', 'd', simd=['avx512_skx']), + TypeDescription('d', None, 'di', 'd', dispatch='loops_exponent_log'), TypeDescription('d', FuncNameSuffix('long'), 'dl', 'd'), TypeDescription('g', None, 'gi', 'g'), TypeDescription('g', FuncNameSuffix('long'), 'gl', 'g'), @@ -924,8 +936,8 @@ def english_upper(s): docstrings.get('numpy.core.umath.frexp'), None, [TypeDescription('e', None, 'e', 'ei'), - TypeDescription('f', None, 'f', 'fi', simd=['avx512_skx']), - TypeDescription('d', None, 'd', 'di', simd=['avx512_skx']), + TypeDescription('f', None, 'f', 'fi', dispatch='loops_exponent_log'), + TypeDescription('d', None, 'd', 'di', dispatch='loops_exponent_log'), TypeDescription('g', None, 'g', 'gi'), ], ), @@ -1000,6 +1012,7 @@ def make_arrays(funcdict): # later code1list = [] code2list = [] + dispdict = {} names = sorted(funcdict.keys()) for name in names: uf = funcdict[name] @@ -1010,44 +1023,33 @@ def make_arrays(funcdict): sub = 0 for t in uf.type_descriptions: + cfunc_alias = t.cfunc_alias if t.cfunc_alias else name + cfunc_fname = None if t.func_data is FullTypeDescr: tname = english_upper(chartoname[t.type]) datalist.append('(void *)NULL') - funclist.append( - '%s_%s_%s_%s' % (tname, t.in_, t.out, name)) + cfunc_fname = f"{tname}_{t.in_}_{t.out}_{cfunc_alias}" elif isinstance(t.func_data, FuncNameSuffix): datalist.append('(void *)NULL') tname = english_upper(chartoname[t.type]) - funclist.append( - '%s_%s_%s' % (tname, name, t.func_data.suffix)) + cfunc_fname = f"{tname}_{cfunc_alias}_{t.func_data.suffix}" elif t.func_data is None: datalist.append('(void *)NULL') tname = english_upper(chartoname[t.type]) - funclist.append('%s_%s' % (tname, name)) + cfunc_fname = f"{tname}_{cfunc_alias}" if t.simd is not None: for vt in t.simd: code2list.append(textwrap.dedent("""\ #ifdef HAVE_ATTRIBUTE_TARGET_{ISA} if (NPY_CPU_HAVE({ISA})) {{ - {fname}_functions[{idx}] = {type}_{fname}_{isa}; + {fname}_functions[{idx}] = {cname}_{isa}; }} #endif """).format( ISA=vt.upper(), isa=vt, - fname=name, type=tname, idx=k - )) - if t.dispatch is not None: - for dname in t.dispatch: - code2list.append(textwrap.dedent("""\ - #ifndef NPY_DISABLE_OPTIMIZATION - #include "{dname}.dispatch.h" - #endif - NPY_CPU_DISPATCH_CALL_XB({name}_functions[{k}] = {tname}_{name}); - """).format( - dname=dname, name=name, tname=tname, k=k + fname=name, cname=cfunc_fname, idx=k )) else: - funclist.append('NULL') try: thedict = arity_lookup[uf.nin, uf.nout] except KeyError as e: @@ -1077,6 +1079,13 @@ def make_arrays(funcdict): #datalist.append('(void *)%s' % t.func_data) sub += 1 + if cfunc_fname: + funclist.append(cfunc_fname) + if t.dispatch: + dispdict.setdefault(t.dispatch, []).append((name, k, cfunc_fname)) + else: + funclist.append('NULL') + for x in t.in_ + t.out: siglist.append('NPY_%s' % (english_upper(chartoname[x]),)) @@ -1091,6 +1100,17 @@ def make_arrays(funcdict): % (name, datanames)) code1list.append("static char %s_signatures[] = {%s};" % (name, signames)) + + for dname, funcs in dispdict.items(): + code2list.append(textwrap.dedent(f""" + #ifndef NPY_DISABLE_OPTIMIZATION + #include "{dname}.dispatch.h" + #endif + """)) + for (ufunc_name, func_idx, cfunc_name) in funcs: + code2list.append(textwrap.dedent(f"""\ + NPY_CPU_DISPATCH_CALL_XB({ufunc_name}_functions[{func_idx}] = {cfunc_name}); + """)) return "\n".join(code1list), "\n".join(code2list) def make_ufuncs(funcdict): diff --git a/numpy/core/code_generators/ufunc_docstrings.py b/numpy/core/code_generators/ufunc_docstrings.py index b7edd2834b43..f19946be408a 100644 --- a/numpy/core/code_generators/ufunc_docstrings.py +++ b/numpy/core/code_generators/ufunc_docstrings.py @@ -185,7 +185,7 @@ def add_newdoc(place, name, doc): Notes ----- `arccos` is a multivalued function: for each `x` there are infinitely - many numbers `z` such that `cos(z) = x`. The convention is to return + many numbers `z` such that ``cos(z) = x``. The convention is to return the angle `z` whose real part lies in `[0, pi]`. For real-valued input data types, `arccos` always returns real output. @@ -193,7 +193,7 @@ def add_newdoc(place, name, doc): it yields ``nan`` and sets the `invalid` floating point error flag. For complex-valued input, `arccos` is a complex analytic function that - has branch cuts `[-inf, -1]` and `[1, inf]` and is continuous from + has branch cuts ``[-inf, -1]`` and `[1, inf]` and is continuous from above on the former and from below on the latter. The inverse `cos` is also known as `acos` or cos^-1. @@ -245,7 +245,7 @@ def add_newdoc(place, name, doc): ----- `arccosh` is a multivalued function: for each `x` there are infinitely many numbers `z` such that `cosh(z) = x`. The convention is to return the - `z` whose imaginary part lies in `[-pi, pi]` and the real part in + `z` whose imaginary part lies in ``[-pi, pi]`` and the real part in ``[0, inf]``. For real-valued input data types, `arccosh` always returns real output. @@ -406,7 +406,7 @@ def add_newdoc(place, name, doc): it yields ``nan`` and sets the `invalid` floating point error flag. For complex-valued input, `arctan` is a complex analytic function that - has [`1j, infj`] and [`-1j, -infj`] as branch cuts, and is continuous + has [``1j, infj``] and [``-1j, -infj``] as branch cuts, and is continuous from the left on the former and from the right on the latter. The inverse tangent is also known as `atan` or tan^{-1}. @@ -544,7 +544,7 @@ def add_newdoc(place, name, doc): Notes ----- `arctanh` is a multivalued function: for each `x` there are infinitely - many numbers `z` such that `tanh(z) = x`. The convention is to return + many numbers `z` such that ``tanh(z) = x``. The convention is to return the `z` whose imaginary part lies in `[-pi/2, pi/2]`. For real-valued input data types, `arctanh` always returns real output. @@ -765,7 +765,7 @@ def add_newdoc(place, name, doc): Return the ceiling of the input, element-wise. The ceil of the scalar `x` is the smallest integer `i`, such that - `i >= x`. It is often denoted as :math:`\\lceil x \\rceil`. + ``i >= x``. It is often denoted as :math:`\\lceil x \\rceil`. Parameters ---------- @@ -781,7 +781,7 @@ def add_newdoc(place, name, doc): See Also -------- - floor, trunc, rint + floor, trunc, rint, fix Examples -------- @@ -813,7 +813,7 @@ def add_newdoc(place, name, doc): See Also -------- - ceil, floor, rint + ceil, floor, rint, fix Notes ----- @@ -1374,13 +1374,14 @@ def add_newdoc(place, name, doc): See Also -------- - ceil, trunc, rint + ceil, trunc, rint, fix Notes ----- - Some spreadsheet programs calculate the "floor-towards-zero", in other - words ``floor(-2.5) == -2``. NumPy instead uses the definition of - `floor` where `floor(-2.5) == -3`. + Some spreadsheet programs calculate the "floor-towards-zero", where + ``floor(-2.5) == -2``. NumPy instead uses the definition of + `floor` where `floor(-2.5) == -3`. The "floor-towards-zero" + function is called ``fix`` in NumPy. Examples -------- @@ -2808,8 +2809,8 @@ def add_newdoc(place, name, doc): (9, 5, 7, 3) >>> # n is 7, k is 4, m is 3 - The matmul function implements the semantics of the `@` operator introduced - in Python 3.5 following PEP465. + The matmul function implements the semantics of the ``@`` operator introduced + in Python 3.5 following :pep:`465`. Examples -------- @@ -3486,7 +3487,13 @@ def add_newdoc(place, name, doc): See Also -------- - ceil, floor, trunc + fix, ceil, floor, trunc + + Notes + ----- + For values exactly halfway between rounded decimal values, NumPy + rounds to the nearest even value. Thus 1.5 and 2.5 round to 2.0, + -0.5 and 0.5 round to 0.0, etc. Examples -------- @@ -4074,7 +4081,7 @@ def add_newdoc(place, name, doc): Decompose the elements of x into mantissa and twos exponent. Returns (`mantissa`, `exponent`), where `x = mantissa * 2**exponent``. - The mantissa is lies in the open interval(-1, 1), while the twos + The mantissa lies in the open interval(-1, 1), while the twos exponent is a signed integer. Parameters diff --git a/numpy/core/defchararray.py b/numpy/core/defchararray.py index 9d7b54a1a144..ab1166ad263f 100644 --- a/numpy/core/defchararray.py +++ b/numpy/core/defchararray.py @@ -273,7 +273,7 @@ def str_len(a): out : ndarray Output array of integers - See also + See Also -------- builtins.len """ @@ -368,7 +368,7 @@ def mod(a, values): out : ndarray Output array of str or unicode, depending on input types - See also + See Also -------- str.__mod__ @@ -398,7 +398,7 @@ def capitalize(a): Output array of str or unicode, depending on input types - See also + See Also -------- str.capitalize @@ -443,7 +443,7 @@ def center(a, width, fillchar=' '): Output array of str or unicode, depending on input types - See also + See Also -------- str.center @@ -485,7 +485,7 @@ def count(a, sub, start=0, end=None): out : ndarray Output array of ints. - See also + See Also -------- str.count @@ -534,7 +534,7 @@ def decode(a, encoding=None, errors=None): ------- out : ndarray - See also + See Also -------- str.decode @@ -580,7 +580,7 @@ def encode(a, encoding=None, errors=None): ------- out : ndarray - See also + See Also -------- str.encode @@ -620,7 +620,7 @@ def endswith(a, suffix, start=0, end=None): out : ndarray Outputs an array of bools. - See also + See Also -------- str.endswith @@ -672,7 +672,7 @@ def expandtabs(a, tabsize=8): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.expandtabs @@ -708,7 +708,7 @@ def find(a, sub, start=0, end=None): out : ndarray or int Output array of ints. Returns -1 if `sub` is not found. - See also + See Also -------- str.find @@ -737,7 +737,7 @@ def index(a, sub, start=0, end=None): out : ndarray Output array of ints. Returns -1 if `sub` is not found. - See also + See Also -------- find, str.find @@ -765,7 +765,7 @@ def isalnum(a): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.isalnum """ @@ -791,7 +791,7 @@ def isalpha(a): out : ndarray Output array of bools - See also + See Also -------- str.isalpha """ @@ -817,7 +817,7 @@ def isdigit(a): out : ndarray Output array of bools - See also + See Also -------- str.isdigit """ @@ -844,7 +844,7 @@ def islower(a): out : ndarray Output array of bools - See also + See Also -------- str.islower """ @@ -871,7 +871,7 @@ def isspace(a): out : ndarray Output array of bools - See also + See Also -------- str.isspace """ @@ -897,7 +897,7 @@ def istitle(a): out : ndarray Output array of bools - See also + See Also -------- str.istitle """ @@ -924,7 +924,7 @@ def isupper(a): out : ndarray Output array of bools - See also + See Also -------- str.isupper """ @@ -953,7 +953,7 @@ def join(sep, seq): out : ndarray Output array of str or unicode, depending on input types - See also + See Also -------- str.join """ @@ -988,7 +988,7 @@ def ljust(a, width, fillchar=' '): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.ljust @@ -1021,7 +1021,7 @@ def lower(a): out : ndarray, {str, unicode} Output array of str or unicode, depending on input type - See also + See Also -------- str.lower @@ -1066,7 +1066,7 @@ def lstrip(a, chars=None): out : ndarray, {str, unicode} Output array of str or unicode, depending on input type - See also + See Also -------- str.lstrip @@ -1127,7 +1127,7 @@ def partition(a, sep): The output array will have an extra dimension with 3 elements per input element. - See also + See Also -------- str.partition @@ -1163,7 +1163,7 @@ def replace(a, old, new, count=None): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.replace @@ -1197,7 +1197,7 @@ def rfind(a, sub, start=0, end=None): out : ndarray Output array of ints. Return -1 on failure. - See also + See Also -------- str.rfind @@ -1227,7 +1227,7 @@ def rindex(a, sub, start=0, end=None): out : ndarray Output array of ints. - See also + See Also -------- rfind, str.rindex @@ -1258,7 +1258,7 @@ def rjust(a, width, fillchar=' '): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.rjust @@ -1299,7 +1299,7 @@ def rpartition(a, sep): type. The output array will have an extra dimension with 3 elements per input element. - See also + See Also -------- str.rpartition @@ -1339,7 +1339,7 @@ def rsplit(a, sep=None, maxsplit=None): out : ndarray Array of list objects - See also + See Also -------- str.rsplit, split @@ -1378,7 +1378,7 @@ def rstrip(a, chars=None): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.rstrip @@ -1423,7 +1423,7 @@ def split(a, sep=None, maxsplit=None): out : ndarray Array of list objects - See also + See Also -------- str.split, rsplit @@ -1459,7 +1459,7 @@ def splitlines(a, keepends=None): out : ndarray Array of list objects - See also + See Also -------- str.splitlines @@ -1495,7 +1495,7 @@ def startswith(a, prefix, start=0, end=None): out : ndarray Array of booleans - See also + See Also -------- str.startswith @@ -1528,7 +1528,7 @@ def strip(a, chars=None): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.strip @@ -1569,7 +1569,7 @@ def swapcase(a): out : ndarray, {str, unicode} Output array of str or unicode, depending on input type - See also + See Also -------- str.swapcase @@ -1609,7 +1609,7 @@ def title(a): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.title @@ -1654,7 +1654,7 @@ def translate(a, table, deletechars=None): out : ndarray Output array of str or unicode, depending on input type - See also + See Also -------- str.translate @@ -1687,7 +1687,7 @@ def upper(a): out : ndarray, {str, unicode} Output array of str or unicode, depending on input type - See also + See Also -------- str.upper @@ -1726,7 +1726,7 @@ def zfill(a, width): out : ndarray, {str, unicode} Output array of str or unicode, depending on input type - See also + See Also -------- str.zfill @@ -1760,7 +1760,7 @@ def isnumeric(a): out : ndarray, bool Array of booleans of same shape as `a`. - See also + See Also -------- unicode.isnumeric @@ -1792,7 +1792,7 @@ def isdecimal(a): out : ndarray, bool Array of booleans identical in shape to `a`. - See also + See Also -------- unicode.isdecimal @@ -2004,7 +2004,7 @@ def __eq__(self, other): """ Return (self == other) element-wise. - See also + See Also -------- equal """ @@ -2014,7 +2014,7 @@ def __ne__(self, other): """ Return (self != other) element-wise. - See also + See Also -------- not_equal """ @@ -2024,7 +2024,7 @@ def __ge__(self, other): """ Return (self >= other) element-wise. - See also + See Also -------- greater_equal """ @@ -2034,7 +2034,7 @@ def __le__(self, other): """ Return (self <= other) element-wise. - See also + See Also -------- less_equal """ @@ -2044,7 +2044,7 @@ def __gt__(self, other): """ Return (self > other) element-wise. - See also + See Also -------- greater """ @@ -2054,7 +2054,7 @@ def __lt__(self, other): """ Return (self < other) element-wise. - See also + See Also -------- less """ @@ -2065,7 +2065,7 @@ def __add__(self, other): Return (self + other), that is string concatenation, element-wise for a pair of array_likes of str or unicode. - See also + See Also -------- add """ @@ -2076,7 +2076,7 @@ def __radd__(self, other): Return (other + self), that is string concatenation, element-wise for a pair of array_likes of `string_` or `unicode_`. - See also + See Also -------- add """ @@ -2087,7 +2087,7 @@ def __mul__(self, i): Return (self * i), that is string multiple concatenation, element-wise. - See also + See Also -------- multiply """ @@ -2098,7 +2098,7 @@ def __rmul__(self, i): Return (self * i), that is string multiple concatenation, element-wise. - See also + See Also -------- multiply """ @@ -2110,7 +2110,7 @@ def __mod__(self, i): (interpolation), element-wise for a pair of array_likes of `string_` or `unicode_`. - See also + See Also -------- mod """ @@ -2145,7 +2145,7 @@ def capitalize(self): Return a copy of `self` with only the first character of each element capitalized. - See also + See Also -------- char.capitalize @@ -2157,7 +2157,7 @@ def center(self, width, fillchar=' '): Return a copy of `self` with its elements centered in a string of length `width`. - See also + See Also -------- center """ @@ -2168,7 +2168,7 @@ def count(self, sub, start=0, end=None): Returns an array with the number of non-overlapping occurrences of substring `sub` in the range [`start`, `end`]. - See also + See Also -------- char.count @@ -2179,7 +2179,7 @@ def decode(self, encoding=None, errors=None): """ Calls `str.decode` element-wise. - See also + See Also -------- char.decode @@ -2190,7 +2190,7 @@ def encode(self, encoding=None, errors=None): """ Calls `str.encode` element-wise. - See also + See Also -------- char.encode @@ -2202,7 +2202,7 @@ def endswith(self, suffix, start=0, end=None): Returns a boolean array which is `True` where the string element in `self` ends with `suffix`, otherwise `False`. - See also + See Also -------- char.endswith @@ -2214,7 +2214,7 @@ def expandtabs(self, tabsize=8): Return a copy of each string element where all tab characters are replaced by one or more spaces. - See also + See Also -------- char.expandtabs @@ -2226,7 +2226,7 @@ def find(self, sub, start=0, end=None): For each element, return the lowest index in the string where substring `sub` is found. - See also + See Also -------- char.find @@ -2237,7 +2237,7 @@ def index(self, sub, start=0, end=None): """ Like `find`, but raises `ValueError` when the substring is not found. - See also + See Also -------- char.index @@ -2250,7 +2250,7 @@ def isalnum(self): are alphanumeric and there is at least one character, false otherwise. - See also + See Also -------- char.isalnum @@ -2263,7 +2263,7 @@ def isalpha(self): are alphabetic and there is at least one character, false otherwise. - See also + See Also -------- char.isalpha @@ -2275,7 +2275,7 @@ def isdigit(self): Returns true for each element if all characters in the string are digits and there is at least one character, false otherwise. - See also + See Also -------- char.isdigit @@ -2288,7 +2288,7 @@ def islower(self): string are lowercase and there is at least one cased character, false otherwise. - See also + See Also -------- char.islower @@ -2301,7 +2301,7 @@ def isspace(self): characters in the string and there is at least one character, false otherwise. - See also + See Also -------- char.isspace @@ -2313,7 +2313,7 @@ def istitle(self): Returns true for each element if the element is a titlecased string and there is at least one character, false otherwise. - See also + See Also -------- char.istitle @@ -2326,7 +2326,7 @@ def isupper(self): string are uppercase and there is at least one character, false otherwise. - See also + See Also -------- char.isupper @@ -2338,7 +2338,7 @@ def join(self, seq): Return a string which is the concatenation of the strings in the sequence `seq`. - See also + See Also -------- char.join @@ -2350,7 +2350,7 @@ def ljust(self, width, fillchar=' '): Return an array with the elements of `self` left-justified in a string of length `width`. - See also + See Also -------- char.ljust @@ -2362,7 +2362,7 @@ def lower(self): Return an array with the elements of `self` converted to lowercase. - See also + See Also -------- char.lower @@ -2374,7 +2374,7 @@ def lstrip(self, chars=None): For each element in `self`, return a copy with the leading characters removed. - See also + See Also -------- char.lstrip @@ -2385,7 +2385,7 @@ def partition(self, sep): """ Partition each element in `self` around `sep`. - See also + See Also -------- partition """ @@ -2396,7 +2396,7 @@ def replace(self, old, new, count=None): For each element in `self`, return a copy of the string with all occurrences of substring `old` replaced by `new`. - See also + See Also -------- char.replace @@ -2409,7 +2409,7 @@ def rfind(self, sub, start=0, end=None): where substring `sub` is found, such that `sub` is contained within [`start`, `end`]. - See also + See Also -------- char.rfind @@ -2421,7 +2421,7 @@ def rindex(self, sub, start=0, end=None): Like `rfind`, but raises `ValueError` when the substring `sub` is not found. - See also + See Also -------- char.rindex @@ -2433,7 +2433,7 @@ def rjust(self, width, fillchar=' '): Return an array with the elements of `self` right-justified in a string of length `width`. - See also + See Also -------- char.rjust @@ -2444,7 +2444,7 @@ def rpartition(self, sep): """ Partition each element in `self` around `sep`. - See also + See Also -------- rpartition """ @@ -2455,7 +2455,7 @@ def rsplit(self, sep=None, maxsplit=None): For each element in `self`, return a list of the words in the string, using `sep` as the delimiter string. - See also + See Also -------- char.rsplit @@ -2467,7 +2467,7 @@ def rstrip(self, chars=None): For each element in `self`, return a copy with the trailing characters removed. - See also + See Also -------- char.rstrip @@ -2479,7 +2479,7 @@ def split(self, sep=None, maxsplit=None): For each element in `self`, return a list of the words in the string, using `sep` as the delimiter string. - See also + See Also -------- char.split @@ -2491,7 +2491,7 @@ def splitlines(self, keepends=None): For each element in `self`, return a list of the lines in the element, breaking at line boundaries. - See also + See Also -------- char.splitlines @@ -2503,7 +2503,7 @@ def startswith(self, prefix, start=0, end=None): Returns a boolean array which is `True` where the string element in `self` starts with `prefix`, otherwise `False`. - See also + See Also -------- char.startswith @@ -2515,7 +2515,7 @@ def strip(self, chars=None): For each element in `self`, return a copy with the leading and trailing characters removed. - See also + See Also -------- char.strip @@ -2527,7 +2527,7 @@ def swapcase(self): For each element in `self`, return a copy of the string with uppercase characters converted to lowercase and vice versa. - See also + See Also -------- char.swapcase @@ -2540,7 +2540,7 @@ def title(self): string: words start with uppercase characters, all remaining cased characters are lowercase. - See also + See Also -------- char.title @@ -2554,7 +2554,7 @@ def translate(self, table, deletechars=None): `deletechars` are removed, and the remaining characters have been mapped through the given translation table. - See also + See Also -------- char.translate @@ -2566,7 +2566,7 @@ def upper(self): Return an array with the elements of `self` converted to uppercase. - See also + See Also -------- char.upper @@ -2578,7 +2578,7 @@ def zfill(self, width): Return the numeric string left-filled with zeros in a string of length `width`. - See also + See Also -------- char.zfill @@ -2590,7 +2590,7 @@ def isnumeric(self): For each element in `self`, return True if there are only numeric characters in the element. - See also + See Also -------- char.isnumeric @@ -2602,7 +2602,7 @@ def isdecimal(self): For each element in `self`, return True if there are only decimal characters in the element. - See also + See Also -------- char.isdecimal diff --git a/numpy/core/einsumfunc.py b/numpy/core/einsumfunc.py index e0942becaaa5..18157641aaf4 100644 --- a/numpy/core/einsumfunc.py +++ b/numpy/core/einsumfunc.py @@ -327,7 +327,7 @@ def _greedy_path(input_sets, output_set, idx_dict, memory_limit): Set that represents the rhs side of the overall einsum subscript idx_dict : dictionary Dictionary of index sizes - memory_limit_limit : int + memory_limit : int The maximum number of elements in a temporary array Returns @@ -1061,14 +1061,12 @@ def einsum(*operands, out=None, optimize=False, **kwargs): See Also -------- einsum_path, dot, inner, outer, tensordot, linalg.multi_dot - - einops: + einops : similar verbose interface is provided by `einops `_ package to cover additional operations: transpose, reshape/flatten, repeat/tile, squeeze/unsqueeze and reductions. - - opt_einsum: + opt_einsum : `opt_einsum `_ optimizes contraction order for einsum-like expressions in backend-agnostic manner. diff --git a/numpy/core/einsumfunc.pyi b/numpy/core/einsumfunc.pyi new file mode 100644 index 000000000000..2457e8719df4 --- /dev/null +++ b/numpy/core/einsumfunc.pyi @@ -0,0 +1,142 @@ +import sys +from typing import List, TypeVar, Optional, Any, overload, Union, Tuple, Sequence + +from numpy import ( + ndarray, + dtype, + bool_, + unsignedinteger, + signedinteger, + floating, + complexfloating, + number, + _OrderKACF, +) +from numpy.typing import ( + _ArrayLikeBool_co, + _ArrayLikeUInt_co, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _DTypeLikeBool, + _DTypeLikeUInt, + _DTypeLikeInt, + _DTypeLikeFloat, + _DTypeLikeComplex, + _DTypeLikeComplex_co, +) + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_ArrayType = TypeVar( + "_ArrayType", + bound=ndarray[Any, dtype[Union[bool_, number[Any]]]], +) + +_OptimizeKind = Union[ + None, bool, Literal["greedy", "optimal"], Sequence[Any] +] +_CastingSafe = Literal["no", "equiv", "safe", "same_kind"] +_CastingUnsafe = Literal["unsafe"] + +__all__: List[str] + +# TODO: Properly handle the `casting`-based combinatorics +# TODO: We need to evaluate the content `__subscripts` in order +# to identify whether or an array or scalar is returned. At a cursory +# glance this seems like something that can quite easilly be done with +# a mypy plugin. +# Something like `is_scalar = bool(__subscripts.partition("->")[-1])` +@overload +def einsum( + __subscripts: str, + *operands: _ArrayLikeBool_co, + out: None = ..., + dtype: Optional[_DTypeLikeBool] = ..., + order: _OrderKACF = ..., + casting: _CastingSafe = ..., + optimize: _OptimizeKind = ..., +) -> Any: ... +@overload +def einsum( + __subscripts: str, + *operands: _ArrayLikeUInt_co, + out: None = ..., + dtype: Optional[_DTypeLikeUInt] = ..., + order: _OrderKACF = ..., + casting: _CastingSafe = ..., + optimize: _OptimizeKind = ..., +) -> Any: ... +@overload +def einsum( + __subscripts: str, + *operands: _ArrayLikeInt_co, + out: None = ..., + dtype: Optional[_DTypeLikeInt] = ..., + order: _OrderKACF = ..., + casting: _CastingSafe = ..., + optimize: _OptimizeKind = ..., +) -> Any: ... +@overload +def einsum( + __subscripts: str, + *operands: _ArrayLikeFloat_co, + out: None = ..., + dtype: Optional[_DTypeLikeFloat] = ..., + order: _OrderKACF = ..., + casting: _CastingSafe = ..., + optimize: _OptimizeKind = ..., +) -> Any: ... +@overload +def einsum( + __subscripts: str, + *operands: _ArrayLikeComplex_co, + out: None = ..., + dtype: Optional[_DTypeLikeComplex] = ..., + order: _OrderKACF = ..., + casting: _CastingSafe = ..., + optimize: _OptimizeKind = ..., +) -> Any: ... +@overload +def einsum( + __subscripts: str, + *operands: Any, + casting: _CastingUnsafe, + dtype: Optional[_DTypeLikeComplex_co] = ..., + out: None = ..., + order: _OrderKACF = ..., + optimize: _OptimizeKind = ..., +) -> Any: ... +@overload +def einsum( + __subscripts: str, + *operands: _ArrayLikeComplex_co, + out: _ArrayType, + dtype: Optional[_DTypeLikeComplex_co] = ..., + order: _OrderKACF = ..., + casting: _CastingSafe = ..., + optimize: _OptimizeKind = ..., +) -> _ArrayType: ... +@overload +def einsum( + __subscripts: str, + *operands: Any, + out: _ArrayType, + casting: _CastingUnsafe, + dtype: Optional[_DTypeLikeComplex_co] = ..., + order: _OrderKACF = ..., + optimize: _OptimizeKind = ..., +) -> _ArrayType: ... + +# NOTE: `einsum_call` is a hidden kwarg unavailable for public use. +# It is therefore excluded from the signatures below. +# NOTE: In practice the list consists of a `str` (first element) +# and a variable number of integer tuples. +def einsum_path( + __subscripts: str, + *operands: _ArrayLikeComplex_co, + optimize: _OptimizeKind = ..., +) -> Tuple[List[Any], str]: ... diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index d65e26827751..65a42eb1ee72 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -10,8 +10,7 @@ from . import overrides from . import umath as um from . import numerictypes as nt -from ._asarray import asarray, array, asanyarray -from .multiarray import concatenate +from .multiarray import asarray, array, asanyarray, concatenate from . import _methods _dt_ = nt.sctype2char @@ -308,7 +307,7 @@ def _choose_dispatcher(a, choices, out=None, mode=None): @array_function_dispatch(_choose_dispatcher) def choose(a, choices, out=None, mode='raise'): """ - Construct an array from an index array and a set of arrays to choose from. + Construct an array from an index array and a list of arrays to choose from. First of all, if confused or uncertain, definitely look at the Examples - in its full generality, this function is less simple than it might @@ -319,34 +318,34 @@ def choose(a, choices, out=None, mode='raise'): But this omits some subtleties. Here is a fully general summary: - Given an "index" array (`a`) of integers and a sequence of `n` arrays + Given an "index" array (`a`) of integers and a sequence of ``n`` arrays (`choices`), `a` and each choice array are first broadcast, as necessary, to arrays of a common shape; calling these *Ba* and *Bchoices[i], i = 0,...,n-1* we have that, necessarily, ``Ba.shape == Bchoices[i].shape`` - for each `i`. Then, a new array with shape ``Ba.shape`` is created as + for each ``i``. Then, a new array with shape ``Ba.shape`` is created as follows: - * if ``mode=raise`` (the default), then, first of all, each element of - `a` (and thus `Ba`) must be in the range `[0, n-1]`; now, suppose that - `i` (in that range) is the value at the `(j0, j1, ..., jm)` position - in `Ba` - then the value at the same position in the new array is the - value in `Bchoices[i]` at that same position; + * if ``mode='raise'`` (the default), then, first of all, each element of + ``a`` (and thus ``Ba``) must be in the range ``[0, n-1]``; now, suppose + that ``i`` (in that range) is the value at the ``(j0, j1, ..., jm)`` + position in ``Ba`` - then the value at the same position in the new array + is the value in ``Bchoices[i]`` at that same position; - * if ``mode=wrap``, values in `a` (and thus `Ba`) may be any (signed) + * if ``mode='wrap'``, values in `a` (and thus `Ba`) may be any (signed) integer; modular arithmetic is used to map integers outside the range `[0, n-1]` back into that range; and then the new array is constructed as above; - * if ``mode=clip``, values in `a` (and thus `Ba`) may be any (signed) - integer; negative integers are mapped to 0; values greater than `n-1` - are mapped to `n-1`; and then the new array is constructed as above. + * if ``mode='clip'``, values in `a` (and thus ``Ba``) may be any (signed) + integer; negative integers are mapped to 0; values greater than ``n-1`` + are mapped to ``n-1``; and then the new array is constructed as above. Parameters ---------- a : int array - This array must contain integers in `[0, n-1]`, where `n` is the number - of choices, unless ``mode=wrap`` or ``mode=clip``, in which cases any - integers are permissible. + This array must contain integers in ``[0, n-1]``, where ``n`` is the + number of choices, unless ``mode=wrap`` or ``mode=clip``, in which + cases any integers are permissible. choices : sequence of arrays Choice arrays. `a` and all of the choices must be broadcastable to the same shape. If `choices` is itself an array (not recommended), then @@ -355,12 +354,12 @@ def choose(a, choices, out=None, mode='raise'): out : array, optional If provided, the result will be inserted into this array. It should be of the appropriate shape and dtype. Note that `out` is always - buffered if `mode='raise'`; use other modes for better performance. + buffered if ``mode='raise'``; use other modes for better performance. mode : {'raise' (default), 'wrap', 'clip'}, optional - Specifies how indices outside `[0, n-1]` will be treated: + Specifies how indices outside ``[0, n-1]`` will be treated: * 'raise' : an exception is raised - * 'wrap' : value becomes value mod `n` + * 'wrap' : value becomes value mod ``n`` * 'clip' : values < 0 are mapped to 0, values > n-1 are mapped to n-1 Returns @@ -606,6 +605,8 @@ def transpose(a, axes=None): For an array a with two axes, transpose(a) gives the matrix transpose. + Refer to `numpy.ndarray.transpose` for full documentation. + Parameters ---------- a : array_like @@ -625,6 +626,7 @@ def transpose(a, axes=None): See Also -------- + ndarray.transpose : Equivalent method moveaxis argsort @@ -1375,13 +1377,13 @@ def resize(a, new_shape): reshaped_array : ndarray The new array is formed from the data in the old array, repeated if necessary to fill out the required number of elements. The - data are repeated in the order that they are stored in memory. + data are repeated iterating over the array in C-order. See Also -------- np.reshape : Reshape an array without changing the total size. np.pad : Enlarge and pad an array. - np.repeat: Repeat elements of an array. + np.repeat : Repeat elements of an array. ndarray.resize : resize an array in-place. Notes @@ -1392,11 +1394,11 @@ def resize(a, new_shape): Warning: This functionality does **not** consider axes separately, i.e. it does not apply interpolation/extrapolation. - It fills the return array with the required number of elements, taken - from `a` as they are laid out in memory, disregarding strides and axes. - (This is in case the new shape is smaller. For larger, see above.) - This functionality is therefore not suitable to resize images, - or data where each axis represents a separate and distinct entity. + It fills the return array with the required number of elements, iterating + over `a` in C-order, disregarding axes (and cycling back from the start if + the new shape is larger). This functionality is therefore not suitable to + resize images, or data where each axis represents a separate and distinct + entity. Examples -------- @@ -2007,7 +2009,7 @@ def compress(condition, a, axis=None, out=None): -------- take, choose, diag, diagonal, select ndarray.compress : Equivalent method in ndarray - extract: Equivalent method when working on 1-D arrays + extract : Equivalent method when working on 1-D arrays :ref:`ufuncs-output-type` Examples @@ -2084,15 +2086,25 @@ def clip(a, a_min, a_max, out=None, **kwargs): -------- :ref:`ufuncs-output-type` + Notes + ----- + When `a_min` is greater than `a_max`, `clip` returns an + array in which all values are equal to `a_max`, + as shown in the second example. + Examples -------- >>> a = np.arange(10) - >>> np.clip(a, 1, 8) - array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8]) >>> a array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.clip(a, 1, 8) + array([1, 1, 2, 3, 4, 5, 6, 7, 8, 8]) + >>> np.clip(a, 8, 1) + array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) >>> np.clip(a, 3, 6, out=a) array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6]) + >>> a + array([3, 3, 3, 3, 4, 5, 6, 6, 6, 6]) >>> a = np.arange(10) >>> a array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -2475,20 +2487,21 @@ def cumsum(a, axis=None, dtype=None, out=None): result has the same size as `a`, and the same shape as `a` if `axis` is not None or `a` is a 1-d array. - See Also -------- sum : Sum array elements. - trapz : Integration of array values using the composite trapezoidal rule. - - diff : Calculate the n-th discrete difference along given axis. + diff : Calculate the n-th discrete difference along given axis. Notes ----- Arithmetic is modular when using integer types, and no error is raised on overflow. + ``cumsum(a)[-1]`` may not be equal to ``sum(a)`` for floating-point + values since ``sum`` may use a pairwise summation routine, reducing + the roundoff-error. See `sum` for more information. + Examples -------- >>> a = np.array([[1,2,3], [4,5,6]]) @@ -2507,6 +2520,14 @@ def cumsum(a, axis=None, dtype=None, out=None): array([[ 1, 3, 6], [ 4, 9, 15]]) + ``cumsum(b)[-1]`` may not be equal to ``sum(b)`` + + >>> b = np.array([1, 2e-9, 3e-9] * 1000000) + >>> b.cumsum()[-1] + 1000000.0050045159 + >>> b.sum() + 1000000.0050000029 + """ return _wrapfunc(a, 'cumsum', axis=axis, dtype=dtype, out=out) diff --git a/numpy/core/fromnumeric.pyi b/numpy/core/fromnumeric.pyi index 66eb3bfb892f..3342ec3ac47b 100644 --- a/numpy/core/fromnumeric.pyi +++ b/numpy/core/fromnumeric.pyi @@ -6,12 +6,11 @@ from numpy import ( ndarray, number, integer, + intp, bool_, generic, _OrderKACF, _OrderACF, - _ArrayLikeBool, - _ArrayLikeIntOrBool, _ModeKind, _PartitionKind, _SortKind, @@ -22,9 +21,9 @@ from numpy.typing import ( ArrayLike, _ShapeLike, _Shape, - _IntLike, - _BoolLike, - _NumberLike, + _ArrayLikeBool_co, + _ArrayLikeInt_co, + _NumberLike_co, ) if sys.version_info >= (3, 8): @@ -42,11 +41,7 @@ _ScalarBuiltin = Union[str, bytes, dt.date, dt.timedelta, bool, int, float, comp _Scalar = Union[_ScalarBuiltin, _ScalarNumpy] # Integers and booleans can generally be used interchangeably -_ScalarIntOrBool = TypeVar("_ScalarIntOrBool", bound=Union[integer, bool_]) _ScalarGeneric = TypeVar("_ScalarGeneric", bound=generic) -_ScalarGenericDT = TypeVar( - "_ScalarGenericDT", bound=Union[dt.datetime, dt.timedelta, generic] -) _Number = TypeVar("_Number", bound=number) @@ -55,144 +50,145 @@ _Number = TypeVar("_Number", bound=number) # 2. A scalar comes in; a generic comes out # 3. An array-like object comes in; some keyword ensures that a generic comes out # 4. An array-like object comes in; an ndarray or generic comes out -@overload -def take( - a: _ScalarGenericDT, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _ModeKind = ..., -) -> _ScalarGenericDT: ... -@overload -def take( - a: _Scalar, - indices: int, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _ModeKind = ..., -) -> _ScalarNumpy: ... -@overload def take( a: ArrayLike, - indices: int, + indices: _ArrayLikeInt_co, axis: Optional[int] = ..., out: Optional[ndarray] = ..., mode: _ModeKind = ..., -) -> _ScalarNumpy: ... -@overload -def take( +) -> Any: ... + +def reshape( a: ArrayLike, - indices: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - out: Optional[ndarray] = ..., - mode: _ModeKind = ..., -) -> Union[_ScalarNumpy, ndarray]: ... -def reshape(a: ArrayLike, newshape: _ShapeLike, order: _OrderACF = ...) -> ndarray: ... -@overload -def choose( - a: _ScalarIntOrBool, - choices: ArrayLike, - out: Optional[ndarray] = ..., - mode: _ModeKind = ..., -) -> _ScalarIntOrBool: ... -@overload -def choose( - a: Union[_IntLike, _BoolLike], choices: ArrayLike, out: Optional[ndarray] = ..., mode: _ModeKind = ... -) -> Union[integer, bool_]: ... -@overload + newshape: _ShapeLike, + order: _OrderACF = ..., +) -> ndarray: ... + def choose( - a: _ArrayLikeIntOrBool, + a: _ArrayLikeInt_co, choices: ArrayLike, out: Optional[ndarray] = ..., mode: _ModeKind = ..., -) -> ndarray: ... +) -> Any: ... + def repeat( - a: ArrayLike, repeats: _ArrayLikeIntOrBool, axis: Optional[int] = ... + a: ArrayLike, + repeats: _ArrayLikeInt_co, + axis: Optional[int] = ..., ) -> ndarray: ... + def put( - a: ndarray, ind: _ArrayLikeIntOrBool, v: ArrayLike, mode: _ModeKind = ... + a: ndarray, + ind: _ArrayLikeInt_co, + v: ArrayLike, + mode: _ModeKind = ..., ) -> None: ... -def swapaxes(a: ArrayLike, axis1: int, axis2: int) -> ndarray: ... + +def swapaxes( + a: ArrayLike, + axis1: int, + axis2: int, +) -> ndarray: ... + def transpose( - a: ArrayLike, axes: Union[None, Sequence[int], ndarray] = ... + a: ArrayLike, + axes: Union[None, Sequence[int], ndarray] = ... ) -> ndarray: ... + def partition( a: ArrayLike, - kth: _ArrayLikeIntOrBool, + kth: _ArrayLikeInt_co, axis: Optional[int] = ..., kind: _PartitionKind = ..., order: Union[None, str, Sequence[str]] = ..., ) -> ndarray: ... -@overload -def argpartition( - a: generic, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> integer: ... -@overload -def argpartition( - a: _ScalarBuiltin, - kth: _ArrayLikeIntOrBool, - axis: Optional[int] = ..., - kind: _PartitionKind = ..., - order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... -@overload + def argpartition( a: ArrayLike, - kth: _ArrayLikeIntOrBool, + kth: _ArrayLikeInt_co, axis: Optional[int] = ..., kind: _PartitionKind = ..., order: Union[None, str, Sequence[str]] = ..., -) -> ndarray: ... +) -> Any: ... + def sort( a: ArrayLike, axis: Optional[int] = ..., kind: Optional[_SortKind] = ..., order: Union[None, str, Sequence[str]] = ..., ) -> ndarray: ... + def argsort( a: ArrayLike, axis: Optional[int] = ..., kind: Optional[_SortKind] = ..., order: Union[None, str, Sequence[str]] = ..., ) -> ndarray: ... + @overload -def argmax(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... +def argmax( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., +) -> intp: ... @overload def argmax( - a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... -) -> Union[integer, ndarray]: ... + a: ArrayLike, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., +) -> Any: ... + @overload -def argmin(a: ArrayLike, axis: None = ..., out: Optional[ndarray] = ...) -> integer: ... +def argmin( + a: ArrayLike, + axis: None = ..., + out: Optional[ndarray] = ..., +) -> intp: ... @overload def argmin( - a: ArrayLike, axis: int = ..., out: Optional[ndarray] = ... -) -> Union[integer, ndarray]: ... + a: ArrayLike, + axis: Optional[int] = ..., + out: Optional[ndarray] = ..., +) -> Any: ... + @overload def searchsorted( a: ArrayLike, v: _Scalar, side: _SortSide = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array -) -> integer: ... + sorter: Optional[_ArrayLikeInt_co] = ..., # 1D int array +) -> intp: ... @overload def searchsorted( a: ArrayLike, v: ArrayLike, side: _SortSide = ..., - sorter: Optional[_ArrayLikeIntOrBool] = ..., # 1D int array + sorter: Optional[_ArrayLikeInt_co] = ..., # 1D int array ) -> ndarray: ... -def resize(a: ArrayLike, new_shape: _ShapeLike) -> ndarray: ... + +def resize( + a: ArrayLike, + new_shape: _ShapeLike, +) -> ndarray: ... + @overload -def squeeze(a: _ScalarGeneric, axis: Optional[_ShapeLike] = ...) -> _ScalarGeneric: ... +def squeeze( + a: _ScalarGeneric, + axis: Optional[_ShapeLike] = ..., +) -> _ScalarGeneric: ... @overload -def squeeze(a: ArrayLike, axis: Optional[_ShapeLike] = ...) -> ndarray: ... +def squeeze( + a: ArrayLike, + axis: Optional[_ShapeLike] = ..., +) -> ndarray: ... + def diagonal( - a: ArrayLike, offset: int = ..., axis1: int = ..., axis2: int = ... # >= 2D array + a: ArrayLike, + offset: int = ..., + axis1: int = ..., + axis2: int = ..., # >= 2D array ) -> ndarray: ... + def trace( a: ArrayLike, # >= 2D array offset: int = ..., @@ -200,32 +196,21 @@ def trace( axis2: int = ..., dtype: DTypeLike = ..., out: Optional[ndarray] = ..., -) -> Union[number, ndarray]: ... +) -> Any: ... + def ravel(a: ArrayLike, order: _OrderKACF = ...) -> ndarray: ... + def nonzero(a: ArrayLike) -> Tuple[ndarray, ...]: ... + def shape(a: ArrayLike) -> _Shape: ... + def compress( condition: ArrayLike, # 1D bool array a: ArrayLike, axis: Optional[int] = ..., out: Optional[ndarray] = ..., ) -> ndarray: ... -@overload -def clip( - a: _Number, - a_min: ArrayLike, - a_max: Optional[ArrayLike], - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> _Number: ... -@overload -def clip( - a: _Number, - a_min: None, - a_max: ArrayLike, - out: Optional[ndarray] = ..., - **kwargs: Any, -) -> _Number: ... + @overload def clip( a: ArrayLike, @@ -233,7 +218,7 @@ def clip( a_max: Optional[ArrayLike], out: Optional[ndarray] = ..., **kwargs: Any, -) -> Union[number, ndarray]: ... +) -> Any: ... @overload def clip( a: ArrayLike, @@ -241,32 +226,23 @@ def clip( a_max: ArrayLike, out: Optional[ndarray] = ..., **kwargs: Any, -) -> Union[number, ndarray]: ... -@overload -def sum( - a: _Number, - axis: Optional[_ShapeLike] = ..., - dtype: DTypeLike = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload +) -> Any: ... + def sum( a: ArrayLike, axis: _ShapeLike = ..., dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., +) -> Any: ... + @overload def all( a: ArrayLike, axis: None = ..., - out: Optional[ndarray] = ..., + out: None = ..., keepdims: Literal[False] = ..., ) -> bool_: ... @overload @@ -275,12 +251,13 @@ def all( axis: Optional[_ShapeLike] = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., -) -> Union[bool_, ndarray]: ... +) -> Any: ... + @overload def any( a: ArrayLike, axis: None = ..., - out: Optional[ndarray] = ..., + out: None = ..., keepdims: Literal[False] = ..., ) -> bool_: ... @overload @@ -289,88 +266,39 @@ def any( axis: Optional[_ShapeLike] = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., -) -> Union[bool_, ndarray]: ... +) -> Any: ... + def cumsum( a: ArrayLike, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ) -> ndarray: ... -@overload -def ptp( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., -) -> _Number: ... -@overload -def ptp( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload + def ptp( a: ArrayLike, axis: Optional[_ShapeLike] = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def amax( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def amax( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload +) -> Any: ... + def amax( a: ArrayLike, axis: Optional[_ShapeLike] = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... -@overload -def amin( - a: _Number, - axis: Optional[_ShapeLike] = ..., - out: Optional[ndarray] = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def amin( - a: ArrayLike, - axis: None = ..., - out: Optional[ndarray] = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., +) -> Any: ... + def amin( a: ArrayLike, axis: Optional[_ShapeLike] = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., +) -> Any: ... # TODO: `np.prod()``: For object arrays `initial` does not necessarily # have to be a numerical scalar. @@ -379,82 +307,41 @@ def amin( # Note that the same situation holds for all wrappers around # `np.ufunc.reduce`, e.g. `np.sum()` (`.__add__()`). -@overload -def prod( - a: _Number, - axis: Optional[_ShapeLike] = ..., - dtype: DTypeLike = ..., - out: None = ..., - keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> _Number: ... -@overload -def prod( - a: ArrayLike, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> number: ... -@overload def prod( a: ArrayLike, axis: Optional[_ShapeLike] = ..., dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., - initial: _NumberLike = ..., - where: _ArrayLikeBool = ..., -) -> Union[number, ndarray]: ... + initial: _NumberLike_co = ..., + where: _ArrayLikeBool_co = ..., +) -> Any: ... + def cumprod( a: ArrayLike, axis: Optional[int] = ..., dtype: DTypeLike = ..., out: Optional[ndarray] = ..., ) -> ndarray: ... + def ndim(a: ArrayLike) -> int: ... + def size(a: ArrayLike, axis: Optional[int] = ...) -> int: ... -@overload -def around( - a: _Number, decimals: int = ..., out: Optional[ndarray] = ... -) -> _Number: ... -@overload -def around( - a: _NumberLike, decimals: int = ..., out: Optional[ndarray] = ... -) -> number: ... -@overload + def around( - a: ArrayLike, decimals: int = ..., out: Optional[ndarray] = ... -) -> ndarray: ... -@overload -def mean( a: ArrayLike, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload + decimals: int = ..., + out: Optional[ndarray] = ..., +) -> Any: ... + def mean( a: ArrayLike, axis: Optional[_ShapeLike] = ..., dtype: DTypeLike = ..., out: Optional[ndarray] = ..., keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def std( - a: ArrayLike, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload +) -> Any: ... + def std( a: ArrayLike, axis: Optional[_ShapeLike] = ..., @@ -462,17 +349,8 @@ def std( out: Optional[ndarray] = ..., ddof: int = ..., keepdims: bool = ..., -) -> Union[number, ndarray]: ... -@overload -def var( - a: ArrayLike, - axis: None = ..., - dtype: DTypeLike = ..., - out: None = ..., - ddof: int = ..., - keepdims: Literal[False] = ..., -) -> number: ... -@overload +) -> Any: ... + def var( a: ArrayLike, axis: Optional[_ShapeLike] = ..., @@ -480,4 +358,4 @@ def var( out: Optional[ndarray] = ..., ddof: int = ..., keepdims: bool = ..., -) -> Union[number, ndarray]: ... +) -> Any: ... diff --git a/numpy/core/function_base.py b/numpy/core/function_base.py index 8a1fee99b9d6..e940ac230537 100644 --- a/numpy/core/function_base.py +++ b/numpy/core/function_base.py @@ -371,7 +371,7 @@ def geomspace(start, stop, num=50, endpoint=True, dtype=None, axis=0): 6.12323400e-17+1.00000000e+00j, 7.07106781e-01+7.07106781e-01j, 1.00000000e+00+0.00000000e+00j]) - Graphical illustration of ``endpoint`` parameter: + Graphical illustration of `endpoint` parameter: >>> import matplotlib.pyplot as plt >>> N = 10 diff --git a/numpy/core/function_base.pyi b/numpy/core/function_base.pyi index 1490bed4aff3..b5d6ca6abe88 100644 --- a/numpy/core/function_base.pyi +++ b/numpy/core/function_base.pyi @@ -1,21 +1,18 @@ import sys from typing import overload, Tuple, Union, Sequence, Any -from numpy import ndarray, inexact -from numpy.typing import ArrayLike, DTypeLike, _SupportsArray, _NumberLike +from numpy import ndarray +from numpy.typing import ArrayLike, DTypeLike, _SupportsArray, _NumberLike_co if sys.version_info >= (3, 8): from typing import SupportsIndex, Literal else: - from typing_extensions import Literal, Protocol - - class SupportsIndex(Protocol): - def __index__(self) -> int: ... + from typing_extensions import SupportsIndex, Literal # TODO: wait for support for recursive types _ArrayLikeNested = Sequence[Sequence[Any]] _ArrayLikeNumber = Union[ - _NumberLike, Sequence[_NumberLike], ndarray, _SupportsArray, _ArrayLikeNested + _NumberLike_co, Sequence[_NumberLike_co], ndarray, _SupportsArray, _ArrayLikeNested ] @overload def linspace( @@ -36,7 +33,8 @@ def linspace( retstep: Literal[True] = ..., dtype: DTypeLike = ..., axis: SupportsIndex = ..., -) -> Tuple[ndarray, inexact]: ... +) -> Tuple[ndarray, Any]: ... + def logspace( start: _ArrayLikeNumber, stop: _ArrayLikeNumber, @@ -46,6 +44,7 @@ def logspace( dtype: DTypeLike = ..., axis: SupportsIndex = ..., ) -> ndarray: ... + def geomspace( start: _ArrayLikeNumber, stop: _ArrayLikeNumber, diff --git a/numpy/core/include/numpy/libdivide/LICENSE.txt b/numpy/core/include/numpy/libdivide/LICENSE.txt new file mode 100644 index 000000000000..d72a7c388d40 --- /dev/null +++ b/numpy/core/include/numpy/libdivide/LICENSE.txt @@ -0,0 +1,21 @@ + zlib License + ------------ + + Copyright (C) 2010 - 2019 ridiculous_fish, + Copyright (C) 2016 - 2019 Kim Walisch, + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. diff --git a/numpy/core/include/numpy/libdivide/libdivide.h b/numpy/core/include/numpy/libdivide/libdivide.h new file mode 100644 index 000000000000..81057b7b43de --- /dev/null +++ b/numpy/core/include/numpy/libdivide/libdivide.h @@ -0,0 +1,2079 @@ +// libdivide.h - Optimized integer division +// https://libdivide.com +// +// Copyright (C) 2010 - 2019 ridiculous_fish, +// Copyright (C) 2016 - 2019 Kim Walisch, +// +// libdivide is dual-licensed under the Boost or zlib licenses. +// You may use libdivide under the terms of either of these. +// See LICENSE.txt for more details. + +#ifndef LIBDIVIDE_H +#define LIBDIVIDE_H + +#define LIBDIVIDE_VERSION "3.0" +#define LIBDIVIDE_VERSION_MAJOR 3 +#define LIBDIVIDE_VERSION_MINOR 0 + +#include + +#if defined(__cplusplus) + #include + #include + #include +#else + #include + #include +#endif + +#if defined(LIBDIVIDE_AVX512) + #include +#elif defined(LIBDIVIDE_AVX2) + #include +#elif defined(LIBDIVIDE_SSE2) + #include +#endif + +#if defined(_MSC_VER) + #include + // disable warning C4146: unary minus operator applied + // to unsigned type, result still unsigned + #pragma warning(disable: 4146) + #define LIBDIVIDE_VC +#endif + +#if !defined(__has_builtin) + #define __has_builtin(x) 0 +#endif + +#if defined(__SIZEOF_INT128__) + #define HAS_INT128_T + // clang-cl on Windows does not yet support 128-bit division + #if !(defined(__clang__) && defined(LIBDIVIDE_VC)) + #define HAS_INT128_DIV + #endif +#endif + +#if defined(__x86_64__) || defined(_M_X64) + #define LIBDIVIDE_X86_64 +#endif + +#if defined(__i386__) + #define LIBDIVIDE_i386 +#endif + +#if defined(__GNUC__) || defined(__clang__) + #define LIBDIVIDE_GCC_STYLE_ASM +#endif + +#if defined(__cplusplus) || defined(LIBDIVIDE_VC) + #define LIBDIVIDE_FUNCTION __FUNCTION__ +#else + #define LIBDIVIDE_FUNCTION __func__ +#endif + +#define LIBDIVIDE_ERROR(msg) \ + do { \ + fprintf(stderr, "libdivide.h:%d: %s(): Error: %s\n", \ + __LINE__, LIBDIVIDE_FUNCTION, msg); \ + abort(); \ + } while (0) + +#if defined(LIBDIVIDE_ASSERTIONS_ON) + #define LIBDIVIDE_ASSERT(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "libdivide.h:%d: %s(): Assertion failed: %s\n", \ + __LINE__, LIBDIVIDE_FUNCTION, #x); \ + abort(); \ + } \ + } while (0) +#else + #define LIBDIVIDE_ASSERT(x) +#endif + +#ifdef __cplusplus +namespace libdivide { +#endif + +// pack divider structs to prevent compilers from padding. +// This reduces memory usage by up to 43% when using a large +// array of libdivide dividers and improves performance +// by up to 10% because of reduced memory bandwidth. +#pragma pack(push, 1) + +struct libdivide_u32_t { + uint32_t magic; + uint8_t more; +}; + +struct libdivide_s32_t { + int32_t magic; + uint8_t more; +}; + +struct libdivide_u64_t { + uint64_t magic; + uint8_t more; +}; + +struct libdivide_s64_t { + int64_t magic; + uint8_t more; +}; + +struct libdivide_u32_branchfree_t { + uint32_t magic; + uint8_t more; +}; + +struct libdivide_s32_branchfree_t { + int32_t magic; + uint8_t more; +}; + +struct libdivide_u64_branchfree_t { + uint64_t magic; + uint8_t more; +}; + +struct libdivide_s64_branchfree_t { + int64_t magic; + uint8_t more; +}; + +#pragma pack(pop) + +// Explanation of the "more" field: +// +// * Bits 0-5 is the shift value (for shift path or mult path). +// * Bit 6 is the add indicator for mult path. +// * Bit 7 is set if the divisor is negative. We use bit 7 as the negative +// divisor indicator so that we can efficiently use sign extension to +// create a bitmask with all bits set to 1 (if the divisor is negative) +// or 0 (if the divisor is positive). +// +// u32: [0-4] shift value +// [5] ignored +// [6] add indicator +// magic number of 0 indicates shift path +// +// s32: [0-4] shift value +// [5] ignored +// [6] add indicator +// [7] indicates negative divisor +// magic number of 0 indicates shift path +// +// u64: [0-5] shift value +// [6] add indicator +// magic number of 0 indicates shift path +// +// s64: [0-5] shift value +// [6] add indicator +// [7] indicates negative divisor +// magic number of 0 indicates shift path +// +// In s32 and s64 branchfree modes, the magic number is negated according to +// whether the divisor is negated. In branchfree strategy, it is not negated. + +enum { + LIBDIVIDE_32_SHIFT_MASK = 0x1F, + LIBDIVIDE_64_SHIFT_MASK = 0x3F, + LIBDIVIDE_ADD_MARKER = 0x40, + LIBDIVIDE_NEGATIVE_DIVISOR = 0x80 +}; + +static inline struct libdivide_s32_t libdivide_s32_gen(int32_t d); +static inline struct libdivide_u32_t libdivide_u32_gen(uint32_t d); +static inline struct libdivide_s64_t libdivide_s64_gen(int64_t d); +static inline struct libdivide_u64_t libdivide_u64_gen(uint64_t d); + +static inline struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d); +static inline struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d); +static inline struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d); +static inline struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d); + +static inline int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom); +static inline uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom); +static inline int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom); +static inline uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom); + +static inline int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom); +static inline uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom); +static inline int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom); +static inline uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom); + +static inline int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom); +static inline uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom); +static inline int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom); +static inline uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom); + +static inline int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom); +static inline uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom); +static inline int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom); +static inline uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom); + +//////// Internal Utility Functions + +static inline uint32_t libdivide_mullhi_u32(uint32_t x, uint32_t y) { + uint64_t xl = x, yl = y; + uint64_t rl = xl * yl; + return (uint32_t)(rl >> 32); +} + +static inline int32_t libdivide_mullhi_s32(int32_t x, int32_t y) { + int64_t xl = x, yl = y; + int64_t rl = xl * yl; + // needs to be arithmetic shift + return (int32_t)(rl >> 32); +} + +static inline uint64_t libdivide_mullhi_u64(uint64_t x, uint64_t y) { +#if defined(LIBDIVIDE_VC) && \ + defined(LIBDIVIDE_X86_64) + return __umulh(x, y); +#elif defined(HAS_INT128_T) + __uint128_t xl = x, yl = y; + __uint128_t rl = xl * yl; + return (uint64_t)(rl >> 64); +#else + // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) + uint32_t mask = 0xFFFFFFFF; + uint32_t x0 = (uint32_t)(x & mask); + uint32_t x1 = (uint32_t)(x >> 32); + uint32_t y0 = (uint32_t)(y & mask); + uint32_t y1 = (uint32_t)(y >> 32); + uint32_t x0y0_hi = libdivide_mullhi_u32(x0, y0); + uint64_t x0y1 = x0 * (uint64_t)y1; + uint64_t x1y0 = x1 * (uint64_t)y0; + uint64_t x1y1 = x1 * (uint64_t)y1; + uint64_t temp = x1y0 + x0y0_hi; + uint64_t temp_lo = temp & mask; + uint64_t temp_hi = temp >> 32; + + return x1y1 + temp_hi + ((temp_lo + x0y1) >> 32); +#endif +} + +static inline int64_t libdivide_mullhi_s64(int64_t x, int64_t y) { +#if defined(LIBDIVIDE_VC) && \ + defined(LIBDIVIDE_X86_64) + return __mulh(x, y); +#elif defined(HAS_INT128_T) + __int128_t xl = x, yl = y; + __int128_t rl = xl * yl; + return (int64_t)(rl >> 64); +#else + // full 128 bits are x0 * y0 + (x0 * y1 << 32) + (x1 * y0 << 32) + (x1 * y1 << 64) + uint32_t mask = 0xFFFFFFFF; + uint32_t x0 = (uint32_t)(x & mask); + uint32_t y0 = (uint32_t)(y & mask); + int32_t x1 = (int32_t)(x >> 32); + int32_t y1 = (int32_t)(y >> 32); + uint32_t x0y0_hi = libdivide_mullhi_u32(x0, y0); + int64_t t = x1 * (int64_t)y0 + x0y0_hi; + int64_t w1 = x0 * (int64_t)y1 + (t & mask); + + return x1 * (int64_t)y1 + (t >> 32) + (w1 >> 32); +#endif +} + +static inline int32_t libdivide_count_leading_zeros32(uint32_t val) { +#if defined(__GNUC__) || \ + __has_builtin(__builtin_clz) + // Fast way to count leading zeros + return __builtin_clz(val); +#elif defined(LIBDIVIDE_VC) + unsigned long result; + if (_BitScanReverse(&result, val)) { + return 31 - result; + } + return 0; +#else + if (val == 0) + return 32; + int32_t result = 8; + uint32_t hi = 0xFFU << 24; + while ((val & hi) == 0) { + hi >>= 8; + result += 8; + } + while (val & hi) { + result -= 1; + hi <<= 1; + } + return result; +#endif +} + +static inline int32_t libdivide_count_leading_zeros64(uint64_t val) { +#if defined(__GNUC__) || \ + __has_builtin(__builtin_clzll) + // Fast way to count leading zeros + return __builtin_clzll(val); +#elif defined(LIBDIVIDE_VC) && defined(_WIN64) + unsigned long result; + if (_BitScanReverse64(&result, val)) { + return 63 - result; + } + return 0; +#else + uint32_t hi = val >> 32; + uint32_t lo = val & 0xFFFFFFFF; + if (hi != 0) return libdivide_count_leading_zeros32(hi); + return 32 + libdivide_count_leading_zeros32(lo); +#endif +} + +// libdivide_64_div_32_to_32: divides a 64-bit uint {u1, u0} by a 32-bit +// uint {v}. The result must fit in 32 bits. +// Returns the quotient directly and the remainder in *r +static inline uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { +#if (defined(LIBDIVIDE_i386) || defined(LIBDIVIDE_X86_64)) && \ + defined(LIBDIVIDE_GCC_STYLE_ASM) + uint32_t result; + __asm__("divl %[v]" + : "=a"(result), "=d"(*r) + : [v] "r"(v), "a"(u0), "d"(u1) + ); + return result; +#else + uint64_t n = ((uint64_t)u1 << 32) | u0; + uint32_t result = (uint32_t)(n / v); + *r = (uint32_t)(n - result * (uint64_t)v); + return result; +#endif +} + +// libdivide_128_div_64_to_64: divides a 128-bit uint {u1, u0} by a 64-bit +// uint {v}. The result must fit in 64 bits. +// Returns the quotient directly and the remainder in *r +static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, uint64_t *r) { +#if defined(LIBDIVIDE_X86_64) && \ + defined(LIBDIVIDE_GCC_STYLE_ASM) + uint64_t result; + __asm__("divq %[v]" + : "=a"(result), "=d"(*r) + : [v] "r"(v), "a"(u0), "d"(u1) + ); + return result; +#elif defined(HAS_INT128_T) && \ + defined(HAS_INT128_DIV) + __uint128_t n = ((__uint128_t)u1 << 64) | u0; + uint64_t result = (uint64_t)(n / v); + *r = (uint64_t)(n - result * (__uint128_t)v); + return result; +#else + // Code taken from Hacker's Delight: + // http://www.hackersdelight.org/HDcode/divlu.c. + // License permits inclusion here per: + // http://www.hackersdelight.org/permissions.htm + + const uint64_t b = (1ULL << 32); // Number base (32 bits) + uint64_t un1, un0; // Norm. dividend LSD's + uint64_t vn1, vn0; // Norm. divisor digits + uint64_t q1, q0; // Quotient digits + uint64_t un64, un21, un10; // Dividend digit pairs + uint64_t rhat; // A remainder + int32_t s; // Shift amount for norm + + // If overflow, set rem. to an impossible value, + // and return the largest possible quotient + if (u1 >= v) { + *r = (uint64_t) -1; + return (uint64_t) -1; + } + + // count leading zeros + s = libdivide_count_leading_zeros64(v); + if (s > 0) { + // Normalize divisor + v = v << s; + un64 = (u1 << s) | (u0 >> (64 - s)); + un10 = u0 << s; // Shift dividend left + } else { + // Avoid undefined behavior of (u0 >> 64). + // The behavior is undefined if the right operand is + // negative, or greater than or equal to the length + // in bits of the promoted left operand. + un64 = u1; + un10 = u0; + } + + // Break divisor up into two 32-bit digits + vn1 = v >> 32; + vn0 = v & 0xFFFFFFFF; + + // Break right half of dividend into two digits + un1 = un10 >> 32; + un0 = un10 & 0xFFFFFFFF; + + // Compute the first quotient digit, q1 + q1 = un64 / vn1; + rhat = un64 - q1 * vn1; + + while (q1 >= b || q1 * vn0 > b * rhat + un1) { + q1 = q1 - 1; + rhat = rhat + vn1; + if (rhat >= b) + break; + } + + // Multiply and subtract + un21 = un64 * b + un1 - q1 * v; + + // Compute the second quotient digit + q0 = un21 / vn1; + rhat = un21 - q0 * vn1; + + while (q0 >= b || q0 * vn0 > b * rhat + un0) { + q0 = q0 - 1; + rhat = rhat + vn1; + if (rhat >= b) + break; + } + + *r = (un21 * b + un0 - q0 * v) >> s; + return q1 * b + q0; +#endif +} + +// Bitshift a u128 in place, left (signed_shift > 0) or right (signed_shift < 0) +static inline void libdivide_u128_shift(uint64_t *u1, uint64_t *u0, int32_t signed_shift) { + if (signed_shift > 0) { + uint32_t shift = signed_shift; + *u1 <<= shift; + *u1 |= *u0 >> (64 - shift); + *u0 <<= shift; + } + else if (signed_shift < 0) { + uint32_t shift = -signed_shift; + *u0 >>= shift; + *u0 |= *u1 << (64 - shift); + *u1 >>= shift; + } +} + +// Computes a 128 / 128 -> 64 bit division, with a 128 bit remainder. +static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64_t v_hi, uint64_t v_lo, uint64_t *r_hi, uint64_t *r_lo) { +#if defined(HAS_INT128_T) && \ + defined(HAS_INT128_DIV) + __uint128_t ufull = u_hi; + __uint128_t vfull = v_hi; + ufull = (ufull << 64) | u_lo; + vfull = (vfull << 64) | v_lo; + uint64_t res = (uint64_t)(ufull / vfull); + __uint128_t remainder = ufull - (vfull * res); + *r_lo = (uint64_t)remainder; + *r_hi = (uint64_t)(remainder >> 64); + return res; +#else + // Adapted from "Unsigned Doubleword Division" in Hacker's Delight + // We want to compute u / v + typedef struct { uint64_t hi; uint64_t lo; } u128_t; + u128_t u = {u_hi, u_lo}; + u128_t v = {v_hi, v_lo}; + + if (v.hi == 0) { + // divisor v is a 64 bit value, so we just need one 128/64 division + // Note that we are simpler than Hacker's Delight here, because we know + // the quotient fits in 64 bits whereas Hacker's Delight demands a full + // 128 bit quotient + *r_hi = 0; + return libdivide_128_div_64_to_64(u.hi, u.lo, v.lo, r_lo); + } + // Here v >= 2**64 + // We know that v.hi != 0, so count leading zeros is OK + // We have 0 <= n <= 63 + uint32_t n = libdivide_count_leading_zeros64(v.hi); + + // Normalize the divisor so its MSB is 1 + u128_t v1t = v; + libdivide_u128_shift(&v1t.hi, &v1t.lo, n); + uint64_t v1 = v1t.hi; // i.e. v1 = v1t >> 64 + + // To ensure no overflow + u128_t u1 = u; + libdivide_u128_shift(&u1.hi, &u1.lo, -1); + + // Get quotient from divide unsigned insn. + uint64_t rem_ignored; + uint64_t q1 = libdivide_128_div_64_to_64(u1.hi, u1.lo, v1, &rem_ignored); + + // Undo normalization and division of u by 2. + u128_t q0 = {0, q1}; + libdivide_u128_shift(&q0.hi, &q0.lo, n); + libdivide_u128_shift(&q0.hi, &q0.lo, -63); + + // Make q0 correct or too small by 1 + // Equivalent to `if (q0 != 0) q0 = q0 - 1;` + if (q0.hi != 0 || q0.lo != 0) { + q0.hi -= (q0.lo == 0); // borrow + q0.lo -= 1; + } + + // Now q0 is correct. + // Compute q0 * v as q0v + // = (q0.hi << 64 + q0.lo) * (v.hi << 64 + v.lo) + // = (q0.hi * v.hi << 128) + (q0.hi * v.lo << 64) + + // (q0.lo * v.hi << 64) + q0.lo * v.lo) + // Each term is 128 bit + // High half of full product (upper 128 bits!) are dropped + u128_t q0v = {0, 0}; + q0v.hi = q0.hi*v.lo + q0.lo*v.hi + libdivide_mullhi_u64(q0.lo, v.lo); + q0v.lo = q0.lo*v.lo; + + // Compute u - q0v as u_q0v + // This is the remainder + u128_t u_q0v = u; + u_q0v.hi -= q0v.hi + (u.lo < q0v.lo); // second term is borrow + u_q0v.lo -= q0v.lo; + + // Check if u_q0v >= v + // This checks if our remainder is larger than the divisor + if ((u_q0v.hi > v.hi) || + (u_q0v.hi == v.hi && u_q0v.lo >= v.lo)) { + // Increment q0 + q0.lo += 1; + q0.hi += (q0.lo == 0); // carry + + // Subtract v from remainder + u_q0v.hi -= v.hi + (u_q0v.lo < v.lo); + u_q0v.lo -= v.lo; + } + + *r_hi = u_q0v.hi; + *r_lo = u_q0v.lo; + + LIBDIVIDE_ASSERT(q0.hi == 0); + return q0.lo; +#endif +} + +////////// UINT32 + +static inline struct libdivide_u32_t libdivide_internal_u32_gen(uint32_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } + + struct libdivide_u32_t result; + uint32_t floor_log_2_d = 31 - libdivide_count_leading_zeros32(d); + + // Power of 2 + if ((d & (d - 1)) == 0) { + // We need to subtract 1 from the shift value in case of an unsigned + // branchfree divider because there is a hardcoded right shift by 1 + // in its division algorithm. Because of this we also need to add back + // 1 in its recovery algorithm. + result.magic = 0; + result.more = (uint8_t)(floor_log_2_d - (branchfree != 0)); + } else { + uint8_t more; + uint32_t rem, proposed_m; + proposed_m = libdivide_64_div_32_to_32(1U << floor_log_2_d, 0, d, &rem); + + LIBDIVIDE_ASSERT(rem > 0 && rem < d); + const uint32_t e = d - rem; + + // This power works if e < 2**floor_log_2_d. + if (!branchfree && (e < (1U << floor_log_2_d))) { + // This power works + more = floor_log_2_d; + } else { + // We have to use the general 33-bit algorithm. We need to compute + // (2**power) / d. However, we already have (2**(power-1))/d and + // its remainder. By doubling both, and then correcting the + // remainder, we can compute the larger division. + // don't care about overflow here - in fact, we expect it + proposed_m += proposed_m; + const uint32_t twice_rem = rem + rem; + if (twice_rem >= d || twice_rem < rem) proposed_m += 1; + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; + } + result.magic = 1 + proposed_m; + result.more = more; + // result.more's shift should in general be ceil_log_2_d. But if we + // used the smaller power, we subtract one from the shift because we're + // using the smaller power. If we're using the larger power, we + // subtract one from the shift because it's taken care of by the add + // indicator. So floor_log_2_d happens to be correct in both cases. + } + return result; +} + +struct libdivide_u32_t libdivide_u32_gen(uint32_t d) { + return libdivide_internal_u32_gen(d, 0); +} + +struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } + struct libdivide_u32_t tmp = libdivide_internal_u32_gen(d, 1); + struct libdivide_u32_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_32_SHIFT_MASK)}; + return ret; +} + +uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return numer >> more; + } + else { + uint32_t q = libdivide_mullhi_u32(denom->magic, numer); + if (more & LIBDIVIDE_ADD_MARKER) { + uint32_t t = ((numer - q) >> 1) + q; + return t >> (more & LIBDIVIDE_32_SHIFT_MASK); + } + else { + // All upper bits are 0, + // don't need to mask them off. + return q >> more; + } + } +} + +uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom) { + uint32_t q = libdivide_mullhi_u32(denom->magic, numer); + uint32_t t = ((numer - q) >> 1) + q; + return t >> denom->more; +} + +uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + + if (!denom->magic) { + return 1U << shift; + } else if (!(more & LIBDIVIDE_ADD_MARKER)) { + // We compute q = n/d = n*m / 2^(32 + shift) + // Therefore we have d = 2^(32 + shift) / m + // We need to ceil it. + // We know d is not a power of 2, so m is not a power of 2, + // so we can just add 1 to the floor + uint32_t hi_dividend = 1U << shift; + uint32_t rem_ignored; + return 1 + libdivide_64_div_32_to_32(hi_dividend, 0, denom->magic, &rem_ignored); + } else { + // Here we wish to compute d = 2^(32+shift+1)/(m+2^32). + // Notice (m + 2^32) is a 33 bit number. Use 64 bit division for now + // Also note that shift may be as high as 31, so shift + 1 will + // overflow. So we have to compute it as 2^(32+shift)/(m+2^32), and + // then double the quotient and remainder. + uint64_t half_n = 1ULL << (32 + shift); + uint64_t d = (1ULL << 32) | denom->magic; + // Note that the quotient is guaranteed <= 32 bits, but the remainder + // may need 33! + uint32_t half_q = (uint32_t)(half_n / d); + uint64_t rem = half_n % d; + // We computed 2^(32+shift)/(m+2^32) + // Need to double it, and then add 1 to the quotient if doubling th + // remainder would increase the quotient. + // Note that rem<<1 cannot overflow, since rem < d and d is 33 bits + uint32_t full_q = half_q + half_q + ((rem<<1) >= d); + + // We rounded down in gen (hence +1) + return full_q + 1; + } +} + +uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + + if (!denom->magic) { + return 1U << (shift + 1); + } else { + // Here we wish to compute d = 2^(32+shift+1)/(m+2^32). + // Notice (m + 2^32) is a 33 bit number. Use 64 bit division for now + // Also note that shift may be as high as 31, so shift + 1 will + // overflow. So we have to compute it as 2^(32+shift)/(m+2^32), and + // then double the quotient and remainder. + uint64_t half_n = 1ULL << (32 + shift); + uint64_t d = (1ULL << 32) | denom->magic; + // Note that the quotient is guaranteed <= 32 bits, but the remainder + // may need 33! + uint32_t half_q = (uint32_t)(half_n / d); + uint64_t rem = half_n % d; + // We computed 2^(32+shift)/(m+2^32) + // Need to double it, and then add 1 to the quotient if doubling th + // remainder would increase the quotient. + // Note that rem<<1 cannot overflow, since rem < d and d is 33 bits + uint32_t full_q = half_q + half_q + ((rem<<1) >= d); + + // We rounded down in gen (hence +1) + return full_q + 1; + } +} + +/////////// UINT64 + +static inline struct libdivide_u64_t libdivide_internal_u64_gen(uint64_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } + + struct libdivide_u64_t result; + uint32_t floor_log_2_d = 63 - libdivide_count_leading_zeros64(d); + + // Power of 2 + if ((d & (d - 1)) == 0) { + // We need to subtract 1 from the shift value in case of an unsigned + // branchfree divider because there is a hardcoded right shift by 1 + // in its division algorithm. Because of this we also need to add back + // 1 in its recovery algorithm. + result.magic = 0; + result.more = (uint8_t)(floor_log_2_d - (branchfree != 0)); + } else { + uint64_t proposed_m, rem; + uint8_t more; + // (1 << (64 + floor_log_2_d)) / d + proposed_m = libdivide_128_div_64_to_64(1ULL << floor_log_2_d, 0, d, &rem); + + LIBDIVIDE_ASSERT(rem > 0 && rem < d); + const uint64_t e = d - rem; + + // This power works if e < 2**floor_log_2_d. + if (!branchfree && e < (1ULL << floor_log_2_d)) { + // This power works + more = floor_log_2_d; + } else { + // We have to use the general 65-bit algorithm. We need to compute + // (2**power) / d. However, we already have (2**(power-1))/d and + // its remainder. By doubling both, and then correcting the + // remainder, we can compute the larger division. + // don't care about overflow here - in fact, we expect it + proposed_m += proposed_m; + const uint64_t twice_rem = rem + rem; + if (twice_rem >= d || twice_rem < rem) proposed_m += 1; + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; + } + result.magic = 1 + proposed_m; + result.more = more; + // result.more's shift should in general be ceil_log_2_d. But if we + // used the smaller power, we subtract one from the shift because we're + // using the smaller power. If we're using the larger power, we + // subtract one from the shift because it's taken care of by the add + // indicator. So floor_log_2_d happens to be correct in both cases, + // which is why we do it outside of the if statement. + } + return result; +} + +struct libdivide_u64_t libdivide_u64_gen(uint64_t d) { + return libdivide_internal_u64_gen(d, 0); +} + +struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) { + if (d == 1) { + LIBDIVIDE_ERROR("branchfree divider must be != 1"); + } + struct libdivide_u64_t tmp = libdivide_internal_u64_gen(d, 1); + struct libdivide_u64_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_64_SHIFT_MASK)}; + return ret; +} + +uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return numer >> more; + } + else { + uint64_t q = libdivide_mullhi_u64(denom->magic, numer); + if (more & LIBDIVIDE_ADD_MARKER) { + uint64_t t = ((numer - q) >> 1) + q; + return t >> (more & LIBDIVIDE_64_SHIFT_MASK); + } + else { + // All upper bits are 0, + // don't need to mask them off. + return q >> more; + } + } +} + +uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom) { + uint64_t q = libdivide_mullhi_u64(denom->magic, numer); + uint64_t t = ((numer - q) >> 1) + q; + return t >> denom->more; +} + +uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + + if (!denom->magic) { + return 1ULL << shift; + } else if (!(more & LIBDIVIDE_ADD_MARKER)) { + // We compute q = n/d = n*m / 2^(64 + shift) + // Therefore we have d = 2^(64 + shift) / m + // We need to ceil it. + // We know d is not a power of 2, so m is not a power of 2, + // so we can just add 1 to the floor + uint64_t hi_dividend = 1ULL << shift; + uint64_t rem_ignored; + return 1 + libdivide_128_div_64_to_64(hi_dividend, 0, denom->magic, &rem_ignored); + } else { + // Here we wish to compute d = 2^(64+shift+1)/(m+2^64). + // Notice (m + 2^64) is a 65 bit number. This gets hairy. See + // libdivide_u32_recover for more on what we do here. + // TODO: do something better than 128 bit math + + // Full n is a (potentially) 129 bit value + // half_n is a 128 bit value + // Compute the hi half of half_n. Low half is 0. + uint64_t half_n_hi = 1ULL << shift, half_n_lo = 0; + // d is a 65 bit value. The high bit is always set to 1. + const uint64_t d_hi = 1, d_lo = denom->magic; + // Note that the quotient is guaranteed <= 64 bits, + // but the remainder may need 65! + uint64_t r_hi, r_lo; + uint64_t half_q = libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); + // We computed 2^(64+shift)/(m+2^64) + // Double the remainder ('dr') and check if that is larger than d + // Note that d is a 65 bit value, so r1 is small and so r1 + r1 + // cannot overflow + uint64_t dr_lo = r_lo + r_lo; + uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry + int dr_exceeds_d = (dr_hi > d_hi) || (dr_hi == d_hi && dr_lo >= d_lo); + uint64_t full_q = half_q + half_q + (dr_exceeds_d ? 1 : 0); + return full_q + 1; + } +} + +uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + + if (!denom->magic) { + return 1ULL << (shift + 1); + } else { + // Here we wish to compute d = 2^(64+shift+1)/(m+2^64). + // Notice (m + 2^64) is a 65 bit number. This gets hairy. See + // libdivide_u32_recover for more on what we do here. + // TODO: do something better than 128 bit math + + // Full n is a (potentially) 129 bit value + // half_n is a 128 bit value + // Compute the hi half of half_n. Low half is 0. + uint64_t half_n_hi = 1ULL << shift, half_n_lo = 0; + // d is a 65 bit value. The high bit is always set to 1. + const uint64_t d_hi = 1, d_lo = denom->magic; + // Note that the quotient is guaranteed <= 64 bits, + // but the remainder may need 65! + uint64_t r_hi, r_lo; + uint64_t half_q = libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); + // We computed 2^(64+shift)/(m+2^64) + // Double the remainder ('dr') and check if that is larger than d + // Note that d is a 65 bit value, so r1 is small and so r1 + r1 + // cannot overflow + uint64_t dr_lo = r_lo + r_lo; + uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry + int dr_exceeds_d = (dr_hi > d_hi) || (dr_hi == d_hi && dr_lo >= d_lo); + uint64_t full_q = half_q + half_q + (dr_exceeds_d ? 1 : 0); + return full_q + 1; + } +} + +/////////// SINT32 + +static inline struct libdivide_s32_t libdivide_internal_s32_gen(int32_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } + + struct libdivide_s32_t result; + + // If d is a power of 2, or negative a power of 2, we have to use a shift. + // This is especially important because the magic algorithm fails for -1. + // To check if d is a power of 2 or its inverse, it suffices to check + // whether its absolute value has exactly one bit set. This works even for + // INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set + // and is a power of 2. + uint32_t ud = (uint32_t)d; + uint32_t absD = (d < 0) ? -ud : ud; + uint32_t floor_log_2_d = 31 - libdivide_count_leading_zeros32(absD); + // check if exactly one bit is set, + // don't care if absD is 0 since that's divide by zero + if ((absD & (absD - 1)) == 0) { + // Branchfree and normal paths are exactly the same + result.magic = 0; + result.more = floor_log_2_d | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); + } else { + LIBDIVIDE_ASSERT(floor_log_2_d >= 1); + + uint8_t more; + // the dividend here is 2**(floor_log_2_d + 31), so the low 32 bit word + // is 0 and the high word is floor_log_2_d - 1 + uint32_t rem, proposed_m; + proposed_m = libdivide_64_div_32_to_32(1U << (floor_log_2_d - 1), 0, absD, &rem); + const uint32_t e = absD - rem; + + // We are going to start with a power of floor_log_2_d - 1. + // This works if works if e < 2**floor_log_2_d. + if (!branchfree && e < (1U << floor_log_2_d)) { + // This power works + more = floor_log_2_d - 1; + } else { + // We need to go one higher. This should not make proposed_m + // overflow, but it will make it negative when interpreted as an + // int32_t. + proposed_m += proposed_m; + const uint32_t twice_rem = rem + rem; + if (twice_rem >= absD || twice_rem < rem) proposed_m += 1; + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; + } + + proposed_m += 1; + int32_t magic = (int32_t)proposed_m; + + // Mark if we are negative. Note we only negate the magic number in the + // branchfull case. + if (d < 0) { + more |= LIBDIVIDE_NEGATIVE_DIVISOR; + if (!branchfree) { + magic = -magic; + } + } + + result.more = more; + result.magic = magic; + } + return result; +} + +struct libdivide_s32_t libdivide_s32_gen(int32_t d) { + return libdivide_internal_s32_gen(d, 0); +} + +struct libdivide_s32_branchfree_t libdivide_s32_branchfree_gen(int32_t d) { + struct libdivide_s32_t tmp = libdivide_internal_s32_gen(d, 1); + struct libdivide_s32_branchfree_t result = {tmp.magic, tmp.more}; + return result; +} + +int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + + if (!denom->magic) { + uint32_t sign = (int8_t)more >> 7; + uint32_t mask = (1U << shift) - 1; + uint32_t uq = numer + ((numer >> 31) & mask); + int32_t q = (int32_t)uq; + q >>= shift; + q = (q ^ sign) - sign; + return q; + } else { + uint32_t uq = (uint32_t)libdivide_mullhi_s32(denom->magic, numer); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift and then sign extend + int32_t sign = (int8_t)more >> 7; + // q += (more < 0 ? -numer : numer) + // cast required to avoid UB + uq += ((uint32_t)numer ^ sign) - sign; + } + int32_t q = (int32_t)uq; + q >>= shift; + q += (q < 0); + return q; + } +} + +int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift and then sign extend + int32_t sign = (int8_t)more >> 7; + int32_t magic = denom->magic; + int32_t q = libdivide_mullhi_s32(magic, numer); + q += numer; + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is a power of + // 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + uint32_t q_sign = (uint32_t)(q >> 31); + q += q_sign & ((1U << shift) - is_power_of_2); + + // Now arithmetic right shift + q >>= shift; + // Negate if needed + q = (q ^ sign) - sign; + + return q; +} + +int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + if (!denom->magic) { + uint32_t absD = 1U << shift; + if (more & LIBDIVIDE_NEGATIVE_DIVISOR) { + absD = -absD; + } + return (int32_t)absD; + } else { + // Unsigned math is much easier + // We negate the magic number only in the branchfull case, and we don't + // know which case we're in. However we have enough information to + // determine the correct sign of the magic number. The divisor was + // negative if LIBDIVIDE_NEGATIVE_DIVISOR is set. If ADD_MARKER is set, + // the magic number's sign is opposite that of the divisor. + // We want to compute the positive magic number. + int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) + ? denom->magic > 0 : denom->magic < 0; + + // Handle the power of 2 case (including branchfree) + if (denom->magic == 0) { + int32_t result = 1U << shift; + return negative_divisor ? -result : result; + } + + uint32_t d = (uint32_t)(magic_was_negated ? -denom->magic : denom->magic); + uint64_t n = 1ULL << (32 + shift); // this shift cannot exceed 30 + uint32_t q = (uint32_t)(n / d); + int32_t result = (int32_t)q; + result += 1; + return negative_divisor ? -result : result; + } +} + +int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom) { + return libdivide_s32_recover((const struct libdivide_s32_t *)denom); +} + +///////////// SINT64 + +static inline struct libdivide_s64_t libdivide_internal_s64_gen(int64_t d, int branchfree) { + if (d == 0) { + LIBDIVIDE_ERROR("divider must be != 0"); + } + + struct libdivide_s64_t result; + + // If d is a power of 2, or negative a power of 2, we have to use a shift. + // This is especially important because the magic algorithm fails for -1. + // To check if d is a power of 2 or its inverse, it suffices to check + // whether its absolute value has exactly one bit set. This works even for + // INT_MIN, because abs(INT_MIN) == INT_MIN, and INT_MIN has one bit set + // and is a power of 2. + uint64_t ud = (uint64_t)d; + uint64_t absD = (d < 0) ? -ud : ud; + uint32_t floor_log_2_d = 63 - libdivide_count_leading_zeros64(absD); + // check if exactly one bit is set, + // don't care if absD is 0 since that's divide by zero + if ((absD & (absD - 1)) == 0) { + // Branchfree and non-branchfree cases are the same + result.magic = 0; + result.more = floor_log_2_d | (d < 0 ? LIBDIVIDE_NEGATIVE_DIVISOR : 0); + } else { + // the dividend here is 2**(floor_log_2_d + 63), so the low 64 bit word + // is 0 and the high word is floor_log_2_d - 1 + uint8_t more; + uint64_t rem, proposed_m; + proposed_m = libdivide_128_div_64_to_64(1ULL << (floor_log_2_d - 1), 0, absD, &rem); + const uint64_t e = absD - rem; + + // We are going to start with a power of floor_log_2_d - 1. + // This works if works if e < 2**floor_log_2_d. + if (!branchfree && e < (1ULL << floor_log_2_d)) { + // This power works + more = floor_log_2_d - 1; + } else { + // We need to go one higher. This should not make proposed_m + // overflow, but it will make it negative when interpreted as an + // int32_t. + proposed_m += proposed_m; + const uint64_t twice_rem = rem + rem; + if (twice_rem >= absD || twice_rem < rem) proposed_m += 1; + // note that we only set the LIBDIVIDE_NEGATIVE_DIVISOR bit if we + // also set ADD_MARKER this is an annoying optimization that + // enables algorithm #4 to avoid the mask. However we always set it + // in the branchfree case + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; + } + proposed_m += 1; + int64_t magic = (int64_t)proposed_m; + + // Mark if we are negative + if (d < 0) { + more |= LIBDIVIDE_NEGATIVE_DIVISOR; + if (!branchfree) { + magic = -magic; + } + } + + result.more = more; + result.magic = magic; + } + return result; +} + +struct libdivide_s64_t libdivide_s64_gen(int64_t d) { + return libdivide_internal_s64_gen(d, 0); +} + +struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d) { + struct libdivide_s64_t tmp = libdivide_internal_s64_gen(d, 1); + struct libdivide_s64_branchfree_t ret = {tmp.magic, tmp.more}; + return ret; +} + +int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + + if (!denom->magic) { // shift path + uint64_t mask = (1ULL << shift) - 1; + uint64_t uq = numer + ((numer >> 63) & mask); + int64_t q = (int64_t)uq; + q >>= shift; + // must be arithmetic shift and then sign-extend + int64_t sign = (int8_t)more >> 7; + q = (q ^ sign) - sign; + return q; + } else { + uint64_t uq = (uint64_t)libdivide_mullhi_s64(denom->magic, numer); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift and then sign extend + int64_t sign = (int8_t)more >> 7; + // q += (more < 0 ? -numer : numer) + // cast required to avoid UB + uq += ((uint64_t)numer ^ sign) - sign; + } + int64_t q = (int64_t)uq; + q >>= shift; + q += (q < 0); + return q; + } +} + +int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift and then sign extend + int64_t sign = (int8_t)more >> 7; + int64_t magic = denom->magic; + int64_t q = libdivide_mullhi_s64(magic, numer); + q += numer; + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is a power of + // 2, or (2**shift) if it is not a power of 2. + uint64_t is_power_of_2 = (magic == 0); + uint64_t q_sign = (uint64_t)(q >> 63); + q += q_sign & ((1ULL << shift) - is_power_of_2); + + // Arithmetic right shift + q >>= shift; + // Negate if needed + q = (q ^ sign) - sign; + + return q; +} + +int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + if (denom->magic == 0) { // shift path + uint64_t absD = 1ULL << shift; + if (more & LIBDIVIDE_NEGATIVE_DIVISOR) { + absD = -absD; + } + return (int64_t)absD; + } else { + // Unsigned math is much easier + int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) + ? denom->magic > 0 : denom->magic < 0; + + uint64_t d = (uint64_t)(magic_was_negated ? -denom->magic : denom->magic); + uint64_t n_hi = 1ULL << shift, n_lo = 0; + uint64_t rem_ignored; + uint64_t q = libdivide_128_div_64_to_64(n_hi, n_lo, d, &rem_ignored); + int64_t result = (int64_t)(q + 1); + if (negative_divisor) { + result = -result; + } + return result; + } +} + +int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom) { + return libdivide_s64_recover((const struct libdivide_s64_t *)denom); +} + +#if defined(LIBDIVIDE_AVX512) + +static inline __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom); +static inline __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom); +static inline __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom); +static inline __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom); + +static inline __m512i libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m512i libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m512i libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m512i libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +static inline __m512i libdivide_s64_signbits(__m512i v) {; + return _mm512_srai_epi64(v, 63); +} + +static inline __m512i libdivide_s64_shift_right_vector(__m512i v, int amt) { + return _mm512_srai_epi64(v, amt); +} + +// Here, b is assumed to contain one 32-bit value repeated. +static inline __m512i libdivide_mullhi_u32_vector(__m512i a, __m512i b) { + __m512i hi_product_0Z2Z = _mm512_srli_epi64(_mm512_mul_epu32(a, b), 32); + __m512i a1X3X = _mm512_srli_epi64(a, 32); + __m512i mask = _mm512_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); + __m512i hi_product_Z1Z3 = _mm512_and_si512(_mm512_mul_epu32(a1X3X, b), mask); + return _mm512_or_si512(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// b is one 32-bit value repeated. +static inline __m512i libdivide_mullhi_s32_vector(__m512i a, __m512i b) { + __m512i hi_product_0Z2Z = _mm512_srli_epi64(_mm512_mul_epi32(a, b), 32); + __m512i a1X3X = _mm512_srli_epi64(a, 32); + __m512i mask = _mm512_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); + __m512i hi_product_Z1Z3 = _mm512_and_si512(_mm512_mul_epi32(a1X3X, b), mask); + return _mm512_or_si512(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// Here, y is assumed to contain one 64-bit value repeated. +// https://stackoverflow.com/a/28827013 +static inline __m512i libdivide_mullhi_u64_vector(__m512i x, __m512i y) { + __m512i lomask = _mm512_set1_epi64(0xffffffff); + __m512i xh = _mm512_shuffle_epi32(x, (_MM_PERM_ENUM) 0xB1); + __m512i yh = _mm512_shuffle_epi32(y, (_MM_PERM_ENUM) 0xB1); + __m512i w0 = _mm512_mul_epu32(x, y); + __m512i w1 = _mm512_mul_epu32(x, yh); + __m512i w2 = _mm512_mul_epu32(xh, y); + __m512i w3 = _mm512_mul_epu32(xh, yh); + __m512i w0h = _mm512_srli_epi64(w0, 32); + __m512i s1 = _mm512_add_epi64(w1, w0h); + __m512i s1l = _mm512_and_si512(s1, lomask); + __m512i s1h = _mm512_srli_epi64(s1, 32); + __m512i s2 = _mm512_add_epi64(w2, s1l); + __m512i s2h = _mm512_srli_epi64(s2, 32); + __m512i hi = _mm512_add_epi64(w3, s1h); + hi = _mm512_add_epi64(hi, s2h); + + return hi; +} + +// y is one 64-bit value repeated. +static inline __m512i libdivide_mullhi_s64_vector(__m512i x, __m512i y) { + __m512i p = libdivide_mullhi_u64_vector(x, y); + __m512i t1 = _mm512_and_si512(libdivide_s64_signbits(x), y); + __m512i t2 = _mm512_and_si512(libdivide_s64_signbits(y), x); + p = _mm512_sub_epi64(p, t1); + p = _mm512_sub_epi64(p, t2); + return p; +} + +////////// UINT32 + +__m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm512_srli_epi32(numers, more); + } + else { + __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + __m512i t = _mm512_add_epi32(_mm512_srli_epi32(_mm512_sub_epi32(numers, q), 1), q); + return _mm512_srli_epi32(t, shift); + } + else { + return _mm512_srli_epi32(q, more); + } + } +} + +__m512i libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom) { + __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic)); + __m512i t = _mm512_add_epi32(_mm512_srli_epi32(_mm512_sub_epi32(numers, q), 1), q); + return _mm512_srli_epi32(t, denom->more); +} + +////////// UINT64 + +__m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm512_srli_epi64(numers, more); + } + else { + __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + __m512i t = _mm512_add_epi64(_mm512_srli_epi64(_mm512_sub_epi64(numers, q), 1), q); + return _mm512_srli_epi64(t, shift); + } + else { + return _mm512_srli_epi64(q, more); + } + } +} + +__m512i libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom) { + __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic)); + __m512i t = _mm512_add_epi64(_mm512_srli_epi64(_mm512_sub_epi64(numers, q), 1), q); + return _mm512_srli_epi64(t, denom->more); +} + +////////// SINT32 + +__m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + __m512i roundToZeroTweak = _mm512_set1_epi32(mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + __m512i q = _mm512_add_epi32(numers, _mm512_and_si512(_mm512_srai_epi32(numers, 31), roundToZeroTweak)); + q = _mm512_srai_epi32(q, shift); + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); + return q; + } + else { + __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm512_add_epi32(q, _mm512_sub_epi32(_mm512_xor_si512(numers, sign), sign)); + } + // q >>= shift + q = _mm512_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = _mm512_add_epi32(q, _mm512_srli_epi32(q, 31)); // q += (q < 0) + return q; + } +} + +__m512i libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(magic)); + q = _mm512_add_epi32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + __m512i q_sign = _mm512_srai_epi32(q, 31); // q_sign = q >> 31 + __m512i mask = _mm512_set1_epi32((1U << shift) - is_power_of_2); + q = _mm512_add_epi32(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm512_srai_epi32(q, shift); // q >>= shift + q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +__m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + __m512i roundToZeroTweak = _mm512_set1_epi64(mask); + // q = numer + ((numer >> 63) & roundToZeroTweak); + __m512i q = _mm512_add_epi64(numers, _mm512_and_si512(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vector(q, shift); + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); + return q; + } + else { + __m512i q = libdivide_mullhi_s64_vector(numers, _mm512_set1_epi64(magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm512_add_epi64(q, _mm512_sub_epi64(_mm512_xor_si512(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm512_add_epi64(q, _mm512_srli_epi64(q, 63)); // q += (q < 0) + return q; + } +} + +__m512i libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); + + // libdivide_mullhi_s64(numers, magic); + __m512i q = libdivide_mullhi_s64_vector(numers, _mm512_set1_epi64(magic)); + q = _mm512_add_epi64(q, numers); // q += numers + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + __m512i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m512i mask = _mm512_set1_epi64((1ULL << shift) - is_power_of_2); + q = _mm512_add_epi64(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift + q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +#elif defined(LIBDIVIDE_AVX2) + +static inline __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom); +static inline __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom); +static inline __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom); +static inline __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom); + +static inline __m256i libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m256i libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m256i libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m256i libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +// Implementation of _mm256_srai_epi64(v, 63) (from AVX512). +static inline __m256i libdivide_s64_signbits(__m256i v) { + __m256i hiBitsDuped = _mm256_shuffle_epi32(v, _MM_SHUFFLE(3, 3, 1, 1)); + __m256i signBits = _mm256_srai_epi32(hiBitsDuped, 31); + return signBits; +} + +// Implementation of _mm256_srai_epi64 (from AVX512). +static inline __m256i libdivide_s64_shift_right_vector(__m256i v, int amt) { + const int b = 64 - amt; + __m256i m = _mm256_set1_epi64x(1ULL << (b - 1)); + __m256i x = _mm256_srli_epi64(v, amt); + __m256i result = _mm256_sub_epi64(_mm256_xor_si256(x, m), m); + return result; +} + +// Here, b is assumed to contain one 32-bit value repeated. +static inline __m256i libdivide_mullhi_u32_vector(__m256i a, __m256i b) { + __m256i hi_product_0Z2Z = _mm256_srli_epi64(_mm256_mul_epu32(a, b), 32); + __m256i a1X3X = _mm256_srli_epi64(a, 32); + __m256i mask = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); + __m256i hi_product_Z1Z3 = _mm256_and_si256(_mm256_mul_epu32(a1X3X, b), mask); + return _mm256_or_si256(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// b is one 32-bit value repeated. +static inline __m256i libdivide_mullhi_s32_vector(__m256i a, __m256i b) { + __m256i hi_product_0Z2Z = _mm256_srli_epi64(_mm256_mul_epi32(a, b), 32); + __m256i a1X3X = _mm256_srli_epi64(a, 32); + __m256i mask = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); + __m256i hi_product_Z1Z3 = _mm256_and_si256(_mm256_mul_epi32(a1X3X, b), mask); + return _mm256_or_si256(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// Here, y is assumed to contain one 64-bit value repeated. +// https://stackoverflow.com/a/28827013 +static inline __m256i libdivide_mullhi_u64_vector(__m256i x, __m256i y) { + __m256i lomask = _mm256_set1_epi64x(0xffffffff); + __m256i xh = _mm256_shuffle_epi32(x, 0xB1); // x0l, x0h, x1l, x1h + __m256i yh = _mm256_shuffle_epi32(y, 0xB1); // y0l, y0h, y1l, y1h + __m256i w0 = _mm256_mul_epu32(x, y); // x0l*y0l, x1l*y1l + __m256i w1 = _mm256_mul_epu32(x, yh); // x0l*y0h, x1l*y1h + __m256i w2 = _mm256_mul_epu32(xh, y); // x0h*y0l, x1h*y0l + __m256i w3 = _mm256_mul_epu32(xh, yh); // x0h*y0h, x1h*y1h + __m256i w0h = _mm256_srli_epi64(w0, 32); + __m256i s1 = _mm256_add_epi64(w1, w0h); + __m256i s1l = _mm256_and_si256(s1, lomask); + __m256i s1h = _mm256_srli_epi64(s1, 32); + __m256i s2 = _mm256_add_epi64(w2, s1l); + __m256i s2h = _mm256_srli_epi64(s2, 32); + __m256i hi = _mm256_add_epi64(w3, s1h); + hi = _mm256_add_epi64(hi, s2h); + + return hi; +} + +// y is one 64-bit value repeated. +static inline __m256i libdivide_mullhi_s64_vector(__m256i x, __m256i y) { + __m256i p = libdivide_mullhi_u64_vector(x, y); + __m256i t1 = _mm256_and_si256(libdivide_s64_signbits(x), y); + __m256i t2 = _mm256_and_si256(libdivide_s64_signbits(y), x); + p = _mm256_sub_epi64(p, t1); + p = _mm256_sub_epi64(p, t2); + return p; +} + +////////// UINT32 + +__m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm256_srli_epi32(numers, more); + } + else { + __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + __m256i t = _mm256_add_epi32(_mm256_srli_epi32(_mm256_sub_epi32(numers, q), 1), q); + return _mm256_srli_epi32(t, shift); + } + else { + return _mm256_srli_epi32(q, more); + } + } +} + +__m256i libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom) { + __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic)); + __m256i t = _mm256_add_epi32(_mm256_srli_epi32(_mm256_sub_epi32(numers, q), 1), q); + return _mm256_srli_epi32(t, denom->more); +} + +////////// UINT64 + +__m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm256_srli_epi64(numers, more); + } + else { + __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + __m256i t = _mm256_add_epi64(_mm256_srli_epi64(_mm256_sub_epi64(numers, q), 1), q); + return _mm256_srli_epi64(t, shift); + } + else { + return _mm256_srli_epi64(q, more); + } + } +} + +__m256i libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom) { + __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic)); + __m256i t = _mm256_add_epi64(_mm256_srli_epi64(_mm256_sub_epi64(numers, q), 1), q); + return _mm256_srli_epi64(t, denom->more); +} + +////////// SINT32 + +__m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + __m256i roundToZeroTweak = _mm256_set1_epi32(mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + __m256i q = _mm256_add_epi32(numers, _mm256_and_si256(_mm256_srai_epi32(numers, 31), roundToZeroTweak)); + q = _mm256_srai_epi32(q, shift); + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); + return q; + } + else { + __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm256_add_epi32(q, _mm256_sub_epi32(_mm256_xor_si256(numers, sign), sign)); + } + // q >>= shift + q = _mm256_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = _mm256_add_epi32(q, _mm256_srli_epi32(q, 31)); // q += (q < 0) + return q; + } +} + +__m256i libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(magic)); + q = _mm256_add_epi32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + __m256i q_sign = _mm256_srai_epi32(q, 31); // q_sign = q >> 31 + __m256i mask = _mm256_set1_epi32((1U << shift) - is_power_of_2); + q = _mm256_add_epi32(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm256_srai_epi32(q, shift); // q >>= shift + q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +__m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + __m256i roundToZeroTweak = _mm256_set1_epi64x(mask); + // q = numer + ((numer >> 63) & roundToZeroTweak); + __m256i q = _mm256_add_epi64(numers, _mm256_and_si256(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vector(q, shift); + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); + return q; + } + else { + __m256i q = libdivide_mullhi_s64_vector(numers, _mm256_set1_epi64x(magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm256_add_epi64(q, _mm256_sub_epi64(_mm256_xor_si256(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm256_add_epi64(q, _mm256_srli_epi64(q, 63)); // q += (q < 0) + return q; + } +} + +__m256i libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); + + // libdivide_mullhi_s64(numers, magic); + __m256i q = libdivide_mullhi_s64_vector(numers, _mm256_set1_epi64x(magic)); + q = _mm256_add_epi64(q, numers); // q += numers + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + __m256i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m256i mask = _mm256_set1_epi64x((1ULL << shift) - is_power_of_2); + q = _mm256_add_epi64(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift + q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +#elif defined(LIBDIVIDE_SSE2) + +static inline __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom); +static inline __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom); +static inline __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom); +static inline __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom); + +static inline __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +// Implementation of _mm_srai_epi64(v, 63) (from AVX512). +static inline __m128i libdivide_s64_signbits(__m128i v) { + __m128i hiBitsDuped = _mm_shuffle_epi32(v, _MM_SHUFFLE(3, 3, 1, 1)); + __m128i signBits = _mm_srai_epi32(hiBitsDuped, 31); + return signBits; +} + +// Implementation of _mm_srai_epi64 (from AVX512). +static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { + const int b = 64 - amt; + __m128i m = _mm_set1_epi64x(1ULL << (b - 1)); + __m128i x = _mm_srli_epi64(v, amt); + __m128i result = _mm_sub_epi64(_mm_xor_si128(x, m), m); + return result; +} + +// Here, b is assumed to contain one 32-bit value repeated. +static inline __m128i libdivide_mullhi_u32_vector(__m128i a, __m128i b) { + __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epu32(a, b), 32); + __m128i a1X3X = _mm_srli_epi64(a, 32); + __m128i mask = _mm_set_epi32(-1, 0, -1, 0); + __m128i hi_product_Z1Z3 = _mm_and_si128(_mm_mul_epu32(a1X3X, b), mask); + return _mm_or_si128(hi_product_0Z2Z, hi_product_Z1Z3); +} + +// SSE2 does not have a signed multiplication instruction, but we can convert +// unsigned to signed pretty efficiently. Again, b is just a 32 bit value +// repeated four times. +static inline __m128i libdivide_mullhi_s32_vector(__m128i a, __m128i b) { + __m128i p = libdivide_mullhi_u32_vector(a, b); + // t1 = (a >> 31) & y, arithmetic shift + __m128i t1 = _mm_and_si128(_mm_srai_epi32(a, 31), b); + __m128i t2 = _mm_and_si128(_mm_srai_epi32(b, 31), a); + p = _mm_sub_epi32(p, t1); + p = _mm_sub_epi32(p, t2); + return p; +} + +// Here, y is assumed to contain one 64-bit value repeated. +// https://stackoverflow.com/a/28827013 +static inline __m128i libdivide_mullhi_u64_vector(__m128i x, __m128i y) { + __m128i lomask = _mm_set1_epi64x(0xffffffff); + __m128i xh = _mm_shuffle_epi32(x, 0xB1); // x0l, x0h, x1l, x1h + __m128i yh = _mm_shuffle_epi32(y, 0xB1); // y0l, y0h, y1l, y1h + __m128i w0 = _mm_mul_epu32(x, y); // x0l*y0l, x1l*y1l + __m128i w1 = _mm_mul_epu32(x, yh); // x0l*y0h, x1l*y1h + __m128i w2 = _mm_mul_epu32(xh, y); // x0h*y0l, x1h*y0l + __m128i w3 = _mm_mul_epu32(xh, yh); // x0h*y0h, x1h*y1h + __m128i w0h = _mm_srli_epi64(w0, 32); + __m128i s1 = _mm_add_epi64(w1, w0h); + __m128i s1l = _mm_and_si128(s1, lomask); + __m128i s1h = _mm_srli_epi64(s1, 32); + __m128i s2 = _mm_add_epi64(w2, s1l); + __m128i s2h = _mm_srli_epi64(s2, 32); + __m128i hi = _mm_add_epi64(w3, s1h); + hi = _mm_add_epi64(hi, s2h); + + return hi; +} + +// y is one 64-bit value repeated. +static inline __m128i libdivide_mullhi_s64_vector(__m128i x, __m128i y) { + __m128i p = libdivide_mullhi_u64_vector(x, y); + __m128i t1 = _mm_and_si128(libdivide_s64_signbits(x), y); + __m128i t2 = _mm_and_si128(libdivide_s64_signbits(y), x); + p = _mm_sub_epi64(p, t1); + p = _mm_sub_epi64(p, t2); + return p; +} + +////////// UINT32 + +__m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm_srli_epi32(numers, more); + } + else { + __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); + return _mm_srli_epi32(t, shift); + } + else { + return _mm_srli_epi32(q, more); + } + } +} + +__m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom) { + __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic)); + __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); + return _mm_srli_epi32(t, denom->more); +} + +////////// UINT64 + +__m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return _mm_srli_epi64(numers, more); + } + else { + __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); + return _mm_srli_epi64(t, shift); + } + else { + return _mm_srli_epi64(q, more); + } + } +} + +__m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom) { + __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic)); + __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); + return _mm_srli_epi64(t, denom->more); +} + +////////// SINT32 + +__m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + __m128i roundToZeroTweak = _mm_set1_epi32(mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + __m128i q = _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); + q = _mm_srai_epi32(q, shift); + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); + return q; + } + else { + __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(denom->magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm_add_epi32(q, _mm_sub_epi32(_mm_xor_si128(numers, sign), sign)); + } + // q >>= shift + q = _mm_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) + return q; + } +} + +__m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(magic)); + q = _mm_add_epi32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + __m128i q_sign = _mm_srai_epi32(q, 31); // q_sign = q >> 31 + __m128i mask = _mm_set1_epi32((1U << shift) - is_power_of_2); + q = _mm_add_epi32(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm_srai_epi32(q, shift); // q >>= shift + q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +__m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + __m128i roundToZeroTweak = _mm_set1_epi64x(mask); + // q = numer + ((numer >> 63) & roundToZeroTweak); + __m128i q = _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vector(q, shift); + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); + return q; + } + else { + __m128i q = libdivide_mullhi_s64_vector(numers, _mm_set1_epi64x(magic)); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = _mm_add_epi64(q, _mm_sub_epi64(_mm_xor_si128(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) + return q; + } +} + +__m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + __m128i sign = _mm_set1_epi32((int8_t)more >> 7); + + // libdivide_mullhi_s64(numers, magic); + __m128i q = libdivide_mullhi_s64_vector(numers, _mm_set1_epi64x(magic)); + q = _mm_add_epi64(q, numers); // q += numers + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + __m128i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m128i mask = _mm_set1_epi64x((1ULL << shift) - is_power_of_2); + q = _mm_add_epi64(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift + q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +#endif + +/////////// C++ stuff + +#ifdef __cplusplus + +// The C++ divider class is templated on both an integer type +// (like uint64_t) and an algorithm type. +// * BRANCHFULL is the default algorithm type. +// * BRANCHFREE is the branchfree algorithm type. +enum { + BRANCHFULL, + BRANCHFREE +}; + +#if defined(LIBDIVIDE_AVX512) + #define LIBDIVIDE_VECTOR_TYPE __m512i +#elif defined(LIBDIVIDE_AVX2) + #define LIBDIVIDE_VECTOR_TYPE __m256i +#elif defined(LIBDIVIDE_SSE2) + #define LIBDIVIDE_VECTOR_TYPE __m128i +#endif + +#if !defined(LIBDIVIDE_VECTOR_TYPE) + #define LIBDIVIDE_DIVIDE_VECTOR(ALGO) +#else + #define LIBDIVIDE_DIVIDE_VECTOR(ALGO) \ + LIBDIVIDE_VECTOR_TYPE divide(LIBDIVIDE_VECTOR_TYPE n) const { \ + return libdivide_##ALGO##_do_vector(n, &denom); \ + } +#endif + +// The DISPATCHER_GEN() macro generates C++ methods (for the given integer +// and algorithm types) that redirect to libdivide's C API. +#define DISPATCHER_GEN(T, ALGO) \ + libdivide_##ALGO##_t denom; \ + dispatcher() { } \ + dispatcher(T d) \ + : denom(libdivide_##ALGO##_gen(d)) \ + { } \ + T divide(T n) const { \ + return libdivide_##ALGO##_do(n, &denom); \ + } \ + LIBDIVIDE_DIVIDE_VECTOR(ALGO) \ + T recover() const { \ + return libdivide_##ALGO##_recover(&denom); \ + } + +// The dispatcher selects a specific division algorithm for a given +// type and ALGO using partial template specialization. +template struct dispatcher { }; + +template<> struct dispatcher { DISPATCHER_GEN(int32_t, s32) }; +template<> struct dispatcher { DISPATCHER_GEN(int32_t, s32_branchfree) }; +template<> struct dispatcher { DISPATCHER_GEN(uint32_t, u32) }; +template<> struct dispatcher { DISPATCHER_GEN(uint32_t, u32_branchfree) }; +template<> struct dispatcher { DISPATCHER_GEN(int64_t, s64) }; +template<> struct dispatcher { DISPATCHER_GEN(int64_t, s64_branchfree) }; +template<> struct dispatcher { DISPATCHER_GEN(uint64_t, u64) }; +template<> struct dispatcher { DISPATCHER_GEN(uint64_t, u64_branchfree) }; + +// This is the main divider class for use by the user (C++ API). +// The actual division algorithm is selected using the dispatcher struct +// based on the integer and algorithm template parameters. +template +class divider { +public: + // We leave the default constructor empty so that creating + // an array of dividers and then initializing them + // later doesn't slow us down. + divider() { } + + // Constructor that takes the divisor as a parameter + divider(T d) : div(d) { } + + // Divides n by the divisor + T divide(T n) const { + return div.divide(n); + } + + // Recovers the divisor, returns the value that was + // used to initialize this divider object. + T recover() const { + return div.recover(); + } + + bool operator==(const divider& other) const { + return div.denom.magic == other.denom.magic && + div.denom.more == other.denom.more; + } + + bool operator!=(const divider& other) const { + return !(*this == other); + } + +#if defined(LIBDIVIDE_VECTOR_TYPE) + // Treats the vector as packed integer values with the same type as + // the divider (e.g. s32, u32, s64, u64) and divides each of + // them by the divider, returning the packed quotients. + LIBDIVIDE_VECTOR_TYPE divide(LIBDIVIDE_VECTOR_TYPE n) const { + return div.divide(n); + } +#endif + +private: + // Storage for the actual divisor + dispatcher::value, + std::is_signed::value, sizeof(T), ALGO> div; +}; + +// Overload of operator / for scalar division +template +T operator/(T n, const divider& div) { + return div.divide(n); +} + +// Overload of operator /= for scalar division +template +T& operator/=(T& n, const divider& div) { + n = div.divide(n); + return n; +} + +#if defined(LIBDIVIDE_VECTOR_TYPE) + // Overload of operator / for vector division + template + LIBDIVIDE_VECTOR_TYPE operator/(LIBDIVIDE_VECTOR_TYPE n, const divider& div) { + return div.divide(n); + } + // Overload of operator /= for vector division + template + LIBDIVIDE_VECTOR_TYPE& operator/=(LIBDIVIDE_VECTOR_TYPE& n, const divider& div) { + n = div.divide(n); + return n; + } +#endif + +// libdivdie::branchfree_divider +template +using branchfree_divider = divider; + +} // namespace libdivide + +#endif // __cplusplus + +#endif // LIBDIVIDE_H diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 63e8bf974e96..d1acfdf26235 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -236,6 +236,12 @@ typedef enum { NPY_RAISE=2 } NPY_CLIPMODE; +typedef enum { + NPY_VALID=0, + NPY_SAME=1, + NPY_FULL=2 +} NPY_CORRELATEMODE; + /* The special not-a-time (NaT) value */ #define NPY_DATETIME_NAT NPY_MIN_INT64 @@ -850,6 +856,17 @@ typedef int (PyArray_FinalizeFunc)(PyArrayObject *, PyObject *); */ #define NPY_ARRAY_ENSUREARRAY 0x0040 +#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD + /* + * Dual use of the ENSUREARRAY flag, to indicate that this was converted + * from a python float, int, or complex. + * An array using this flag must be a temporary array that can never + * leave the C internals of NumPy. Even if it does, ENSUREARRAY is + * absolutely safe to abuse, since it already is a base class array :). + */ + #define _NPY_ARRAY_WAS_PYSCALAR 0x0040 +#endif /* NPY_INTERNAL_BUILD */ + /* * Make sure that the strides are in units of the element size Needed * for some operations with record-arrays. @@ -1861,6 +1878,8 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, typedef PyArray_Descr *(default_descr_function)(PyArray_DTypeMeta *cls); typedef PyArray_DTypeMeta *(common_dtype_function)( PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtyep2); + typedef PyArray_DTypeMeta *(common_dtype_with_value_function)( + PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtyep2, PyObject *value); typedef PyArray_Descr *(common_instance_function)( PyArray_Descr *dtype1, PyArray_Descr *dtyep2); @@ -1919,7 +1938,13 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size, is_known_scalar_type_function *is_known_scalar_type; default_descr_function *default_descr; common_dtype_function *common_dtype; + common_dtype_with_value_function *common_dtype_with_value; common_instance_function *common_instance; + /* + * The casting implementation (ArrayMethod) to convert between two + * instances of this DType, stored explicitly for fast access: + */ + PyObject *within_dtype_castingimpl; /* * Dictionary of ArrayMethods representing most possible casts * (structured and object are exceptions). diff --git a/numpy/core/include/numpy/npy_3kcompat.h b/numpy/core/include/numpy/npy_3kcompat.h index 191cd244f875..551ec6be8c2d 100644 --- a/numpy/core/include/numpy/npy_3kcompat.h +++ b/numpy/core/include/numpy/npy_3kcompat.h @@ -216,6 +216,7 @@ static NPY_INLINE FILE* npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos) { int fd, fd2, unbuf; + Py_ssize_t fd2_tmp; PyObject *ret, *os, *io, *io_raw; npy_off_t pos; FILE *handle; @@ -251,8 +252,17 @@ npy_PyFile_Dup2(PyObject *file, char *mode, npy_off_t *orig_pos) if (ret == NULL) { return NULL; } - fd2 = PyNumber_AsSsize_t(ret, NULL); + fd2_tmp = PyNumber_AsSsize_t(ret, PyExc_IOError); Py_DECREF(ret); + if (fd2_tmp == -1 && PyErr_Occurred()) { + return NULL; + } + if (fd2_tmp < INT_MIN || fd2_tmp > INT_MAX) { + PyErr_SetString(PyExc_IOError, + "Getting an 'int' from os.dup() failed"); + return NULL; + } + fd2 = (int)fd2_tmp; /* Convert to FILE* handle */ #ifdef _WIN32 diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h index 78d6d31207f1..d5f329b66754 100644 --- a/numpy/core/include/numpy/npy_common.h +++ b/numpy/core/include/numpy/npy_common.h @@ -14,9 +14,11 @@ * using static inline modifiers when defining npy_math functions * allows the compiler to make optimizations when possible */ -#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD #ifndef NPY_INLINE_MATH -#define NPY_INLINE_MATH 1 +#if defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD + #define NPY_INLINE_MATH 1 +#else + #define NPY_INLINE_MATH 0 #endif #endif diff --git a/numpy/core/include/numpy/npy_cpu.h b/numpy/core/include/numpy/npy_cpu.h index 4dbf9d84e384..065176ac5fb6 100644 --- a/numpy/core/include/numpy/npy_cpu.h +++ b/numpy/core/include/numpy/npy_cpu.h @@ -110,10 +110,16 @@ information about your platform (OS, CPU and compiler) #endif -#if (defined(NPY_CPU_X86) || defined(NPY_CPU_AMD64)) -#define NPY_CPU_HAVE_UNALIGNED_ACCESS 1 -#else -#define NPY_CPU_HAVE_UNALIGNED_ACCESS 0 +/* + * Except for the following architectures, memory access is limited to the natural + * alignment of data types otherwise it may lead to bus error or performance regression. + * For more details about unaligned access, see https://www.kernel.org/doc/Documentation/unaligned-memory-access.txt. +*/ +#if defined(NPY_CPU_X86) || defined(NPY_CPU_AMD64) || defined(__aarch64__) || defined(__powerpc64__) + #define NPY_ALIGNMENT_REQUIRED 0 +#endif +#ifndef NPY_ALIGNMENT_REQUIRED + #define NPY_ALIGNMENT_REQUIRED 1 #endif #endif diff --git a/numpy/core/include/numpy/npy_math.h b/numpy/core/include/numpy/npy_math.h index dbade058f9dc..f32e298f081f 100644 --- a/numpy/core/include/numpy/npy_math.h +++ b/numpy/core/include/numpy/npy_math.h @@ -150,15 +150,6 @@ NPY_INPLACE npy_long npy_lshiftl(npy_long a, npy_long b); NPY_INPLACE npy_longlong npy_rshiftll(npy_longlong a, npy_longlong b); NPY_INPLACE npy_longlong npy_lshiftll(npy_longlong a, npy_longlong b); -/* - * avx function has a common API for both sin & cos. This enum is used to - * distinguish between the two - */ -typedef enum { - npy_compute_sin, - npy_compute_cos -} NPY_TRIG_OP; - /* * C99 double math funcs */ @@ -211,7 +202,7 @@ double npy_spacing(double x); /* use builtins to avoid function calls in tight loops * only available if npy_config.h is available (= numpys own build) */ -#if HAVE___BUILTIN_ISNAN +#ifdef HAVE___BUILTIN_ISNAN #define npy_isnan(x) __builtin_isnan(x) #else #ifndef NPY_HAVE_DECL_ISNAN @@ -227,7 +218,7 @@ double npy_spacing(double x); /* only available if npy_config.h is available (= numpys own build) */ -#if HAVE___BUILTIN_ISFINITE +#ifdef HAVE___BUILTIN_ISFINITE #define npy_isfinite(x) __builtin_isfinite(x) #else #ifndef NPY_HAVE_DECL_ISFINITE @@ -242,7 +233,7 @@ double npy_spacing(double x); #endif /* only available if npy_config.h is available (= numpys own build) */ -#if HAVE___BUILTIN_ISINF +#ifdef HAVE___BUILTIN_ISINF #define npy_isinf(x) __builtin_isinf(x) #else #ifndef NPY_HAVE_DECL_ISINF diff --git a/numpy/core/include/numpy/numpyconfig.h b/numpy/core/include/numpy/numpyconfig.h index e39303123822..a1b1de0ef14c 100644 --- a/numpy/core/include/numpy/numpyconfig.h +++ b/numpy/core/include/numpy/numpyconfig.h @@ -42,5 +42,6 @@ #define NPY_1_18_API_VERSION 0x00000008 #define NPY_1_19_API_VERSION 0x00000008 #define NPY_1_20_API_VERSION 0x0000000e +#define NPY_1_21_API_VERSION 0x0000000e #endif diff --git a/numpy/core/include/numpy/random/distributions.h b/numpy/core/include/numpy/random/distributions.h index c474c4d14a3d..c58024605ff5 100644 --- a/numpy/core/include/numpy/random/distributions.h +++ b/numpy/core/include/numpy/random/distributions.h @@ -1,6 +1,10 @@ #ifndef _RANDOMDGEN__DISTRIBUTIONS_H_ #define _RANDOMDGEN__DISTRIBUTIONS_H_ +#ifdef __cplusplus +extern "C" { +#endif + #include "Python.h" #include "numpy/npy_common.h" #include @@ -119,8 +123,9 @@ DECLDIR RAND_INT_TYPE random_negative_binomial(bitgen_t *bitgen_state, double n, DECLDIR int64_t random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial); -DECLDIR RAND_INT_TYPE random_logseries(bitgen_t *bitgen_state, double p); -DECLDIR RAND_INT_TYPE random_geometric(bitgen_t *bitgen_state, double p); +DECLDIR int64_t random_logseries(bitgen_t *bitgen_state, double p); +DECLDIR int64_t random_geometric(bitgen_t *bitgen_state, double p); +DECLDIR RAND_INT_TYPE random_geometric_search(bitgen_t *bitgen_state, double p); DECLDIR RAND_INT_TYPE random_zipf(bitgen_t *bitgen_state, double a); DECLDIR int64_t random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, int64_t sample); @@ -197,4 +202,8 @@ static NPY_INLINE double next_double(bitgen_t *bitgen_state) { return bitgen_state->next_double(bitgen_state->state); } +#ifdef __cplusplus +} +#endif + #endif diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h index e5d845842976..333a326ee60e 100644 --- a/numpy/core/include/numpy/ufuncobject.h +++ b/numpy/core/include/numpy/ufuncobject.h @@ -194,7 +194,11 @@ typedef struct _tagPyUFuncObject { * but this was never implemented. (This is also why the above * selector is called the "legacy" selector.) */ + #if PY_VERSION_HEX >= 0x03080000 + vectorcallfunc vectorcall; + #else void *reserved2; + #endif /* * A function which returns a masked inner loop for the ufunc. */ diff --git a/numpy/core/memmap.py b/numpy/core/memmap.py index 66653c0c155b..b0d9cb3af7bf 100644 --- a/numpy/core/memmap.py +++ b/numpy/core/memmap.py @@ -1,8 +1,8 @@ +from contextlib import nullcontext + import numpy as np from .numeric import uint8, ndarray, dtype -from numpy.compat import ( - os_fspath, contextlib_nullcontext, is_pathlib_path -) +from numpy.compat import os_fspath, is_pathlib_path from numpy.core.overrides import set_module __all__ = ['memmap'] @@ -38,7 +38,7 @@ class memmap(ndarray): which returns a view into an mmap buffer. Flush the memmap instance to write the changes to the file. Currently there - is no API to close the underlying ``mmap``. It is tricky to ensure the + is no API to close the underlying ``mmap``. It is tricky to ensure the resource is actually closed, since it may be shared between different memmap instances. @@ -112,7 +112,7 @@ class memmap(ndarray): The memmap object can be used anywhere an ndarray is accepted. Given a memmap ``fp``, ``isinstance(fp, numpy.ndarray)`` returns ``True``. - + Memory-mapped files cannot be larger than 2GB on 32-bit systems. When a memmap causes a file to be created or extended beyond its @@ -223,7 +223,7 @@ def __new__(subtype, filename, dtype=uint8, mode='r+', offset=0, raise ValueError("shape must be given") if hasattr(filename, 'read'): - f_ctx = contextlib_nullcontext(filename) + f_ctx = nullcontext(filename) else: f_ctx = open(os_fspath(filename), ('r' if mode == 'c' else mode)+'b') @@ -316,7 +316,7 @@ def flush(self): self.base.flush() def __array_wrap__(self, arr, context=None): - arr = super(memmap, self).__array_wrap__(arr, context) + arr = super().__array_wrap__(arr, context) # Return a memmap if a memmap was given as the output of the # ufunc. Leave the arr class unchanged if self is not a memmap @@ -331,7 +331,7 @@ def __array_wrap__(self, arr, context=None): return arr.view(np.ndarray) def __getitem__(self, index): - res = super(memmap, self).__getitem__(index) + res = super().__getitem__(index) if type(res) is memmap and res._mmap is None: return res.view(type=ndarray) return res diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index f311fad8d390..b7a3a8d67534 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -26,7 +26,8 @@ 'MAY_SHARE_BOUNDS', 'MAY_SHARE_EXACT', 'NEEDS_INIT', 'NEEDS_PYAPI', 'RAISE', 'USE_GETITEM', 'USE_SETITEM', 'WRAP', '_fastCopyAndTranspose', '_flagdict', '_insert', '_reconstruct', '_vec_string', '_monotonicity', - 'add_docstring', 'arange', 'array', 'bincount', 'broadcast', + 'add_docstring', 'arange', 'array', 'asarray', 'asanyarray', + 'ascontiguousarray', 'asfortranarray', 'bincount', 'broadcast', 'busday_count', 'busday_offset', 'busdaycalendar', 'can_cast', 'compare_chararrays', 'concatenate', 'copyto', 'correlate', 'correlate2', 'count_nonzero', 'c_einsum', 'datetime_as_string', 'datetime_data', @@ -49,6 +50,10 @@ arange.__module__ = 'numpy' array.__module__ = 'numpy' +asarray.__module__ = 'numpy' +asanyarray.__module__ = 'numpy' +ascontiguousarray.__module__ = 'numpy' +asfortranarray.__module__ = 'numpy' datetime_data.__module__ = 'numpy' empty.__module__ = 'numpy' frombuffer.__module__ = 'numpy' @@ -259,12 +264,16 @@ def inner(a, b): Returns ------- out : ndarray - `out.shape = a.shape[:-1] + b.shape[:-1]` + If `a` and `b` are both + scalars or both 1-D arrays then a scalar is returned; otherwise + an array is returned. + ``out.shape = (*a.shape[:-1], *b.shape[:-1])`` Raises ------ ValueError - If the last dimension of `a` and `b` has different size. + If both `a` and `b` are nonscalar and their last dimensions have + different sizes. See Also -------- @@ -284,8 +293,8 @@ def inner(a, b): or explicitly:: - np.inner(a, b)[i0,...,ir-1,j0,...,js-1] - = sum(a[i0,...,ir-1,:]*b[j0,...,js-1,:]) + np.inner(a, b)[i0,...,ir-2,j0,...,js-2] + = sum(a[i0,...,ir-2,:]*b[j0,...,js-2,:]) In addition `a` or `b` may be scalars, in which case:: @@ -300,14 +309,25 @@ def inner(a, b): >>> np.inner(a, b) 2 - A multidimensional example: + Some multidimensional examples: >>> a = np.arange(24).reshape((2,3,4)) >>> b = np.arange(4) - >>> np.inner(a, b) + >>> c = np.inner(a, b) + >>> c.shape + (2, 3) + >>> c array([[ 14, 38, 62], [ 86, 110, 134]]) + >>> a = np.arange(2).reshape((1,1,2)) + >>> b = np.arange(6).reshape((3,2)) + >>> c = np.inner(a, b) + >>> c.shape + (1, 1, 3) + >>> c + array([[[1, 3, 5]]]) + An example where `b` is a scalar: >>> np.inner(np.eye(2), 7) @@ -999,7 +1019,7 @@ def ravel_multi_index(multi_index, dims, mode=None, order=None): @array_function_from_c_func_and_dispatcher(_multiarray_umath.unravel_index) -def unravel_index(indices, shape=None, order=None, dims=None): +def unravel_index(indices, shape=None, order=None): """ unravel_index(indices, shape, order='C') @@ -1045,9 +1065,6 @@ def unravel_index(indices, shape=None, order=None, dims=None): (3, 1, 4, 1) """ - if dims is not None: - warnings.warn("'shape' argument should be used instead of 'dims'", - DeprecationWarning, stacklevel=3) return (indices,) @@ -1429,7 +1446,7 @@ def is_busday(dates, weekmask=None, holidays=None, busdaycal=None, out=None): See Also -------- - busdaycalendar: An object that specifies a custom set of valid days. + busdaycalendar : An object that specifies a custom set of valid days. busday_offset : Applies an offset counted in valid days. busday_count : Counts how many valid days are in a half-open date range. @@ -1504,7 +1521,7 @@ def busday_offset(dates, offsets, roll=None, weekmask=None, holidays=None, See Also -------- - busdaycalendar: An object that specifies a custom set of valid days. + busdaycalendar : An object that specifies a custom set of valid days. is_busday : Returns a boolean array indicating valid days. busday_count : Counts how many valid days are in a half-open date range. @@ -1586,7 +1603,7 @@ def busday_count(begindates, enddates, weekmask=None, holidays=None, See Also -------- - busdaycalendar: An object that specifies a custom set of valid days. + busdaycalendar : An object that specifies a custom set of valid days. is_busday : Returns a boolean array indicating valid days. busday_offset : Applies an offset counted in valid days. diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index c95c48d7139d..8bb37e291016 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -10,7 +10,8 @@ from .multiarray import ( _fastCopyAndTranspose as fastCopyAndTranspose, ALLOW_THREADS, BUFSIZE, CLIP, MAXDIMS, MAY_SHARE_BOUNDS, MAY_SHARE_EXACT, RAISE, - WRAP, arange, array, broadcast, can_cast, compare_chararrays, + WRAP, arange, array, asarray, asanyarray, ascontiguousarray, + asfortranarray, broadcast, can_cast, compare_chararrays, concatenate, copyto, dot, dtype, empty, empty_like, flatiter, frombuffer, fromfile, fromiter, fromstring, inner, lexsort, matmul, may_share_memory, @@ -26,7 +27,6 @@ from . import numerictypes from .numerictypes import longlong, intc, int_, float_, complex_, bool_ from ._exceptions import TooHardError, AxisError -from ._asarray import asarray, asanyarray from ._ufunc_config import errstate bitwise_not = invert @@ -39,7 +39,8 @@ __all__ = [ 'newaxis', 'ndarray', 'flatiter', 'nditer', 'nested_iters', 'ufunc', - 'arange', 'array', 'zeros', 'count_nonzero', 'empty', 'broadcast', 'dtype', + 'arange', 'array', 'asarray', 'asanyarray', 'ascontiguousarray', + 'asfortranarray', 'zeros', 'count_nonzero', 'empty', 'broadcast', 'dtype', 'fromstring', 'fromfile', 'frombuffer', 'where', 'argwhere', 'copyto', 'concatenate', 'fastCopyAndTranspose', 'lexsort', 'set_numeric_ops', 'can_cast', 'promote_types', 'min_scalar_type', @@ -299,7 +300,7 @@ def full(shape, fill_value, dtype=None, order='C', *, like=None): Fill value. dtype : data-type, optional The desired data-type for the array The default, None, means - `np.array(fill_value).dtype`. + ``np.array(fill_value).dtype``. order : {'C', 'F'}, optional Whether to store multidimensional data in C- or Fortran-contiguous (row- or column-wise) order in memory. @@ -662,17 +663,6 @@ def flatnonzero(a): return np.nonzero(np.ravel(a))[0] -_mode_from_name_dict = {'v': 0, - 's': 1, - 'f': 2} - - -def _mode_from_name(mode): - if isinstance(mode, str): - return _mode_from_name_dict[mode.lower()[0]] - return mode - - def _correlate_dispatcher(a, v, mode=None): return (a, v) @@ -710,6 +700,7 @@ def correlate(a, v, mode='valid'): -------- convolve : Discrete, linear convolution of two one-dimensional sequences. multiarray.correlate : Old, no conjugate, version of correlate. + scipy.signal.correlate : uses FFT which has superior performance on large arrays. Notes ----- @@ -720,6 +711,11 @@ def correlate(a, v, mode='valid'): which is related to ``c_{av}[k]`` by ``c'_{av}[k] = c_{av}[-k]``. + `numpy.correlate` may perform slowly in large arrays (i.e. n = 1e5) because it does + not use the FFT to compute the convolution; in that case, `scipy.signal.correlate` might + be preferable. + + Examples -------- >>> np.correlate([1, 2, 3], [0, 1, 0.5]) @@ -742,7 +738,6 @@ def correlate(a, v, mode='valid'): array([ 0.0+0.j , 3.0+1.j , 1.5+1.5j, 1.0+0.j , 0.5+0.5j]) """ - mode = _mode_from_name(mode) return multiarray.correlate2(a, v, mode) @@ -846,7 +841,6 @@ def convolve(a, v, mode='full'): raise ValueError('a cannot be empty') if len(v) == 0: raise ValueError('v cannot be empty') - mode = _mode_from_name(mode) return multiarray.correlate(a, v[::-1], mode) @@ -1427,12 +1421,11 @@ def moveaxis(a, source, destination): See Also -------- - transpose: Permute the dimensions of an array. - swapaxes: Interchange two axes of an array. + transpose : Permute the dimensions of an array. + swapaxes : Interchange two axes of an array. Examples -------- - >>> x = np.zeros((3, 4, 5)) >>> np.moveaxis(x, 0, -1).shape (4, 5, 3) @@ -2351,8 +2344,13 @@ def within_tol(x, y, atol, rtol): # Make sure y is an inexact type to avoid bad behavior on abs(MIN_INT). # This will cause casting of x later. Also, make sure to allow subclasses # (e.g., for numpy.ma). - dt = multiarray.result_type(y, 1.) - y = array(y, dtype=dt, copy=False, subok=True) + # NOTE: We explicitly allow timedelta, which used to work. This could + # possibly be deprecated. See also gh-18286. + # timedelta works if `atol` is an integer or also a timedelta. + # Although, the default tolerances are unlikely to be useful + if y.dtype.kind != "m": + dt = multiarray.result_type(y, 1.) + y = asanyarray(y, dtype=dt) xfin = isfinite(x) yfin = isfinite(y) diff --git a/numpy/core/numeric.pyi b/numpy/core/numeric.pyi index d91cb31c2031..f579514349da 100644 --- a/numpy/core/numeric.pyi +++ b/numpy/core/numeric.pyi @@ -41,6 +41,7 @@ def zeros_like( subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... + def ones( shape: _ShapeLike, dtype: DTypeLike = ..., @@ -48,6 +49,7 @@ def ones( *, like: ArrayLike = ..., ) -> ndarray: ... + @overload def ones_like( a: _ArrayType, @@ -64,6 +66,7 @@ def ones_like( subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... + @overload def empty_like( a: _ArrayType, @@ -80,6 +83,7 @@ def empty_like( subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... + def full( shape: _ShapeLike, fill_value: Any, @@ -88,6 +92,7 @@ def full( *, like: ArrayLike = ..., ) -> ndarray: ... + @overload def full_like( a: _ArrayType, @@ -106,39 +111,73 @@ def full_like( subok: bool = ..., shape: Optional[_ShapeLike] = ..., ) -> ndarray: ... + @overload def count_nonzero( - a: ArrayLike, axis: None = ..., *, keepdims: Literal[False] = ... + a: ArrayLike, + axis: None = ..., + *, + keepdims: Literal[False] = ..., ) -> int: ... @overload def count_nonzero( - a: ArrayLike, axis: _ShapeLike = ..., *, keepdims: bool = ... -) -> Union[signedinteger[Any], ndarray]: ... # TODO: np.intp + a: ArrayLike, + axis: _ShapeLike = ..., + *, + keepdims: bool = ..., +) -> Any: ... # TODO: np.intp or ndarray[np.intp] + def isfortran(a: Union[ndarray, generic]) -> bool: ... + def argwhere(a: ArrayLike) -> ndarray: ... + def flatnonzero(a: ArrayLike) -> ndarray: ... -def correlate(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... -def convolve(a: ArrayLike, v: ArrayLike, mode: _CorrelateMode = ...) -> ndarray: ... + +def correlate( + a: ArrayLike, + v: ArrayLike, + mode: _CorrelateMode = ..., +) -> ndarray: ... + +def convolve( + a: ArrayLike, + v: ArrayLike, + mode: _CorrelateMode = ..., +) -> ndarray: ... + @overload -def outer(a: ArrayLike, b: ArrayLike, out: None = ...) -> ndarray: ... +def outer( + a: ArrayLike, + b: ArrayLike, + out: None = ..., +) -> ndarray: ... @overload -def outer(a: ArrayLike, b: ArrayLike, out: _ArrayType = ...) -> _ArrayType: ... +def outer( + a: ArrayLike, + b: ArrayLike, + out: _ArrayType = ..., +) -> _ArrayType: ... + def tensordot( a: ArrayLike, b: ArrayLike, axes: Union[int, Tuple[_ShapeLike, _ShapeLike]] = ..., ) -> ndarray: ... + def roll( a: ArrayLike, shift: _ShapeLike, axis: Optional[_ShapeLike] = ..., ) -> ndarray: ... + def rollaxis(a: ndarray, axis: int, start: int = ...) -> ndarray: ... + def moveaxis( a: ndarray, source: _ShapeLike, destination: _ShapeLike, ) -> ndarray: ... + def cross( a: ArrayLike, b: ArrayLike, @@ -147,6 +186,7 @@ def cross( axisc: int = ..., axis: Optional[int] = ..., ) -> ndarray: ... + @overload def indices( dimensions: Sequence[int], @@ -159,6 +199,7 @@ def indices( dtype: DTypeLike = ..., sparse: Literal[True] = ..., ) -> Tuple[ndarray, ...]: ... + def fromfunction( function: Callable[..., _T], shape: Sequence[int], @@ -167,10 +208,20 @@ def fromfunction( like: ArrayLike = ..., **kwargs: Any, ) -> _T: ... + def isscalar(element: Any) -> bool: ... + def binary_repr(num: int, width: Optional[int] = ...) -> str: ... + def base_repr(number: int, base: int = ..., padding: int = ...) -> str: ... -def identity(n: int, dtype: DTypeLike = ..., *, like: ArrayLike = ...) -> ndarray: ... + +def identity( + n: int, + dtype: DTypeLike = ..., + *, + like: ArrayLike = ..., +) -> ndarray: ... + def allclose( a: ArrayLike, b: ArrayLike, @@ -178,12 +229,15 @@ def allclose( atol: float = ..., equal_nan: bool = ..., ) -> bool: ... + def isclose( a: ArrayLike, b: ArrayLike, rtol: float = ..., atol: float = ..., equal_nan: bool = ..., -) -> Union[bool_, ndarray]: ... -def array_equal(a1: ArrayLike, a2: ArrayLike) -> bool: ... +) -> Any: ... + +def array_equal(a1: ArrayLike, a2: ArrayLike, equal_nan: bool = ...) -> bool: ... + def array_equiv(a1: ArrayLike, a2: ArrayLike) -> bool: ... diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index e705dd3ea855..12f424fd4167 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -5,7 +5,7 @@ Exported symbols include: Dictionary with all registered number types (including aliases): - typeDict + sctypeDict Type objects (not all will be available, depends on platform): see variable sctypes for which ones you have @@ -79,7 +79,6 @@ \\-> object_ (not used much) (kind=O) """ -import types as _types import numbers import warnings @@ -91,7 +90,7 @@ from numpy.core.overrides import set_module # we add more at the bottom -__all__ = ['sctypeDict', 'typeDict', 'sctypes', +__all__ = ['sctypeDict', 'sctypes', 'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', 'issubdtype', 'datetime_data', 'datetime_as_string', @@ -512,15 +511,15 @@ def sctype2char(sctype): for key in _concrete_types: cast[key] = lambda x, k=key: array(x, copy=False).astype(k) -try: - ScalarType = [_types.IntType, _types.FloatType, _types.ComplexType, - _types.LongType, _types.BooleanType, - _types.StringType, _types.UnicodeType, _types.BufferType] -except AttributeError: - # Py3K - ScalarType = [int, float, complex, int, bool, bytes, str, memoryview] -ScalarType.extend(_concrete_types) +def _scalar_type_key(typ): + """A ``key`` function for `sorted`.""" + dt = dtype(typ) + return (dt.kind.lower(), dt.itemsize) + + +ScalarType = [int, float, complex, int, bool, bytes, str, memoryview] +ScalarType += sorted(_concrete_types, key=_scalar_type_key) ScalarType = tuple(ScalarType) @@ -542,6 +541,7 @@ def sctype2char(sctype): 'All':'?bhilqpBHILQPefdgFDGSUVOMm'} # backwards compatibility --- deprecated name +# Formal deprecation: Numpy 1.20.0, 2020-10-19 (see numpy/__init__.py) typeDict = sctypeDict # b -> boolean diff --git a/numpy/core/numerictypes.pyi b/numpy/core/numerictypes.pyi index 192015ff13b4..fd4aa3fdada3 100644 --- a/numpy/core/numerictypes.pyi +++ b/numpy/core/numerictypes.pyi @@ -1,9 +1,85 @@ -from typing import TypeVar, Optional, Type, Union, Tuple, Sequence, overload, Any +import sys +from typing import ( + TypeVar, + Optional, + Type, + Union, + Tuple, + Sequence, + overload, + Any, + TypeVar, + Dict, + List, +) -from numpy import generic, ndarray, dtype -from numpy.typing import DTypeLike +from numpy import ( + ndarray, + dtype, + generic, + bool_, + ubyte, + ushort, + uintc, + uint, + ulonglong, + byte, + short, + intc, + int_, + longlong, + half, + single, + double, + longdouble, + csingle, + cdouble, + clongdouble, + datetime64, + timedelta64, + object_, + str_, + bytes_, + void, +) -_DefaultType = TypeVar("_DefaultType") +from numpy.core._type_aliases import ( + sctypeDict as sctypeDict, + sctypes as sctypes, +) + +from numpy.typing import DTypeLike, ArrayLike + +if sys.version_info >= (3, 8): + from typing import Literal, Protocol, TypedDict +else: + from typing_extensions import Literal, Protocol, TypedDict + +_T = TypeVar("_T") +_ScalarType = TypeVar("_ScalarType", bound=generic) + +class _CastFunc(Protocol): + def __call__( + self, x: ArrayLike, k: DTypeLike = ... + ) -> ndarray[Any, dtype[Any]]: ... + +class _TypeCodes(TypedDict): + Character: Literal['c'] + Integer: Literal['bhilqp'] + UnsignedInteger: Literal['BHILQP'] + Float: Literal['efdg'] + Complex: Literal['FDG'] + AllInteger: Literal['bBhHiIlLqQpP'] + AllFloat: Literal['efdgFDG'] + Datetime: Literal['Mm'] + All: Literal['?bhilqpBHILQPefdgFDGSUVOMm'] + +class _typedict(Dict[Type[generic], _T]): + def __getitem__(self, key: DTypeLike) -> _T: ... + +__all__: List[str] + +# TODO: Clean up the annotations for the 7 functions below def maximum_sctype(t: DTypeLike) -> dtype: ... def issctype(rep: object) -> bool: ... @@ -13,8 +89,8 @@ def obj2sctype(rep: object) -> Optional[generic]: ... def obj2sctype(rep: object, default: None) -> Optional[generic]: ... @overload def obj2sctype( - rep: object, default: Type[_DefaultType] -) -> Union[generic, Type[_DefaultType]]: ... + rep: object, default: Type[_T] +) -> Union[generic, Type[_T]]: ... def issubclass_(arg1: object, arg2: Union[object, Tuple[object, ...]]) -> bool: ... def issubsctype( arg1: Union[ndarray, DTypeLike], arg2: Union[ndarray, DTypeLike] @@ -25,5 +101,40 @@ def find_common_type( array_types: Sequence[DTypeLike], scalar_types: Sequence[DTypeLike] ) -> dtype: ... -# TODO: Add annotations for the following objects: -# typeDict, nbytes, cast, ScalarType & typecodes +cast: _typedict[_CastFunc] +nbytes: _typedict[int] +typecodes: _TypeCodes +ScalarType: Tuple[ + Type[int], + Type[float], + Type[complex], + Type[int], + Type[bool], + Type[bytes], + Type[str], + Type[memoryview], + Type[bool_], + Type[csingle], + Type[cdouble], + Type[clongdouble], + Type[half], + Type[single], + Type[double], + Type[longdouble], + Type[byte], + Type[short], + Type[intc], + Type[int_], + Type[longlong], + Type[timedelta64], + Type[datetime64], + Type[object_], + Type[bytes_], + Type[str_], + Type[ubyte], + Type[ushort], + Type[uintc], + Type[uint], + Type[ulonglong], + Type[void], +] diff --git a/numpy/core/overrides.py b/numpy/core/overrides.py index c2b5fb7fa34d..70085d896fe7 100644 --- a/numpy/core/overrides.py +++ b/numpy/core/overrides.py @@ -18,11 +18,7 @@ NumPy arrays. If an array-like passed in as ``like`` supports the ``__array_function__`` protocol, the result will be defined by it. In this case, it ensures the creation of an array object - compatible with that passed in via this argument. - - .. note:: - The ``like`` keyword is an experimental feature pending on - acceptance of :ref:`NEP 35 `.""" + compatible with that passed in via this argument.""" ) def set_array_function_like_doc(public_api): diff --git a/numpy/core/records.py b/numpy/core/records.py index c2f6c6965d48..5bd13a698739 100644 --- a/numpy/core/records.py +++ b/numpy/core/records.py @@ -35,18 +35,20 @@ """ import os import warnings -from collections import Counter, OrderedDict +from collections import Counter +from contextlib import nullcontext from . import numeric as sb from . import numerictypes as nt -from numpy.compat import ( - os_fspath, contextlib_nullcontext -) +from numpy.compat import os_fspath from numpy.core.overrides import set_module from .arrayprint import get_printoptions # All of the functions allow formats to be a dtype -__all__ = ['record', 'recarray', 'format_parser'] +__all__ = [ + 'record', 'recarray', 'format_parser', + 'fromarrays', 'fromrecords', 'fromstring', 'fromfile', 'array', +] ndarray = sb.ndarray @@ -71,25 +73,14 @@ # of the letter code '(2,3)f4' and ' ( 2 , 3 ) f4 ' # are equally allowed -numfmt = nt.typeDict - -# taken from OrderedDict recipes in the Python documentation -# https://docs.python.org/3.3/library/collections.html#ordereddict-examples-and-recipes -class _OrderedCounter(Counter, OrderedDict): - """Counter that remembers the order elements are first encountered""" - - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, OrderedDict(self)) - - def __reduce__(self): - return self.__class__, (OrderedDict(self),) +numfmt = nt.sctypeDict def find_duplicate(list): """Find duplication in a list, return a list of duplicated elements""" return [ item - for item, counts in _OrderedCounter(list).items() + for item, counts in Counter(list).items() if counts > 1 ] @@ -242,12 +233,12 @@ class record(nt.void): def __repr__(self): if get_printoptions()['legacy'] == '1.13': return self.__str__() - return super(record, self).__repr__() + return super().__repr__() def __str__(self): if get_printoptions()['legacy'] == '1.13': return str(self.item()) - return super(record, self).__str__() + return super().__str__() def __getattribute__(self, attr): if attr in ('setfield', 'getfield', 'dtype'): @@ -516,7 +507,7 @@ def __setattr__(self, attr, val): return self.setfield(val, *res) def __getitem__(self, indx): - obj = super(recarray, self).__getitem__(indx) + obj = super().__getitem__(indx) # copy behavior of getattr, except that here # we might also be returning a single element @@ -914,7 +905,7 @@ def fromfile(fd, dtype=None, shape=None, offset=0, formats=None, # GH issue 2504. fd supports io.RawIOBase or io.BufferedIOBase interface. # Example of fd: gzip, BytesIO, BufferedReader # file already opened - ctx = contextlib_nullcontext(fd) + ctx = nullcontext(fd) else: # open file ctx = open(os_fspath(fd), 'rb') @@ -963,16 +954,16 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None, Parameters ---------- - obj: any + obj : any Input object. See Notes for details on how various input types are treated. - dtype: data-type, optional + dtype : data-type, optional Valid dtype for array. - shape: int or tuple of ints, optional + shape : int or tuple of ints, optional Shape of each array. - offset: int, optional + offset : int, optional Position in the file or buffer to start reading from. - strides: tuple of ints, optional + strides : tuple of ints, optional Buffer (`buf`) is interpreted according to these strides (strides define how many bytes each array element, row, column, etc. occupy in memory). @@ -980,7 +971,7 @@ def array(obj, dtype=None, shape=None, offset=0, strides=None, formats=None, If `dtype` is ``None``, these arguments are passed to `numpy.format_parser` to construct a dtype. See that function for detailed documentation. - copy: bool, optional + copy : bool, optional Whether to copy the input object (True), or to use a reference instead. This option only applies when the input is an ndarray or recarray. Defaults to True. diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 2ec5e1a64b37..b03e9f99005e 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -23,11 +23,6 @@ NPY_RELAXED_STRIDES_DEBUG = (os.environ.get('NPY_RELAXED_STRIDES_DEBUG', "0") != "0") NPY_RELAXED_STRIDES_DEBUG = NPY_RELAXED_STRIDES_DEBUG and NPY_RELAXED_STRIDES_CHECKING -# Set to True to use the new casting implementation as much as implemented. -# Allows running the full test suit to exercise the new machinery until -# it is used as default and the old version is eventually deleted. -NPY_USE_NEW_CASTINGIMPL = os.environ.get('NPY_USE_NEW_CASTINGIMPL', "0") != "0" - # XXX: ugly, we use a class to avoid calling twice some expensive functions in # config.h/numpyconfig.h. I don't see a better way because distutils force # config.h generation inside an Extension class, and as such sharing @@ -405,11 +400,6 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.system_info import (get_info, blas_opt_info, lapack_opt_info) - # Accelerate is buggy, disallow it. See also numpy/linalg/setup.py - for opt_order in (blas_opt_info.blas_order, lapack_opt_info.lapack_order): - if 'accelerate' in opt_order: - opt_order.remove('accelerate') - config = Configuration('core', parent_package, top_path) local_dir = config.local_path codegen_dir = join(local_dir, 'code_generators') @@ -468,14 +458,14 @@ def generate_config_h(ext, build_dir): # Use relaxed stride checking if NPY_RELAXED_STRIDES_CHECKING: moredefs.append(('NPY_RELAXED_STRIDES_CHECKING', 1)) + else: + moredefs.append(('NPY_RELAXED_STRIDES_CHECKING', 0)) # Use bogus stride debug aid when relaxed strides are enabled if NPY_RELAXED_STRIDES_DEBUG: moredefs.append(('NPY_RELAXED_STRIDES_DEBUG', 1)) - - # Use the new experimental casting implementation in NumPy 1.20: - if NPY_USE_NEW_CASTINGIMPL: - moredefs.append(('NPY_USE_NEW_CASTINGIMPL', 1)) + else: + moredefs.append(('NPY_RELAXED_STRIDES_DEBUG', 0)) # Get long double representation rep = check_long_double_representation(config_cmd) @@ -710,8 +700,10 @@ def get_mathlib_info(*args): config.add_extension('_multiarray_tests', sources=[join('src', 'multiarray', '_multiarray_tests.c.src'), - join('src', 'common', 'mem_overlap.c')], + join('src', 'common', 'mem_overlap.c'), + join('src', 'common', 'npy_argparse.c')], depends=[join('src', 'common', 'mem_overlap.h'), + join('src', 'common', 'npy_argparse.h'), join('src', 'common', 'npy_extint128.h')], libraries=['npymath']) @@ -725,6 +717,7 @@ def get_mathlib_info(*args): join('src', 'common', 'cblasfuncs.h'), join('src', 'common', 'lowlevel_strided_loops.h'), join('src', 'common', 'mem_overlap.h'), + join('src', 'common', 'npy_argparse.h'), join('src', 'common', 'npy_cblas.h'), join('src', 'common', 'npy_config.h'), join('src', 'common', 'npy_ctypes.h'), @@ -743,6 +736,7 @@ def get_mathlib_info(*args): common_src = [ join('src', 'common', 'array_assign.c'), join('src', 'common', 'mem_overlap.c'), + join('src', 'common', 'npy_argparse.c'), join('src', 'common', 'npy_longdouble.c'), join('src', 'common', 'templ_common.h.src'), join('src', 'common', 'ucsnarrow.c'), @@ -782,12 +776,14 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'npy_buffer.h'), join('src', 'multiarray', 'calculation.h'), join('src', 'multiarray', 'common.h'), + join('src', 'multiarray', 'common_dtype.h'), join('src', 'multiarray', 'convert_datatype.h'), join('src', 'multiarray', 'convert.h'), join('src', 'multiarray', 'conversion_utils.h'), join('src', 'multiarray', 'ctors.h'), join('src', 'multiarray', 'descriptor.h'), join('src', 'multiarray', 'dtypemeta.h'), + join('src', 'multiarray', 'dtype_transfer.h'), join('src', 'multiarray', 'dragon4.h'), join('src', 'multiarray', 'einsum_debug.h'), join('src', 'multiarray', 'einsum_sumprod.h'), @@ -843,6 +839,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'calculation.c'), join('src', 'multiarray', 'compiled_base.c'), join('src', 'multiarray', 'common.c'), + join('src', 'multiarray', 'common_dtype.c'), join('src', 'multiarray', 'convert.c'), join('src', 'multiarray', 'convert_datatype.c'), join('src', 'multiarray', 'conversion_utils.c'), @@ -916,8 +913,13 @@ def generate_umath_c(ext, build_dir): join('src', 'umath', 'funcs.inc.src'), join('src', 'umath', 'simd.inc.src'), join('src', 'umath', 'loops.h.src'), + join('src', 'umath', 'loops_utils.h.src'), join('src', 'umath', 'loops.c.src'), join('src', 'umath', 'loops_unary_fp.dispatch.c.src'), + join('src', 'umath', 'loops_arithm_fp.dispatch.c.src'), + join('src', 'umath', 'loops_arithmetic.dispatch.c.src'), + join('src', 'umath', 'loops_trigonometric.dispatch.c.src'), + join('src', 'umath', 'loops_exponent_log.dispatch.c.src'), join('src', 'umath', 'matmul.h.src'), join('src', 'umath', 'matmul.c.src'), join('src', 'umath', 'clip.h.src'), diff --git a/numpy/core/setup_common.py b/numpy/core/setup_common.py index ba3e215b3998..a700610d3dec 100644 --- a/numpy/core/setup_common.py +++ b/numpy/core/setup_common.py @@ -42,6 +42,7 @@ # 0x0000000d - 1.16.x # 0x0000000d - 1.19.x # 0x0000000e - 1.20.x +# 0x0000000e - 1.21.x C_API_VERSION = 0x0000000e class MismatchCAPIWarning(Warning): @@ -51,7 +52,7 @@ def is_released(config): """Return True if a released version of numpy is detected.""" from distutils.version import LooseVersion - v = config.get_version('../version.py') + v = config.get_version('../_version.py') if v is None: raise ValueError("Could not get version") pv = LooseVersion(vstring=v).version @@ -194,7 +195,8 @@ def check_api_version(apiversion, codegen_dir): ('__attribute__((target ("avx512f,avx512dq,avx512bw,avx512vl,avx512cd")))', 'attribute_target_avx512_skx_with_intrinsics', '__mmask8 temp = _mm512_fpclass_pd_mask(_mm512_set1_pd(1.0), 0x01);\ - __m512i temp = _mm512_castps_si512(_mm512_set1_ps(1.0));\ + __m512i unused_temp = \ + _mm512_castps_si512(_mm512_set1_ps(1.0));\ _mm_mask_storeu_epi8(NULL, 0xFF, _mm_broadcastmb_epi64(temp))', 'immintrin.h'), ] @@ -317,8 +319,8 @@ def pyod(filename): out : seq list of lines of od output - Note - ---- + Notes + ----- We only implement enough to get the necessary information for long double representation, this is not intended as a compatible replacement for od. """ diff --git a/numpy/core/shape_base.py b/numpy/core/shape_base.py index e4dc30d4c86e..a81a04f7ff0e 100644 --- a/numpy/core/shape_base.py +++ b/numpy/core/shape_base.py @@ -8,8 +8,7 @@ from . import numeric as _nx from . import overrides -from ._asarray import array, asanyarray -from .multiarray import normalize_axis_index +from .multiarray import array, asanyarray, normalize_axis_index from . import fromnumeric as _from_nx @@ -258,20 +257,20 @@ def vstack(tup): Examples -------- >>> a = np.array([1, 2, 3]) - >>> b = np.array([2, 3, 4]) + >>> b = np.array([4, 5, 6]) >>> np.vstack((a,b)) array([[1, 2, 3], - [2, 3, 4]]) + [4, 5, 6]]) >>> a = np.array([[1], [2], [3]]) - >>> b = np.array([[2], [3], [4]]) + >>> b = np.array([[4], [5], [6]]) >>> np.vstack((a,b)) array([[1], [2], [3], - [2], - [3], - [4]]) + [4], + [5], + [6]]) """ if not overrides.ARRAY_FUNCTION_ENABLED: @@ -321,15 +320,15 @@ def hstack(tup): Examples -------- >>> a = np.array((1,2,3)) - >>> b = np.array((2,3,4)) + >>> b = np.array((4,5,6)) >>> np.hstack((a,b)) - array([1, 2, 3, 2, 3, 4]) + array([1, 2, 3, 4, 5, 6]) >>> a = np.array([[1],[2],[3]]) - >>> b = np.array([[2],[3],[4]]) + >>> b = np.array([[4],[5],[6]]) >>> np.hstack((a,b)) - array([[1, 2], - [2, 3], - [3, 4]]) + array([[1, 4], + [2, 5], + [3, 6]]) """ if not overrides.ARRAY_FUNCTION_ENABLED: @@ -403,15 +402,15 @@ def stack(arrays, axis=0, out=None): (3, 4, 10) >>> a = np.array([1, 2, 3]) - >>> b = np.array([2, 3, 4]) + >>> b = np.array([4, 5, 6]) >>> np.stack((a, b)) array([[1, 2, 3], - [2, 3, 4]]) + [4, 5, 6]]) >>> np.stack((a, b), axis=-1) - array([[1, 2], - [2, 3], - [3, 4]]) + array([[1, 4], + [2, 5], + [3, 6]]) """ if not overrides.ARRAY_FUNCTION_ENABLED: @@ -607,7 +606,7 @@ def _block_info_recursion(arrays, max_depth, result_ndim, depth=0): The arrays to check max_depth : list of int The number of nested lists - result_ndim: int + result_ndim : int The number of dimensions in thefinal array. Returns @@ -786,9 +785,9 @@ def block(arrays): array([1, 2, 3]) >>> a = np.array([1, 2, 3]) - >>> b = np.array([2, 3, 4]) + >>> b = np.array([4, 5, 6]) >>> np.block([a, b, 10]) # hstack([a, b, 10]) - array([ 1, 2, 3, 2, 3, 4, 10]) + array([ 1, 2, 3, 4, 5, 6, 10]) >>> A = np.ones((2, 2), int) >>> B = 2 * A @@ -799,10 +798,10 @@ def block(arrays): With a list of depth 2, `block` can be used in place of `vstack`: >>> a = np.array([1, 2, 3]) - >>> b = np.array([2, 3, 4]) + >>> b = np.array([4, 5, 6]) >>> np.block([[a], [b]]) # vstack([a, b]) array([[1, 2, 3], - [2, 3, 4]]) + [4, 5, 6]]) >>> A = np.ones((2, 2), int) >>> B = 2 * A diff --git a/numpy/core/shape_base.pyi b/numpy/core/shape_base.pyi index b20598b1ab5b..ec40a88143b1 100644 --- a/numpy/core/shape_base.pyi +++ b/numpy/core/shape_base.pyi @@ -7,9 +7,7 @@ from numpy.typing import ArrayLike if sys.version_info >= (3, 8): from typing import SupportsIndex else: - from typing_extensions import Protocol - class SupportsIndex(Protocol): - def __index__(self) -> int: ... + from typing_extensions import SupportsIndex _ArrayType = TypeVar("_ArrayType", bound=ndarray) diff --git a/numpy/core/src/_simd/_simd.dispatch.c.src b/numpy/core/src/_simd/_simd.dispatch.c.src index 18c38387165c..54770959c362 100644 --- a/numpy/core/src/_simd/_simd.dispatch.c.src +++ b/numpy/core/src/_simd/_simd.dispatch.c.src @@ -1,4 +1,4 @@ -/*@targets $werror #simd_test*/ +/*@targets #simd_test*/ #include "_simd.h" #include "_simd_inc.h" @@ -9,20 +9,25 @@ #include "_simd_arg.inc" #include "_simd_easyintrin.inc" -/************************************************************************* - * Defining NPYV intrinsics as module functions - *************************************************************************/ +//######################################################################### +//## Defining NPYV intrinsics as module functions +//######################################################################### /**begin repeat * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# * #bsfx = b8, b8, b16, b16, b32, b32, b64, b64, b32, b64# + * #esfx = u16, s8, u32,s16, u32, s32, u64, s64, f32, f64# + * #expand_sup= 1, 0, 1, 0, 0, 0, 0, 0, 0, 0# * #simd_sup = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# * #fp_only = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# - * #sum_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #sumup_sup = 1, 0, 1, 0, 0, 0, 0, 0, 0, 0# + * #sum_sup = 0, 0, 0, 0, 1, 0, 1, 0, 1, 1# + * #rev64_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 0# * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# + * #intdiv_sup= 1, 1, 1, 1, 1, 1, 1, 1, 0, 0# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# */ @@ -227,7 +232,6 @@ err: /**end repeat1**/ #endif // @ncont_sup@ - /*************************** * Misc ***************************/ @@ -289,6 +293,10 @@ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@x2, v@sfx@, v@sfx@) /**end repeat1**/ +#if @rev64_sup@ +SIMD_IMPL_INTRIN_1(rev64_@sfx@, v@sfx@, v@sfx@) +#endif + /*************************** * Operators ***************************/ @@ -301,7 +309,7 @@ SIMD_IMPL_INTRIN_2IMM(shri_@sfx@, v@sfx@, v@sfx@, @shr_imm@) #endif // shl_imm /**begin repeat1 - * #intrin = and, or, xor# + * #intrin = and, or, xor# */ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) /**end repeat1**/ @@ -309,7 +317,7 @@ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) SIMD_IMPL_INTRIN_1(not_@sfx@, v@sfx@, v@sfx@) /**begin repeat1 - * #intrin = cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# + * #intrin = cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# */ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@bsfx@, v@sfx@, v@sfx@) /**end repeat1**/ @@ -319,19 +327,21 @@ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@bsfx@, v@sfx@, v@sfx@) ***************************/ SIMD_IMPL_INTRIN_1(cvt_@sfx@_@bsfx@, v@sfx@, v@bsfx@) SIMD_IMPL_INTRIN_1(cvt_@bsfx@_@sfx@, v@bsfx@, v@sfx@) - +#if @expand_sup@ +SIMD_IMPL_INTRIN_1(expand_@esfx@_@sfx@, v@esfx@x2, v@sfx@) +#endif // expand_sup /*************************** * Arithmetic ***************************/ /**begin repeat1 - * #intrin = add, sub# + * #intrin = add, sub# */ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) /**end repeat1**/ #if @sat_sup@ /**begin repeat1 - * #intrin = adds, subs# + * #intrin = adds, subs# */ SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) /**end repeat1**/ @@ -345,9 +355,14 @@ SIMD_IMPL_INTRIN_2(mul_@sfx@, v@sfx@, v@sfx@, v@sfx@) SIMD_IMPL_INTRIN_2(div_@sfx@, v@sfx@, v@sfx@, v@sfx@) #endif // div_sup +#if @intdiv_sup@ +SIMD_IMPL_INTRIN_1(divisor_@sfx@, v@sfx@x3, @sfx@) +SIMD_IMPL_INTRIN_2(divc_@sfx@, v@sfx@, v@sfx@, v@sfx@x3) +#endif // intdiv_sup + #if @fused_sup@ /**begin repeat1 - * #intrin = muladd, mulsub, nmuladd, nmulsub# + * #intrin = muladd, mulsub, nmuladd, nmulsub# */ SIMD_IMPL_INTRIN_3(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@, v@sfx@) /**end repeat1**/ @@ -357,39 +372,117 @@ SIMD_IMPL_INTRIN_3(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@, v@sfx@) SIMD_IMPL_INTRIN_1(sum_@sfx@, @sfx@, v@sfx@) #endif // sum_sup +#if @sumup_sup@ +SIMD_IMPL_INTRIN_1(sumup_@sfx@, @esfx@, v@sfx@) +#endif // sumup_sup + /*************************** * Math ***************************/ #if @fp_only@ /**begin repeat1 - * #intrin = sqrt, recip, abs, square# + * #intrin = sqrt, recip, abs, square# */ SIMD_IMPL_INTRIN_1(@intrin@_@sfx@, v@sfx@, v@sfx@) /**end repeat1**/ #endif +/**begin repeat1 + * #intrin = max, min# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + +#if @fp_only@ +/**begin repeat1 + * #intrin = maxp, minp# + */ +SIMD_IMPL_INTRIN_2(@intrin@_@sfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ +#endif + +/*************************** + * Mask operations + ***************************/ +/**begin repeat1 + * #intrin = ifadd, ifsub# + */ + SIMD_IMPL_INTRIN_4(@intrin@_@sfx@, v@sfx@, v@bsfx@, v@sfx@, v@sfx@, v@sfx@) +/**end repeat1**/ + #endif // simd_sup /**end repeat**/ -/*************************** +/************************************************************************* * Variant - ***************************/ + ************************************************************************/ SIMD_IMPL_INTRIN_0N(cleanup) /************************************************************************* - * Attach module functions - *************************************************************************/ + * A special section for f32/f64 intrinsics outside the main repeater + ************************************************************************/ +/*************************** + * Operators + ***************************/ +// check special cases +SIMD_IMPL_INTRIN_1(notnan_f32, vb32, vf32) +#if NPY_SIMD_F64 + SIMD_IMPL_INTRIN_1(notnan_f64, vb64, vf64) +#endif +/*************************** + * Conversions + ***************************/ +// round to nearest integer (assume even) +SIMD_IMPL_INTRIN_1(round_s32_f32, vs32, vf32) +#if NPY_SIMD_F64 + SIMD_IMPL_INTRIN_2(round_s32_f64, vs32, vf64, vf64) +#endif + +/************************************************************************* + * A special section for boolean intrinsics outside the main repeater + ************************************************************************/ +/*************************** + * Operators + ***************************/ +// Logical +/**begin repeat + * #bsfx = b8, b16, b32, b64# + */ +SIMD_IMPL_INTRIN_2(and_@bsfx@, v@bsfx@, v@bsfx@, v@bsfx@) +SIMD_IMPL_INTRIN_2(or_@bsfx@, v@bsfx@, v@bsfx@, v@bsfx@) +SIMD_IMPL_INTRIN_2(xor_@bsfx@, v@bsfx@, v@bsfx@, v@bsfx@) +SIMD_IMPL_INTRIN_1(not_@bsfx@, v@bsfx@, v@bsfx@) +/**end repeat**/ +/*************************** + * Conversions + ***************************/ +// Convert mask vector to integer bitfield +/**begin repeat + * #bsfx = b8, b16, b32, b64# + */ +SIMD_IMPL_INTRIN_1(tobits_@bsfx@, u64, v@bsfx@) +/**end repeat**/ + + +//######################################################################### +//## Attach module functions +//######################################################################### static PyMethodDef simd__intrinsics_methods[] = { /**begin repeat * #sfx = u8, s8, u16, s16, u32, s32, u64, s64, f32, f64# * #bsfx = b8, b8, b16, b16, b32, b32, b64, b64, b32, b64# + * #esfx = u16, s8, u32,s16, u32, s32, u64, s64, f32, f64# + * #expand_sup =1, 0, 1, 0, 0, 0, 0, 0, 0, 0# * #simd_sup = 1, 1, 1, 1, 1, 1, 1, 1, 1, NPY_SIMD_F64# * #fp_only = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #sat_sup = 1, 1, 1, 1, 0, 0, 0, 0, 0, 0# * #mul_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 1# * #div_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# * #fused_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# - * #sum_sup = 0, 0, 0, 0, 0, 0, 0, 0, 1, 1# + * #sumup_sup = 1, 0, 1, 0, 0, 0, 0, 0, 0, 0# + * #sum_sup = 0, 0, 0, 0, 1, 0, 1, 0, 1, 1# + * #rev64_sup = 1, 1, 1, 1, 1, 1, 0, 0, 1, 0# * #ncont_sup = 0, 0, 0, 0, 1, 1, 1, 1, 1, 1# + * #intdiv_sup= 1, 1, 1, 1, 1, 1, 1, 1, 0, 0# * #shl_imm = 0, 0, 15, 15, 31, 31, 63, 63, 0, 0# * #shr_imm = 0, 0, 16, 16, 32, 32, 64, 64, 0, 0# */ @@ -416,7 +509,6 @@ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ #endif // ncont_sup - /*************************** * Misc ***************************/ @@ -444,8 +536,9 @@ SIMD_INTRIN_DEF(@intrin@_@sfx@) SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ -SIMD_INTRIN_DEF(cvt_@sfx@_@bsfx@) -SIMD_INTRIN_DEF(cvt_@bsfx@_@sfx@) +#if @rev64_sup@ +SIMD_INTRIN_DEF(rev64_@sfx@) +#endif /*************************** * Operators @@ -459,7 +552,7 @@ SIMD_INTRIN_DEF(@intrin@_@sfx@) #endif // shl_imm /**begin repeat1 - * #intrin = and, or, xor, not, cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# + * #intrin = and, or, xor, not, cmpeq, cmpneq, cmpgt, cmpge, cmplt, cmple# */ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ @@ -469,19 +562,21 @@ SIMD_INTRIN_DEF(@intrin@_@sfx@) ***************************/ SIMD_INTRIN_DEF(cvt_@sfx@_@bsfx@) SIMD_INTRIN_DEF(cvt_@bsfx@_@sfx@) - +#if @expand_sup@ +SIMD_INTRIN_DEF(expand_@esfx@_@sfx@) +#endif // expand_sup /*************************** * Arithmetic ***************************/ /**begin repeat1 - * #intrin = add, sub# + * #intrin = add, sub# */ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ #if @sat_sup@ /**begin repeat1 - * #intrin = adds, subs# + * #intrin = adds, subs# */ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ @@ -495,9 +590,14 @@ SIMD_INTRIN_DEF(mul_@sfx@) SIMD_INTRIN_DEF(div_@sfx@) #endif // div_sup +#if @intdiv_sup@ +SIMD_INTRIN_DEF(divisor_@sfx@) +SIMD_INTRIN_DEF(divc_@sfx@) +#endif // intdiv_sup + #if @fused_sup@ /**begin repeat1 - * #intrin = muladd, mulsub, nmuladd, nmulsub# + * #intrin = muladd, mulsub, nmuladd, nmulsub# */ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ @@ -507,33 +607,104 @@ SIMD_INTRIN_DEF(@intrin@_@sfx@) SIMD_INTRIN_DEF(sum_@sfx@) #endif // sum_sup +#if @sumup_sup@ +SIMD_INTRIN_DEF(sumup_@sfx@) +#endif // sumup_sup /*************************** * Math ***************************/ #if @fp_only@ /**begin repeat1 - * #intrin = sqrt, recip, abs, square# + * #intrin = sqrt, recip, abs, square# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ +#endif + +/**begin repeat1 + * #intrin = max, min# + */ +SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + +#if @fp_only@ +/**begin repeat1 + * #intrin = maxp, minp# */ SIMD_INTRIN_DEF(@intrin@_@sfx@) /**end repeat1**/ #endif +/*************************** + * Mask operations + ***************************/ +/**begin repeat1 + * #intrin = ifadd, ifsub# + */ + SIMD_INTRIN_DEF(@intrin@_@sfx@) +/**end repeat1**/ + #endif // simd_sup /**end repeat**/ +/************************************************************************* + * Variant + ************************************************************************/ +SIMD_INTRIN_DEF(cleanup) +/************************************************************************* + * A special section for f32/f64 intrinsics outside the main repeater + ************************************************************************/ /*************************** - * Variant + * Operators ***************************/ -SIMD_INTRIN_DEF(cleanup) -/***************************/ +// check special cases +SIMD_INTRIN_DEF(notnan_f32) +#if NPY_SIMD_F64 + SIMD_INTRIN_DEF(notnan_f64) +#endif +/*************************** + * Conversions + ***************************/ +// round to nearest integer (assume even) +SIMD_INTRIN_DEF(round_s32_f32) +#if NPY_SIMD_F64 + SIMD_INTRIN_DEF(round_s32_f64) +#endif + +/************************************************************************* + * A special section for boolean intrinsics outside the main repeater + ************************************************************************/ +/*************************** + * Operators + ***************************/ +// Logical +/**begin repeat + * #bsfx = b8, b16, b32, b64# + */ +SIMD_INTRIN_DEF(and_@bsfx@) +SIMD_INTRIN_DEF(or_@bsfx@) +SIMD_INTRIN_DEF(xor_@bsfx@) +SIMD_INTRIN_DEF(not_@bsfx@) +/**end repeat**/ +/*************************** + * Conversions + ***************************/ +// Convert mask vector to integer bitfield +/**begin repeat + * #bsfx = b8, b16, b32, b64# + */ +SIMD_INTRIN_DEF(tobits_@bsfx@) +/**end repeat**/ + +/************************************************************************/ {NULL, NULL, 0, NULL} }; // PyMethodDef #endif // NPY_SIMD -/************************************************************************* - * Defining a separate module for each target - *************************************************************************/ +//######################################################################### +//## Defining a separate module for each target +//######################################################################### NPY_VISIBILITY_HIDDEN PyObject * NPY_CPU_DISPATCH_CURFX(simd_create_module)(void) { @@ -561,6 +732,9 @@ NPY_CPU_DISPATCH_CURFX(simd_create_module)(void) if (PyModule_AddIntConstant(m, "simd_f64", NPY_SIMD_F64)) { goto err; } + if (PyModule_AddIntConstant(m, "simd_fma3", NPY_SIMD_FMA3)) { + goto err; + } if (PyModule_AddIntConstant(m, "simd_width", NPY_SIMD_WIDTH)) { goto err; } diff --git a/numpy/core/src/_simd/_simd_easyintrin.inc b/numpy/core/src/_simd/_simd_easyintrin.inc index 54e7ccf01f50..4521b2d87f07 100644 --- a/numpy/core/src/_simd/_simd_easyintrin.inc +++ b/numpy/core/src/_simd/_simd_easyintrin.inc @@ -87,10 +87,10 @@ simd_arg_converter, &arg1, \ simd_arg_converter, &arg2 \ )) return NULL; \ - simd_data data; \ + simd_data data = {.u64 = 0}; \ data.RET = NPY_CAT(SIMD__IMPL_COUNT_, CONST_RNG)( \ SIMD__REPEAT_2IMM, NAME, IN0 \ - ) npyv_##NAME(arg1.data.IN0, 0); \ + ) data.RET; \ simd_arg_free(&arg1); \ simd_arg ret = { \ .data = data, .dtype = simd_data_##RET \ @@ -123,6 +123,36 @@ }; \ return simd_arg_to_obj(&ret); \ } + +#define SIMD_IMPL_INTRIN_4(NAME, RET, IN0, IN1, IN2, IN3) \ + static PyObject *simd__intrin_##NAME \ + (PyObject* NPY_UNUSED(self), PyObject *args) \ + { \ + simd_arg arg1 = {.dtype = simd_data_##IN0}; \ + simd_arg arg2 = {.dtype = simd_data_##IN1}; \ + simd_arg arg3 = {.dtype = simd_data_##IN2}; \ + simd_arg arg4 = {.dtype = simd_data_##IN3}; \ + if (!PyArg_ParseTuple( \ + args, "O&O&O&O&:"NPY_TOSTRING(NAME), \ + simd_arg_converter, &arg1, \ + simd_arg_converter, &arg2, \ + simd_arg_converter, &arg3, \ + simd_arg_converter, &arg4 \ + )) return NULL; \ + simd_data data = {.RET = npyv_##NAME( \ + arg1.data.IN0, arg2.data.IN1, \ + arg3.data.IN2, arg4.data.IN3 \ + )}; \ + simd_arg_free(&arg1); \ + simd_arg_free(&arg2); \ + simd_arg_free(&arg3); \ + simd_arg_free(&arg4); \ + simd_arg ret = { \ + .data = data, .dtype = simd_data_##RET \ + }; \ + return simd_arg_to_obj(&ret); \ + } + /** * Helper macros for repeating and expand a certain macro. * Mainly used for converting a scalar to an immediate constant. diff --git a/numpy/core/src/_simd/_simd_vector.inc b/numpy/core/src/_simd/_simd_vector.inc index 2a1378f22100..d4b6310fde04 100644 --- a/numpy/core/src/_simd/_simd_vector.inc +++ b/numpy/core/src/_simd/_simd_vector.inc @@ -86,7 +86,22 @@ static PyTypeObject PySIMDVectorType = { /************************************ ** Protected Definitions ************************************/ -static PySIMDVectorObject * +/* + * Force inlining the following functions on CYGWIN to avoid spilling vector + * registers into the stack to workaround GCC/WIN64 bug that performs + * miss-align load variable of 256/512-bit vector from non-aligned + * 256/512-bit stack pointer. + * + * check the following links for more clearification: + * https://github.com/numpy/numpy/pull/18330#issuecomment-821539919 + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=49001 + */ +#if defined(__CYGWIN__) || (defined(__GNUC__) && defined(_WIN64)) + #define CYG_FINLINE NPY_FINLINE +#else + #define CYG_FINLINE static +#endif +CYG_FINLINE PySIMDVectorObject * PySIMDVector_FromData(simd_data data, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); @@ -118,7 +133,7 @@ PySIMDVector_FromData(simd_data data, simd_data_type dtype) return vec; } -static simd_data +CYG_FINLINE simd_data PySIMDVector_AsData(PySIMDVectorObject *vec, simd_data_type dtype) { const simd_data_info *info = simd_data_getinfo(dtype); diff --git a/numpy/core/src/common/lowlevel_strided_loops.h b/numpy/core/src/common/lowlevel_strided_loops.h index 12aa61822a14..3df054b40727 100644 --- a/numpy/core/src/common/lowlevel_strided_loops.h +++ b/numpy/core/src/common/lowlevel_strided_loops.h @@ -2,6 +2,8 @@ #define __LOWLEVEL_STRIDED_LOOPS_H #include "common.h" #include +#include +#include "dtype_transfer.h" #include "mem_overlap.h" /* For PyArray_ macros used below */ @@ -30,22 +32,26 @@ * Use NPY_AUXDATA_CLONE and NPY_AUXDATA_FREE to deal with this data. * */ -typedef int (PyArray_StridedUnaryOp)( - char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata); +// TODO: FIX! That comment belongs to something now in array-method /* * This is for pointers to functions which behave exactly as - * for PyArray_StridedUnaryOp, but with an additional mask controlling + * for PyArrayMethod_StridedLoop, but with an additional mask controlling * which values are transformed. * + * TODO: We should move this mask "capability" to the ArrayMethod itself + * probably. Although for NumPy internal things this works decently, + * and exposing it there should be well thought out to be useful beyond + * NumPy if possible. + * * In particular, the 'i'-th element is operated on if and only if * mask[i*mask_stride] is true. */ typedef int (PyArray_MaskedStridedUnaryOp)( - char *dst, npy_intp dst_stride, char *src, npy_intp src_stride, + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, npy_bool *mask, npy_intp mask_stride, - npy_intp N, npy_intp src_itemsize, NpyAuxData *transferdata); + NpyAuxData *auxdata); /* * Gives back a function pointer to a specialized function for copying @@ -65,7 +71,7 @@ typedef int (PyArray_MaskedStridedUnaryOp)( * Should be the item size if it will always be the same, 0 otherwise. * */ -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * PyArray_GetStridedCopyFn(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize); @@ -80,7 +86,7 @@ PyArray_GetStridedCopyFn(int aligned, * * Parameters are as for PyArray_GetStridedCopyFn. */ -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * PyArray_GetStridedCopySwapFn(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize); @@ -95,7 +101,7 @@ PyArray_GetStridedCopySwapFn(int aligned, * * Parameters are as for PyArray_GetStridedCopyFn. */ -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * PyArray_GetStridedCopySwapPairFn(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize); @@ -114,7 +120,7 @@ NPY_NO_EXPORT int PyArray_GetStridedZeroPadCopyFn(int aligned, int unicode_swap, npy_intp src_stride, npy_intp dst_stride, npy_intp src_itemsize, npy_intp dst_itemsize, - PyArray_StridedUnaryOp **outstransfer, + PyArrayMethod_StridedLoop **outstransfer, NpyAuxData **outtransferdata); /* @@ -123,7 +129,7 @@ PyArray_GetStridedZeroPadCopyFn(int aligned, int unicode_swap, * to dst_type_num. If a conversion is unsupported, returns NULL * without setting a Python exception. */ -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * PyArray_GetStridedNumericCastFn(int aligned, npy_intp src_stride, npy_intp dst_stride, int src_type_num, int dst_type_num); @@ -138,7 +144,7 @@ NPY_NO_EXPORT int PyArray_GetDTypeCopySwapFn(int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *dtype, - PyArray_StridedUnaryOp **outstransfer, + PyArrayMethod_StridedLoop **outstransfer, NpyAuxData **outtransferdata); /* @@ -159,8 +165,7 @@ PyArray_GetDTypeCopySwapFn(int aligned, * Should be the dst stride if it will always be the same, * NPY_MAX_INTP otherwise. * src_dtype: - * The data type of source data. If this is NULL, a transfer - * function which sets the destination to zeros is produced. + * The data type of source data. Must not be NULL. * dst_dtype: * The data type of destination data. If this is NULL and * move_references is 1, a transfer function which decrements @@ -169,12 +174,10 @@ PyArray_GetDTypeCopySwapFn(int aligned, * If 0, the destination data gets new reference ownership. * If 1, the references from the source data are moved to * the destination data. - * out_stransfer: - * The resulting transfer function is placed here. - * out_transferdata: - * The auxiliary data for the transfer function is placed here. - * When finished with the transfer function, the caller must call - * NPY_AUXDATA_FREE on this data. + * cast_info: + * A pointer to an (uninitialized) `NPY_cast_info` struct, the caller + * must call `NPY_cast_info_xfree` on it (except on error) and handle + * its memory livespan. * out_needs_api: * If this is non-NULL, and the transfer function produced needs * to call into the (Python) API, this gets set to 1. This @@ -192,10 +195,27 @@ PyArray_GetDTypeTransferFunction(int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int move_references, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, + NPY_cast_info *cast_info, int *out_needs_api); +NPY_NO_EXPORT int +get_fields_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api); + +NPY_NO_EXPORT int +get_subarray_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api); + /* * This is identical to PyArray_GetDTypeTransferFunction, but returns a * transfer function which also takes a mask as a parameter. The mask is used @@ -220,8 +240,7 @@ PyArray_GetMaskedDTypeTransferFunction(int aligned, PyArray_Descr *dst_dtype, PyArray_Descr *mask_dtype, int move_references, - PyArray_MaskedStridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, + NPY_cast_info *cast_info, int *out_needs_api); /* @@ -276,11 +295,9 @@ PyArray_CastRawArrays(npy_intp count, * sizes, for example a casting operation, the 'stransfer' function * should be specialized for that, in which case 'stransfer' will use * this parameter as the source item size. - * stransfer: - * The strided transfer function. - * transferdata: - * An auxiliary data pointer passed to the strided transfer function. - * This follows the conventions of NpyAuxData objects. + * cast_info: + * Pointer to the NPY_cast_info struct which summarizes all information + * necessary to perform a cast. */ NPY_NO_EXPORT npy_intp PyArray_TransferNDimToStrided(npy_intp ndim, @@ -289,8 +306,7 @@ PyArray_TransferNDimToStrided(npy_intp ndim, npy_intp const *coords, npy_intp coords_inc, npy_intp const *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, - PyArray_StridedUnaryOp *stransfer, - NpyAuxData *transferdata); + NPY_cast_info *cast_info); NPY_NO_EXPORT npy_intp PyArray_TransferStridedToNDim(npy_intp ndim, @@ -299,8 +315,7 @@ PyArray_TransferStridedToNDim(npy_intp ndim, npy_intp const *coords, npy_intp coords_inc, npy_intp const *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, - PyArray_StridedUnaryOp *stransfer, - NpyAuxData *transferdata); + NPY_cast_info *cast_info); NPY_NO_EXPORT npy_intp PyArray_TransferMaskedStridedToNDim(npy_intp ndim, @@ -310,8 +325,7 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, npy_intp const *coords, npy_intp coords_inc, npy_intp const *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, - PyArray_MaskedStridedUnaryOp *stransfer, - NpyAuxData *data); + NPY_cast_info *cast_info); NPY_NO_EXPORT int mapiter_trivial_get(PyArrayObject *self, PyArrayObject *ind, @@ -633,25 +647,6 @@ npy_bswap8_unaligned(char * x) * // Create iterator, etc... * } * - * Here is example code for a pair of arrays: - * - * if (PyArray_TRIVIALLY_ITERABLE_PAIR(a1, a2)) { - * char *data1, *data2; - * npy_intp count, stride1, stride2; - * - * PyArray_PREPARE_TRIVIAL_PAIR_ITERATION(a1, a2, count, - * data1, data2, stride1, stride2); - * - * while (count--) { - * // Use the data1 and data2 pointers - * - * data1 += stride1; - * data2 += stride2; - * } - * } - * else { - * // Create iterator, etc... - * } */ /* @@ -762,16 +757,6 @@ PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK(PyArrayObject *arr1, PyArrayObject *arr PyArray_STRIDE(arr, 0) : \ PyArray_ITEMSIZE(arr))); -#define PyArray_TRIVIALLY_ITERABLE_PAIR(arr1, arr2, arr1_read, arr2_read) ( \ - PyArray_TRIVIALLY_ITERABLE(arr1) && \ - (PyArray_NDIM(arr2) == 0 || \ - PyArray_EQUIVALENTLY_ITERABLE_BASE(arr1, arr2) || \ - (PyArray_NDIM(arr1) == 0 && \ - PyArray_TRIVIALLY_ITERABLE(arr2) \ - ) \ - ) && \ - PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK(arr1, arr2, arr1_read, arr2_read) \ - ) #define PyArray_PREPARE_TRIVIAL_PAIR_ITERATION(arr1, arr2, \ count, \ data1, data2, \ @@ -785,45 +770,4 @@ PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK(PyArrayObject *arr1, PyArrayObject *arr stride2 = PyArray_TRIVIAL_PAIR_ITERATION_STRIDE(size2, arr2); \ } -#define PyArray_TRIVIALLY_ITERABLE_TRIPLE(arr1, arr2, arr3, arr1_read, arr2_read, arr3_read) ( \ - PyArray_TRIVIALLY_ITERABLE(arr1) && \ - ((PyArray_NDIM(arr2) == 0 && \ - (PyArray_NDIM(arr3) == 0 || \ - PyArray_EQUIVALENTLY_ITERABLE_BASE(arr1, arr3) \ - ) \ - ) || \ - (PyArray_EQUIVALENTLY_ITERABLE_BASE(arr1, arr2) && \ - (PyArray_NDIM(arr3) == 0 || \ - PyArray_EQUIVALENTLY_ITERABLE_BASE(arr1, arr3) \ - ) \ - ) || \ - (PyArray_NDIM(arr1) == 0 && \ - PyArray_TRIVIALLY_ITERABLE(arr2) && \ - (PyArray_NDIM(arr3) == 0 || \ - PyArray_EQUIVALENTLY_ITERABLE_BASE(arr2, arr3) \ - ) \ - ) \ - ) && \ - PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK(arr1, arr2, arr1_read, arr2_read) && \ - PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK(arr1, arr3, arr1_read, arr3_read) && \ - PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK(arr2, arr3, arr2_read, arr3_read) \ - ) - -#define PyArray_PREPARE_TRIVIAL_TRIPLE_ITERATION(arr1, arr2, arr3, \ - count, \ - data1, data2, data3, \ - stride1, stride2, stride3) { \ - npy_intp size1 = PyArray_SIZE(arr1); \ - npy_intp size2 = PyArray_SIZE(arr2); \ - npy_intp size3 = PyArray_SIZE(arr3); \ - count = ((size1 > size2) || size1 == 0) ? size1 : size2; \ - count = ((size3 > count) || size3 == 0) ? size3 : count; \ - data1 = PyArray_BYTES(arr1); \ - data2 = PyArray_BYTES(arr2); \ - data3 = PyArray_BYTES(arr3); \ - stride1 = PyArray_TRIVIAL_PAIR_ITERATION_STRIDE(size1, arr1); \ - stride2 = PyArray_TRIVIAL_PAIR_ITERATION_STRIDE(size2, arr2); \ - stride3 = PyArray_TRIVIAL_PAIR_ITERATION_STRIDE(size3, arr3); \ - } - #endif diff --git a/numpy/core/src/common/npy_argparse.c b/numpy/core/src/common/npy_argparse.c new file mode 100644 index 000000000000..8460a38e6461 --- /dev/null +++ b/numpy/core/src/common/npy_argparse.c @@ -0,0 +1,421 @@ +#include "Python.h" + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include "numpy/ndarraytypes.h" +#include "npy_argparse.h" +#include "npy_pycompat.h" +#include "npy_import.h" + +#include "arrayfunction_override.h" + + +/** + * Small wrapper converting to array just like CPython does. + * + * We could use our own PyArray_PyIntAsInt function, but it handles floats + * differently. + * A disadvantage of this function compared to ``PyArg_*("i")`` code is that + * it will not say which parameter is wrong. + * + * @param obj The python object to convert + * @param value The output value + * + * @returns 0 on failure and 1 on success (`NPY_FAIL`, `NPY_SUCCEED`) + */ +NPY_NO_EXPORT int +PyArray_PythonPyIntFromInt(PyObject *obj, int *value) +{ + /* Pythons behaviour is to check only for float explicitly... */ + if (NPY_UNLIKELY(PyFloat_Check(obj))) { + PyErr_SetString(PyExc_TypeError, + "integer argument expected, got float"); + return NPY_FAIL; + } + + long result = PyLong_AsLong(obj); + if (NPY_UNLIKELY((result == -1) && PyErr_Occurred())) { + return NPY_FAIL; + } + if (NPY_UNLIKELY((result > INT_MAX) || (result < INT_MIN))) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C int"); + return NPY_FAIL; + } + else { + *value = (int)result; + return NPY_SUCCEED; + } +} + + +typedef int convert(PyObject *, void *); + +/** + * Internal function to initialize keyword argument parsing. + * + * This does a few simple jobs: + * + * * Check the input for consistency to find coding errors, for example + * a parameter not marked with | after one marked with | (optional). + * 2. Find the number of positional-only arguments, the number of + * total, required, and keyword arguments. + * 3. Intern all keyword arguments strings to allow fast, identity based + * parsing and avoid string creation overhead on each call. + * + * @param funcname Name of the function, mainly used for errors. + * @param cache A cache object stored statically in the parsing function + * @param va_orig Argument list to npy_parse_arguments + * @return 0 on success, -1 on failure + */ +static int +initialize_keywords(const char *funcname, + _NpyArgParserCache *cache, va_list va_orig) { + va_list va; + int nargs = 0; + int nkwargs = 0; + int npositional_only = 0; + int nrequired = 0; + int npositional = 0; + char state = '\0'; + + va_copy(va, va_orig); + while (1) { + /* Count length first: */ + char *name = va_arg(va, char *); + convert *converter = va_arg(va, convert *); + void *data = va_arg(va, void *); + + /* Check if this is the sentinel, only converter may be NULL */ + if ((name == NULL) && (converter == NULL) && (data == NULL)) { + break; + } + + if (name == NULL) { + PyErr_Format(PyExc_SystemError, + "NumPy internal error: name is NULL in %s() at " + "argument %d.", funcname, nargs); + va_end(va); + return -1; + } + if (data == NULL) { + PyErr_Format(PyExc_SystemError, + "NumPy internal error: data is NULL in %s() at " + "argument %d.", funcname, nargs); + va_end(va); + return -1; + } + + nargs += 1; + if (*name == '|') { + if (state == '$') { + PyErr_Format(PyExc_SystemError, + "NumPy internal error: positional argument `|` " + "after keyword only `$` one to %s() at argument %d.", + funcname, nargs); + va_end(va); + return -1; + } + state = '|'; + name++; /* advance to actual name. */ + npositional += 1; + } + else if (*name == '$') { + state = '$'; + name++; /* advance to actual name. */ + } + else { + if (state != '\0') { + PyErr_Format(PyExc_SystemError, + "NumPy internal error: non-required argument after " + "required | or $ one to %s() at argument %d.", + funcname, nargs); + va_end(va); + return -1; + } + + nrequired += 1; + npositional += 1; + } + + if (*name == '\0') { + /* Empty string signals positional only */ + if (state != '\0') { + PyErr_Format(PyExc_SystemError, + "NumPy internal error: non-kwarg marked with | or $ " + "to %s() at argument %d.", funcname, nargs); + va_end(va); + return -1; + } + npositional_only += 1; + } + else { + nkwargs += 1; + } + } + va_end(va); + + if (npositional == -1) { + npositional = nargs; + } + + if (nargs > _NPY_MAX_KWARGS) { + PyErr_Format(PyExc_SystemError, + "NumPy internal error: function %s() has %d arguments, but " + "the maximum is currently limited to %d for easier parsing; " + "it can be increased by modifying `_NPY_MAX_KWARGS`.", + funcname, nargs, _NPY_MAX_KWARGS); + return -1; + } + + /* + * Do any necessary string allocation and interning, + * creating a caching object. + */ + cache->nargs = nargs; + cache->npositional_only = npositional_only; + cache->npositional = npositional; + cache->nrequired = nrequired; + + /* NULL kw_strings for easier cleanup (and NULL termination) */ + memset(cache->kw_strings, 0, sizeof(PyObject *) * (nkwargs + 1)); + + va_copy(va, va_orig); + for (int i = 0; i < nargs; i++) { + /* Advance through non-kwargs, which do not require setup. */ + char *name = va_arg(va, char *); + va_arg(va, convert *); + va_arg(va, void *); + + if (*name == '|' || *name == '$') { + name++; /* ignore | and $ */ + } + if (i >= npositional_only) { + int i_kwarg = i - npositional_only; + cache->kw_strings[i_kwarg] = PyUString_InternFromString(name); + if (cache->kw_strings[i_kwarg] == NULL) { + va_end(va); + goto error; + } + } + } + + va_end(va); + return 0; + +error: + for (int i = 0; i < nkwargs; i++) { + Py_XDECREF(cache->kw_strings[i]); + } + cache->npositional = -1; /* not initialized */ + return -1; +} + + +static int +raise_incorrect_number_of_positional_args(const char *funcname, + const _NpyArgParserCache *cache, Py_ssize_t len_args) +{ + if (cache->npositional == cache->nrequired) { + PyErr_Format(PyExc_TypeError, + "%s() takes %d positional arguments but %zd were given", + funcname, cache->npositional, len_args); + } + else { + PyErr_Format(PyExc_TypeError, + "%s() takes from %d to %d positional arguments but " + "%zd were given", + funcname, cache->nrequired, cache->npositional, len_args); + } + return -1; +} + +static void +raise_missing_argument(const char *funcname, + const _NpyArgParserCache *cache, int i) +{ + if (i < cache->npositional_only) { + PyErr_Format(PyExc_TypeError, + "%s() missing required positional argument %d", + funcname, i); + } + else { + PyObject *kw = cache->kw_strings[i - cache->npositional_only]; + PyErr_Format(PyExc_TypeError, + "%s() missing required argument '%S' (pos %d)", + funcname, kw, i); + } +} + + +/** + * Generic helper for argument parsing + * + * See macro version for an example pattern of how to use this function. + * + * @param funcname + * @param cache + * @param args Python passed args (METH_FASTCALL) + * @param len_args + * @param kwnames + * @param ... List of arguments (see macro version). + * + * @return Returns 0 on success and -1 on failure. + */ +NPY_NO_EXPORT int +_npy_parse_arguments(const char *funcname, + /* cache_ptr is a NULL initialized persistent storage for data */ + _NpyArgParserCache *cache, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, + /* ... is NULL, NULL, NULL terminated: name, converter, value */ + ...) +{ + if (NPY_UNLIKELY(cache->npositional == -1)) { + va_list va; + va_start(va, kwnames); + + int res = initialize_keywords(funcname, cache, va); + va_end(va); + if (res < 0) { + return -1; + } + } + + if (NPY_UNLIKELY(len_args > cache->npositional)) { + return raise_incorrect_number_of_positional_args( + funcname, cache, len_args); + } + + /* NOTE: Could remove the limit but too many kwargs are slow anyway. */ + PyObject *all_arguments[NPY_MAXARGS]; + + for (Py_ssize_t i = 0; i < len_args; i++) { + all_arguments[i] = args[i]; + } + + /* Without kwargs, do not iterate all converters. */ + int max_nargs = (int)len_args; + Py_ssize_t len_kwargs = 0; + + /* If there are any kwargs, first handle them */ + if (NPY_LIKELY(kwnames != NULL)) { + len_kwargs = PyTuple_GET_SIZE(kwnames); + max_nargs = cache->nargs; + + for (int i = len_args; i < cache->nargs; i++) { + all_arguments[i] = NULL; + } + + for (Py_ssize_t i = 0; i < len_kwargs; i++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = args[i + len_args]; + PyObject *const *name; + + /* Super-fast path, check identity: */ + for (name = cache->kw_strings; *name != NULL; name++) { + if (*name == key) { + break; + } + } + if (NPY_UNLIKELY(*name == NULL)) { + /* Slow fallback, if identity checks failed for some reason */ + for (name = cache->kw_strings; *name != NULL; name++) { + int eq = PyObject_RichCompareBool(*name, key, Py_EQ); + if (eq == -1) { + return -1; + } + else if (eq) { + break; + } + } + if (NPY_UNLIKELY(*name == NULL)) { + /* Invalid keyword argument. */ + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%S'", + funcname, key); + return -1; + } + } + + Py_ssize_t param_pos = ( + (name - cache->kw_strings) + cache->npositional_only); + + /* There could be an identical positional argument */ + if (NPY_UNLIKELY(all_arguments[param_pos] != NULL)) { + PyErr_Format(PyExc_TypeError, + "argument for %s() given by name ('%S') and position " + "(position %zd)", funcname, key, param_pos); + return -1; + } + + all_arguments[param_pos] = value; + } + } + + /* + * There cannot be too many args, too many kwargs would find an + * incorrect one above. + */ + assert(len_args + len_kwargs <= cache->nargs); + + /* At this time `all_arguments` holds either NULLs or the objects */ + va_list va; + va_start(va, kwnames); + + for (int i = 0; i < max_nargs; i++) { + va_arg(va, char *); + convert *converter = va_arg(va, convert *); + void *data = va_arg(va, void *); + + if (all_arguments[i] == NULL) { + continue; + } + + int res; + if (converter == NULL) { + *((PyObject **) data) = all_arguments[i]; + continue; + } + res = converter(all_arguments[i], data); + + if (NPY_UNLIKELY(res == NPY_SUCCEED)) { + continue; + } + else if (NPY_UNLIKELY(res == NPY_FAIL)) { + /* It is usually the users responsibility to clean up. */ + goto converting_failed; + } + else if (NPY_UNLIKELY(res == Py_CLEANUP_SUPPORTED)) { + /* TODO: Implementing cleanup if/when needed should not be hard */ + PyErr_Format(PyExc_SystemError, + "converter cleanup of parameter %d to %s() not supported.", + i, funcname); + goto converting_failed; + } + assert(0); + } + + /* Required arguments are typically not passed as keyword arguments */ + if (NPY_UNLIKELY(len_args < cache->nrequired)) { + /* (PyArg_* also does this after the actual parsing is finished) */ + if (NPY_UNLIKELY(max_nargs < cache->nrequired)) { + raise_missing_argument(funcname, cache, max_nargs); + goto converting_failed; + } + for (int i = 0; i < cache->nrequired; i++) { + if (NPY_UNLIKELY(all_arguments[i] == NULL)) { + raise_missing_argument(funcname, cache, i); + goto converting_failed; + } + } + } + + va_end(va); + return 0; + +converting_failed: + va_end(va); + return -1; + +} diff --git a/numpy/core/src/common/npy_argparse.h b/numpy/core/src/common/npy_argparse.h new file mode 100644 index 000000000000..5da535c9171f --- /dev/null +++ b/numpy/core/src/common/npy_argparse.h @@ -0,0 +1,96 @@ +#ifndef _NPY_ARGPARSE_H +#define _NPY_ARGPARSE_H + +#include "Python.h" +#include "numpy/ndarraytypes.h" + +/* + * This file defines macros to help with keyword argument parsing. + * This solves two issues as of now: + * 1. Pythons C-API PyArg_* keyword argument parsers are slow, due to + * not caching the strings they use. + * 2. It allows the use of METH_ARGPARSE (and `tp_vectorcall`) + * when available in Python, which removes a large chunk of overhead. + * + * Internally CPython achieves similar things by using a code generator + * argument clinic. NumPy may well decide to use argument clinic or a different + * solution in the future. + */ + +NPY_NO_EXPORT int +PyArray_PythonPyIntFromInt(PyObject *obj, int *value); + + +#define _NPY_MAX_KWARGS 15 + +typedef struct { + int npositional; + int nargs; + int npositional_only; + int nrequired; + /* Null terminated list of keyword argument name strings */ + PyObject *kw_strings[_NPY_MAX_KWARGS+1]; +} _NpyArgParserCache; + + +/* + * The sole purpose of this macro is to hide the argument parsing cache. + * Since this cache must be static, this also removes a source of error. + */ +#define NPY_PREPARE_ARGPARSER static _NpyArgParserCache __argparse_cache = {-1} + +/** + * Macro to help with argument parsing. + * + * The pattern for using this macro is by defining the method as: + * + * @code + * static PyObject * + * my_method(PyObject *self, + * PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) + * { + * NPY_PREPARE_ARGPARSER; + * + * PyObject *argument1, *argument3; + * int argument2 = -1; + * if (npy_parse_arguments("method", args, len_args, kwnames), + * "argument1", NULL, &argument1, + * "|argument2", &PyArray_PythonPyIntFromInt, &argument2, + * "$argument3", NULL, &argument3, + * NULL, NULL, NULL) < 0) { + * return NULL; + * } + * } + * @endcode + * + * The `NPY_PREPARE_ARGPARSER` macro sets up a static cache variable necessary + * to hold data for speeding up the parsing. `npy_parse_arguments` must be + * used in cunjunction with the macro defined in the same scope. + * (No two `npy_parse_arguments` may share a single `NPY_PREPARE_ARGPARSER`.) + * + * @param funcname + * @param args Python passed args (METH_FASTCALL) + * @param len_args Number of arguments (not flagged) + * @param kwnames Tuple as passed by METH_FASTCALL or NULL. + * @param ... List of arguments must be param1_name, param1_converter, + * *param1_outvalue, param2_name, ..., NULL, NULL, NULL. + * Where name is ``char *``, ``converter`` a python converter + * function or NULL and ``outvalue`` is the ``void *`` passed to + * the converter (holding the converted data or a borrowed + * reference if converter is NULL). + * + * @return Returns 0 on success and -1 on failure. + */ +NPY_NO_EXPORT int +_npy_parse_arguments(const char *funcname, + /* cache_ptr is a NULL initialized persistent storage for data */ + _NpyArgParserCache *cache_ptr, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, + /* va_list is NULL, NULL, NULL terminated: name, converter, value */ + ...) NPY_GCC_NONNULL(1); + +#define npy_parse_arguments(funcname, args, len_args, kwnames, ...) \ + _npy_parse_arguments(funcname, &__argparse_cache, \ + args, len_args, kwnames, __VA_ARGS__) + +#endif /* _NPY_ARGPARSE_H */ diff --git a/numpy/core/src/common/npy_cpu_dispatch.h b/numpy/core/src/common/npy_cpu_dispatch.h index a0f82fa3da05..c8411104a867 100644 --- a/numpy/core/src/common/npy_cpu_dispatch.h +++ b/numpy/core/src/common/npy_cpu_dispatch.h @@ -7,10 +7,10 @@ #include "npy_cpu_features.h" // NPY_CPU_HAVE #include "numpy/utils.h" // NPY_EXPAND, NPY_CAT /** - * Bringing the main configration header '_cpu_dispatch.h'. + * Including the main configuration header 'npy_cpu_dispatch_config.h'. * * This header is generated by the distutils module 'ccompiler_opt', - * and contains all the #definitions and headers of instruction-sets, + * and contains all the #definitions and headers for platform-specific instruction-sets * that had been configured through command arguments '--cpu-baseline' and '--cpu-dispatch'. * * It also contains extra C #definitions and macros that are used for implementing @@ -33,7 +33,7 @@ #define NPY__DISPATCH_DEFBOOL typedef bool npy__dispatch_bkbool; #endif - #include "_cpu_dispatch.h" + #include "npy_cpu_dispatch_config.h" #ifdef NPY_HAVE_VSX #undef bool #undef vector diff --git a/numpy/core/src/common/npy_cpu_features.c.src b/numpy/core/src/common/npy_cpu_features.c.src index 69bbc83a2e38..4f3a95c717a1 100644 --- a/numpy/core/src/common/npy_cpu_features.c.src +++ b/numpy/core/src/common/npy_cpu_features.c.src @@ -500,7 +500,7 @@ npy__cpu_init_features_arm8(void) npy__cpu_have[NPY_CPU_FEATURE_ASIMD] = 1; } -#ifdef __linux__ +#if defined(__linux__) || defined(__FreeBSD__) /* * we aren't sure of what kind kernel or clib we deal with * so we play it safe @@ -509,10 +509,23 @@ npy__cpu_init_features_arm8(void) #include "npy_cpuinfo_parser.h" __attribute__((weak)) unsigned long getauxval(unsigned long); // linker should handle it +#ifdef __FreeBSD__ +__attribute__((weak)) int elf_aux_info(int, void *, int); // linker should handle it + +static unsigned long getauxval(unsigned long k) +{ + unsigned long val = 0ul; + if (elf_aux_info == 0 || elf_aux_info((int)k, (void *)&val, (int)sizeof(val)) != 0) { + return 0ul; + } + return val; +} +#endif static int npy__cpu_init_features_linux(void) { unsigned long hwcap = 0, hwcap2 = 0; + #ifdef __linux__ if (getauxval != 0) { hwcap = getauxval(NPY__HWCAP); #ifdef __arm__ @@ -539,7 +552,14 @@ npy__cpu_init_features_linux(void) close(fd); } } + #else + hwcap = getauxval(NPY__HWCAP); + #ifdef __arm__ + hwcap2 = getauxval(NPY__HWCAP2); + #endif + #endif if (hwcap == 0 && hwcap2 == 0) { + #ifdef __linux__ /* * try parsing with /proc/cpuinfo, if sandboxed * failback to compiler definitions @@ -547,6 +567,9 @@ npy__cpu_init_features_linux(void) if(!get_feature_from_proc_cpuinfo(&hwcap, &hwcap2)) { return 0; } + #else + return 0; + #endif } #ifdef __arm__ // Detect Arm8 (aarch32 state) diff --git a/numpy/core/src/common/npy_cpuinfo_parser.h b/numpy/core/src/common/npy_cpuinfo_parser.h index 4c00c847bfbe..f4540f6ab170 100644 --- a/numpy/core/src/common/npy_cpuinfo_parser.h +++ b/numpy/core/src/common/npy_cpuinfo_parser.h @@ -1,262 +1,262 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ -#ifndef __NPY_CPUINFO_PARSER_H__ -#define __NPY_CPUINFO_PARSER_H__ -#include -#include -#include -#include -#include - -#define NPY__HWCAP 16 -#define NPY__HWCAP2 26 - -// arch/arm/include/uapi/asm/hwcap.h -#define NPY__HWCAP_HALF (1 << 1) -#define NPY__HWCAP_NEON (1 << 12) -#define NPY__HWCAP_VFPv3 (1 << 13) -#define NPY__HWCAP_VFPv4 (1 << 16) -#define NPY__HWCAP2_AES (1 << 0) -#define NPY__HWCAP2_PMULL (1 << 1) -#define NPY__HWCAP2_SHA1 (1 << 2) -#define NPY__HWCAP2_SHA2 (1 << 3) -#define NPY__HWCAP2_CRC32 (1 << 4) -// arch/arm64/include/uapi/asm/hwcap.h -#define NPY__HWCAP_FP (1 << 0) -#define NPY__HWCAP_ASIMD (1 << 1) -#define NPY__HWCAP_FPHP (1 << 9) -#define NPY__HWCAP_ASIMDHP (1 << 10) -#define NPY__HWCAP_ASIMDDP (1 << 20) -#define NPY__HWCAP_ASIMDFHM (1 << 23) -/* - * Get the size of a file by reading it until the end. This is needed - * because files under /proc do not always return a valid size when - * using fseek(0, SEEK_END) + ftell(). Nor can they be mmap()-ed. - */ -static int -get_file_size(const char* pathname) -{ - int fd, result = 0; - char buffer[256]; - - fd = open(pathname, O_RDONLY); - if (fd < 0) { - return -1; - } - - for (;;) { - int ret = read(fd, buffer, sizeof buffer); - if (ret < 0) { - if (errno == EINTR) { - continue; - } - break; - } - if (ret == 0) { - break; - } - result += ret; - } - close(fd); - return result; -} - -/* - * Read the content of /proc/cpuinfo into a user-provided buffer. - * Return the length of the data, or -1 on error. Does *not* - * zero-terminate the content. Will not read more - * than 'buffsize' bytes. - */ -static int -read_file(const char* pathname, char* buffer, size_t buffsize) -{ - int fd, count; - - fd = open(pathname, O_RDONLY); - if (fd < 0) { - return -1; - } - count = 0; - while (count < (int)buffsize) { - int ret = read(fd, buffer + count, buffsize - count); - if (ret < 0) { - if (errno == EINTR) { - continue; - } - if (count == 0) { - count = -1; - } - break; - } - if (ret == 0) { - break; - } - count += ret; - } - close(fd); - return count; -} - -/* - * Extract the content of a the first occurence of a given field in - * the content of /proc/cpuinfo and return it as a heap-allocated - * string that must be freed by the caller. - * - * Return NULL if not found - */ -static char* -extract_cpuinfo_field(const char* buffer, int buflen, const char* field) -{ - int fieldlen = strlen(field); - const char* bufend = buffer + buflen; - char* result = NULL; - int len; - const char *p, *q; - - /* Look for first field occurence, and ensures it starts the line. */ - p = buffer; - for (;;) { - p = memmem(p, bufend-p, field, fieldlen); - if (p == NULL) { - goto EXIT; - } - - if (p == buffer || p[-1] == '\n') { - break; - } - - p += fieldlen; - } - - /* Skip to the first column followed by a space */ - p += fieldlen; - p = memchr(p, ':', bufend-p); - if (p == NULL || p[1] != ' ') { - goto EXIT; - } - - /* Find the end of the line */ - p += 2; - q = memchr(p, '\n', bufend-p); - if (q == NULL) { - q = bufend; - } - - /* Copy the line into a heap-allocated buffer */ - len = q - p; - result = malloc(len + 1); - if (result == NULL) { - goto EXIT; - } - - memcpy(result, p, len); - result[len] = '\0'; - -EXIT: - return result; -} - -/* - * Checks that a space-separated list of items contains one given 'item'. - * Returns 1 if found, 0 otherwise. - */ -static int -has_list_item(const char* list, const char* item) -{ - const char* p = list; - int itemlen = strlen(item); - - if (list == NULL) { - return 0; - } - - while (*p) { - const char* q; - - /* skip spaces */ - while (*p == ' ' || *p == '\t') { - p++; - } - - /* find end of current list item */ - q = p; - while (*q && *q != ' ' && *q != '\t') { - q++; - } - - if (itemlen == q-p && !memcmp(p, item, itemlen)) { - return 1; - } - - /* skip to next item */ - p = q; - } - return 0; -} - -static void setHwcap(char* cpuFeatures, unsigned long* hwcap) { - *hwcap |= has_list_item(cpuFeatures, "neon") ? NPY__HWCAP_NEON : 0; - *hwcap |= has_list_item(cpuFeatures, "half") ? NPY__HWCAP_HALF : 0; - *hwcap |= has_list_item(cpuFeatures, "vfpv3") ? NPY__HWCAP_VFPv3 : 0; - *hwcap |= has_list_item(cpuFeatures, "vfpv4") ? NPY__HWCAP_VFPv4 : 0; - - *hwcap |= has_list_item(cpuFeatures, "asimd") ? NPY__HWCAP_ASIMD : 0; - *hwcap |= has_list_item(cpuFeatures, "fp") ? NPY__HWCAP_FP : 0; - *hwcap |= has_list_item(cpuFeatures, "fphp") ? NPY__HWCAP_FPHP : 0; - *hwcap |= has_list_item(cpuFeatures, "asimdhp") ? NPY__HWCAP_ASIMDHP : 0; - *hwcap |= has_list_item(cpuFeatures, "asimddp") ? NPY__HWCAP_ASIMDDP : 0; - *hwcap |= has_list_item(cpuFeatures, "asimdfhm") ? NPY__HWCAP_ASIMDFHM : 0; -} - -static int -get_feature_from_proc_cpuinfo(unsigned long *hwcap, unsigned long *hwcap2) { - char* cpuinfo = NULL; - int cpuinfo_len; - cpuinfo_len = get_file_size("/proc/cpuinfo"); - if (cpuinfo_len < 0) { - return 0; - } - cpuinfo = malloc(cpuinfo_len); - if (cpuinfo == NULL) { - return 0; - } - cpuinfo_len = read_file("/proc/cpuinfo", cpuinfo, cpuinfo_len); - char* cpuFeatures = extract_cpuinfo_field(cpuinfo, cpuinfo_len, "Features"); - if(cpuFeatures == NULL) { - return 0; - } - setHwcap(cpuFeatures, hwcap); - *hwcap2 |= *hwcap; - *hwcap2 |= has_list_item(cpuFeatures, "aes") ? NPY__HWCAP2_AES : 0; - *hwcap2 |= has_list_item(cpuFeatures, "pmull") ? NPY__HWCAP2_PMULL : 0; - *hwcap2 |= has_list_item(cpuFeatures, "sha1") ? NPY__HWCAP2_SHA1 : 0; - *hwcap2 |= has_list_item(cpuFeatures, "sha2") ? NPY__HWCAP2_SHA2 : 0; - *hwcap2 |= has_list_item(cpuFeatures, "crc32") ? NPY__HWCAP2_CRC32 : 0; - return 1; -} -#endif +/* + * Copyright (C) 2010 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#ifndef __NPY_CPUINFO_PARSER_H__ +#define __NPY_CPUINFO_PARSER_H__ +#include +#include +#include +#include +#include + +#define NPY__HWCAP 16 +#define NPY__HWCAP2 26 + +// arch/arm/include/uapi/asm/hwcap.h +#define NPY__HWCAP_HALF (1 << 1) +#define NPY__HWCAP_NEON (1 << 12) +#define NPY__HWCAP_VFPv3 (1 << 13) +#define NPY__HWCAP_VFPv4 (1 << 16) +#define NPY__HWCAP2_AES (1 << 0) +#define NPY__HWCAP2_PMULL (1 << 1) +#define NPY__HWCAP2_SHA1 (1 << 2) +#define NPY__HWCAP2_SHA2 (1 << 3) +#define NPY__HWCAP2_CRC32 (1 << 4) +// arch/arm64/include/uapi/asm/hwcap.h +#define NPY__HWCAP_FP (1 << 0) +#define NPY__HWCAP_ASIMD (1 << 1) +#define NPY__HWCAP_FPHP (1 << 9) +#define NPY__HWCAP_ASIMDHP (1 << 10) +#define NPY__HWCAP_ASIMDDP (1 << 20) +#define NPY__HWCAP_ASIMDFHM (1 << 23) +/* + * Get the size of a file by reading it until the end. This is needed + * because files under /proc do not always return a valid size when + * using fseek(0, SEEK_END) + ftell(). Nor can they be mmap()-ed. + */ +static int +get_file_size(const char* pathname) +{ + int fd, result = 0; + char buffer[256]; + + fd = open(pathname, O_RDONLY); + if (fd < 0) { + return -1; + } + + for (;;) { + int ret = read(fd, buffer, sizeof buffer); + if (ret < 0) { + if (errno == EINTR) { + continue; + } + break; + } + if (ret == 0) { + break; + } + result += ret; + } + close(fd); + return result; +} + +/* + * Read the content of /proc/cpuinfo into a user-provided buffer. + * Return the length of the data, or -1 on error. Does *not* + * zero-terminate the content. Will not read more + * than 'buffsize' bytes. + */ +static int +read_file(const char* pathname, char* buffer, size_t buffsize) +{ + int fd, count; + + fd = open(pathname, O_RDONLY); + if (fd < 0) { + return -1; + } + count = 0; + while (count < (int)buffsize) { + int ret = read(fd, buffer + count, buffsize - count); + if (ret < 0) { + if (errno == EINTR) { + continue; + } + if (count == 0) { + count = -1; + } + break; + } + if (ret == 0) { + break; + } + count += ret; + } + close(fd); + return count; +} + +/* + * Extract the content of a the first occurence of a given field in + * the content of /proc/cpuinfo and return it as a heap-allocated + * string that must be freed by the caller. + * + * Return NULL if not found + */ +static char* +extract_cpuinfo_field(const char* buffer, int buflen, const char* field) +{ + int fieldlen = strlen(field); + const char* bufend = buffer + buflen; + char* result = NULL; + int len; + const char *p, *q; + + /* Look for first field occurence, and ensures it starts the line. */ + p = buffer; + for (;;) { + p = memmem(p, bufend-p, field, fieldlen); + if (p == NULL) { + goto EXIT; + } + + if (p == buffer || p[-1] == '\n') { + break; + } + + p += fieldlen; + } + + /* Skip to the first column followed by a space */ + p += fieldlen; + p = memchr(p, ':', bufend-p); + if (p == NULL || p[1] != ' ') { + goto EXIT; + } + + /* Find the end of the line */ + p += 2; + q = memchr(p, '\n', bufend-p); + if (q == NULL) { + q = bufend; + } + + /* Copy the line into a heap-allocated buffer */ + len = q - p; + result = malloc(len + 1); + if (result == NULL) { + goto EXIT; + } + + memcpy(result, p, len); + result[len] = '\0'; + +EXIT: + return result; +} + +/* + * Checks that a space-separated list of items contains one given 'item'. + * Returns 1 if found, 0 otherwise. + */ +static int +has_list_item(const char* list, const char* item) +{ + const char* p = list; + int itemlen = strlen(item); + + if (list == NULL) { + return 0; + } + + while (*p) { + const char* q; + + /* skip spaces */ + while (*p == ' ' || *p == '\t') { + p++; + } + + /* find end of current list item */ + q = p; + while (*q && *q != ' ' && *q != '\t') { + q++; + } + + if (itemlen == q-p && !memcmp(p, item, itemlen)) { + return 1; + } + + /* skip to next item */ + p = q; + } + return 0; +} + +static void setHwcap(char* cpuFeatures, unsigned long* hwcap) { + *hwcap |= has_list_item(cpuFeatures, "neon") ? NPY__HWCAP_NEON : 0; + *hwcap |= has_list_item(cpuFeatures, "half") ? NPY__HWCAP_HALF : 0; + *hwcap |= has_list_item(cpuFeatures, "vfpv3") ? NPY__HWCAP_VFPv3 : 0; + *hwcap |= has_list_item(cpuFeatures, "vfpv4") ? NPY__HWCAP_VFPv4 : 0; + + *hwcap |= has_list_item(cpuFeatures, "asimd") ? NPY__HWCAP_ASIMD : 0; + *hwcap |= has_list_item(cpuFeatures, "fp") ? NPY__HWCAP_FP : 0; + *hwcap |= has_list_item(cpuFeatures, "fphp") ? NPY__HWCAP_FPHP : 0; + *hwcap |= has_list_item(cpuFeatures, "asimdhp") ? NPY__HWCAP_ASIMDHP : 0; + *hwcap |= has_list_item(cpuFeatures, "asimddp") ? NPY__HWCAP_ASIMDDP : 0; + *hwcap |= has_list_item(cpuFeatures, "asimdfhm") ? NPY__HWCAP_ASIMDFHM : 0; +} + +static int +get_feature_from_proc_cpuinfo(unsigned long *hwcap, unsigned long *hwcap2) { + char* cpuinfo = NULL; + int cpuinfo_len; + cpuinfo_len = get_file_size("/proc/cpuinfo"); + if (cpuinfo_len < 0) { + return 0; + } + cpuinfo = malloc(cpuinfo_len); + if (cpuinfo == NULL) { + return 0; + } + cpuinfo_len = read_file("/proc/cpuinfo", cpuinfo, cpuinfo_len); + char* cpuFeatures = extract_cpuinfo_field(cpuinfo, cpuinfo_len, "Features"); + if(cpuFeatures == NULL) { + return 0; + } + setHwcap(cpuFeatures, hwcap); + *hwcap2 |= *hwcap; + *hwcap2 |= has_list_item(cpuFeatures, "aes") ? NPY__HWCAP2_AES : 0; + *hwcap2 |= has_list_item(cpuFeatures, "pmull") ? NPY__HWCAP2_PMULL : 0; + *hwcap2 |= has_list_item(cpuFeatures, "sha1") ? NPY__HWCAP2_SHA1 : 0; + *hwcap2 |= has_list_item(cpuFeatures, "sha2") ? NPY__HWCAP2_SHA2 : 0; + *hwcap2 |= has_list_item(cpuFeatures, "crc32") ? NPY__HWCAP2_CRC32 : 0; + return 1; +} +#endif diff --git a/numpy/core/src/common/npy_pycompat.h b/numpy/core/src/common/npy_pycompat.h index aa0b5c1224d3..9e94a971090a 100644 --- a/numpy/core/src/common/npy_pycompat.h +++ b/numpy/core/src/common/npy_pycompat.h @@ -3,4 +3,20 @@ #include "numpy/npy_3kcompat.h" + +/* + * In Python 3.10a7 (or b1), python started using the identity for the hash + * when a value is NaN. See https://bugs.python.org/issue43475 + */ +#if PY_VERSION_HEX > 0x030a00a6 +#define Npy_HashDouble _Py_HashDouble +#else +static NPY_INLINE Py_hash_t +Npy_HashDouble(PyObject *NPY_UNUSED(identity), double val) +{ + return _Py_HashDouble(val); +} +#endif + + #endif /* _NPY_COMPAT_H_ */ diff --git a/numpy/core/src/common/simd/avx2/arithmetic.h b/numpy/core/src/common/simd/avx2/arithmetic.h index 3a6dc953534e..e1b170863a34 100644 --- a/numpy/core/src/common/simd/avx2/arithmetic.h +++ b/numpy/core/src/common/simd/avx2/arithmetic.h @@ -5,6 +5,7 @@ #ifndef _NPY_SIMD_AVX2_ARITHMETIC_H #define _NPY_SIMD_AVX2_ARITHMETIC_H +#include "../sse/utils.h" /*************************** * Addition ***************************/ @@ -25,7 +26,7 @@ #define npyv_adds_s8 _mm256_adds_epi8 #define npyv_adds_u16 _mm256_adds_epu16 #define npyv_adds_s16 _mm256_adds_epi16 -// TODO: rest, after implment Packs intrins +// TODO: rest, after implement Packs intrins /*************************** * Subtraction @@ -47,7 +48,7 @@ #define npyv_subs_s8 _mm256_subs_epi8 #define npyv_subs_u16 _mm256_subs_epu16 #define npyv_subs_s16 _mm256_subs_epi16 -// TODO: rest, after implment Packs intrins +// TODO: rest, after implement Packs intrins /*************************** * Multiplication @@ -63,8 +64,166 @@ #define npyv_mul_f64 _mm256_mul_pd // saturated -// TODO: after implment Packs intrins +// TODO: after implement Packs intrins +/*************************** + * Integer Division + ***************************/ +// See simd/intdiv.h for more clarification +// divide each unsigned 8-bit element by a precomputed divisor +NPY_FINLINE npyv_u8 npyv_divc_u8(npyv_u8 a, const npyv_u8x3 divisor) +{ + const __m256i bmask = _mm256_set1_epi32(0x00FF00FF); + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + const __m128i shf2 = _mm256_castsi256_si128(divisor.val[2]); + const __m256i shf1b = _mm256_set1_epi8(0xFFU >> _mm_cvtsi128_si32(shf1)); + const __m256i shf2b = _mm256_set1_epi8(0xFFU >> _mm_cvtsi128_si32(shf2)); + // high part of unsigned multiplication + __m256i mulhi_even = _mm256_mullo_epi16(_mm256_and_si256(a, bmask), divisor.val[0]); + mulhi_even = _mm256_srli_epi16(mulhi_even, 8); + __m256i mulhi_odd = _mm256_mullo_epi16(_mm256_srli_epi16(a, 8), divisor.val[0]); + __m256i mulhi = _mm256_blendv_epi8(mulhi_odd, mulhi_even, bmask); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m256i q = _mm256_sub_epi8(a, mulhi); + q = _mm256_and_si256(_mm256_srl_epi16(q, shf1), shf1b); + q = _mm256_add_epi8(mulhi, q); + q = _mm256_and_si256(_mm256_srl_epi16(q, shf2), shf2b); + return q; +} +// divide each signed 8-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor); +NPY_FINLINE npyv_s8 npyv_divc_s8(npyv_s8 a, const npyv_s8x3 divisor) +{ + const __m256i bmask = _mm256_set1_epi32(0x00FF00FF); + // instead of _mm256_cvtepi8_epi16/_mm256_packs_epi16 to wrap around overflow + __m256i divc_even = npyv_divc_s16(_mm256_srai_epi16(_mm256_slli_epi16(a, 8), 8), divisor); + __m256i divc_odd = npyv_divc_s16(_mm256_srai_epi16(a, 8), divisor); + divc_odd = _mm256_slli_epi16(divc_odd, 8); + return _mm256_blendv_epi8(divc_odd, divc_even, bmask); +} +// divide each unsigned 16-bit element by a precomputed divisor +NPY_FINLINE npyv_u16 npyv_divc_u16(npyv_u16 a, const npyv_u16x3 divisor) +{ + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + const __m128i shf2 = _mm256_castsi256_si128(divisor.val[2]); + // high part of unsigned multiplication + __m256i mulhi = _mm256_mulhi_epu16(a, divisor.val[0]); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m256i q = _mm256_sub_epi16(a, mulhi); + q = _mm256_srl_epi16(q, shf1); + q = _mm256_add_epi16(mulhi, q); + q = _mm256_srl_epi16(q, shf2); + return q; +} +// divide each signed 16-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor) +{ + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + // high part of signed multiplication + __m256i mulhi = _mm256_mulhi_epi16(a, divisor.val[0]); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + __m256i q = _mm256_sra_epi16(_mm256_add_epi16(a, mulhi), shf1); + q = _mm256_sub_epi16(q, _mm256_srai_epi16(a, 15)); + q = _mm256_sub_epi16(_mm256_xor_si256(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 32-bit element by a precomputed divisor +NPY_FINLINE npyv_u32 npyv_divc_u32(npyv_u32 a, const npyv_u32x3 divisor) +{ + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + const __m128i shf2 = _mm256_castsi256_si128(divisor.val[2]); + // high part of unsigned multiplication + __m256i mulhi_even = _mm256_srli_epi64(_mm256_mul_epu32(a, divisor.val[0]), 32); + __m256i mulhi_odd = _mm256_mul_epu32(_mm256_srli_epi64(a, 32), divisor.val[0]); + __m256i mulhi = _mm256_blend_epi32(mulhi_even, mulhi_odd, 0xAA); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m256i q = _mm256_sub_epi32(a, mulhi); + q = _mm256_srl_epi32(q, shf1); + q = _mm256_add_epi32(mulhi, q); + q = _mm256_srl_epi32(q, shf2); + return q; +} +// divide each signed 32-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s32 npyv_divc_s32(npyv_s32 a, const npyv_s32x3 divisor) +{ + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + // high part of signed multiplication + __m256i mulhi_even = _mm256_srli_epi64(_mm256_mul_epi32(a, divisor.val[0]), 32); + __m256i mulhi_odd = _mm256_mul_epi32(_mm256_srli_epi64(a, 32), divisor.val[0]); + __m256i mulhi = _mm256_blend_epi32(mulhi_even, mulhi_odd, 0xAA); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + __m256i q = _mm256_sra_epi32(_mm256_add_epi32(a, mulhi), shf1); + q = _mm256_sub_epi32(q, _mm256_srai_epi32(a, 31)); + q = _mm256_sub_epi32(_mm256_xor_si256(q, divisor.val[2]), divisor.val[2]); + return q; +} +// returns the high 64 bits of unsigned 64-bit multiplication +// xref https://stackoverflow.com/a/28827013 +NPY_FINLINE npyv_u64 npyv__mullhi_u64(npyv_u64 a, npyv_u64 b) +{ + __m256i lomask = npyv_setall_s64(0xffffffff); + __m256i a_hi = _mm256_srli_epi64(a, 32); // a0l, a0h, a1l, a1h + __m256i b_hi = _mm256_srli_epi64(b, 32); // b0l, b0h, b1l, b1h + // compute partial products + __m256i w0 = _mm256_mul_epu32(a, b); // a0l*b0l, a1l*b1l + __m256i w1 = _mm256_mul_epu32(a, b_hi); // a0l*b0h, a1l*b1h + __m256i w2 = _mm256_mul_epu32(a_hi, b); // a0h*b0l, a1h*b0l + __m256i w3 = _mm256_mul_epu32(a_hi, b_hi); // a0h*b0h, a1h*b1h + // sum partial products + __m256i w0h = _mm256_srli_epi64(w0, 32); + __m256i s1 = _mm256_add_epi64(w1, w0h); + __m256i s1l = _mm256_and_si256(s1, lomask); + __m256i s1h = _mm256_srli_epi64(s1, 32); + + __m256i s2 = _mm256_add_epi64(w2, s1l); + __m256i s2h = _mm256_srli_epi64(s2, 32); + + __m256i hi = _mm256_add_epi64(w3, s1h); + hi = _mm256_add_epi64(hi, s2h); + return hi; +} +// divide each unsigned 64-bit element by a divisor +NPY_FINLINE npyv_u64 npyv_divc_u64(npyv_u64 a, const npyv_u64x3 divisor) +{ + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + const __m128i shf2 = _mm256_castsi256_si128(divisor.val[2]); + // high part of unsigned multiplication + __m256i mulhi = npyv__mullhi_u64(a, divisor.val[0]); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m256i q = _mm256_sub_epi64(a, mulhi); + q = _mm256_srl_epi64(q, shf1); + q = _mm256_add_epi64(mulhi, q); + q = _mm256_srl_epi64(q, shf2); + return q; +} +// divide each unsigned 64-bit element by a divisor (round towards zero) +NPY_FINLINE npyv_s64 npyv_divc_s64(npyv_s64 a, const npyv_s64x3 divisor) +{ + const __m128i shf1 = _mm256_castsi256_si128(divisor.val[1]); + // high part of unsigned multiplication + __m256i mulhi = npyv__mullhi_u64(a, divisor.val[0]); + // convert unsigned to signed high multiplication + // mulhi - ((a < 0) ? m : 0) - ((m < 0) ? a : 0); + __m256i asign = _mm256_cmpgt_epi64(_mm256_setzero_si256(), a); + __m256i msign = _mm256_cmpgt_epi64(_mm256_setzero_si256(), divisor.val[0]); + __m256i m_asign = _mm256_and_si256(divisor.val[0], asign); + __m256i a_msign = _mm256_and_si256(a, msign); + mulhi = _mm256_sub_epi64(mulhi, m_asign); + mulhi = _mm256_sub_epi64(mulhi, a_msign); + // q = (a + mulhi) >> sh + __m256i q = _mm256_add_epi64(a, mulhi); + // emulate arithmetic right shift + const __m256i sigb = npyv_setall_s64(1LL << 63); + q = _mm256_srl_epi64(_mm256_add_epi64(q, sigb), shf1); + q = _mm256_sub_epi64(q, _mm256_srl_epi64(sigb, shf1)); + // q = q - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + q = _mm256_sub_epi64(q, asign); + q = _mm256_sub_epi64(_mm256_xor_si256(q, divisor.val[2]), divisor.val[2]); + return q; +} /*************************** * Division ***************************/ @@ -117,8 +276,27 @@ } #endif // !NPY_HAVE_FMA3 -// Horizontal add: Calculates the sum of all vector elements. -NPY_FINLINE float npyv_sum_f32(__m256 a) +/*************************** + * Summation + ***************************/ +// reduce sum across vector +NPY_FINLINE npy_uint32 npyv_sum_u32(npyv_u32 a) +{ + __m256i s0 = _mm256_hadd_epi32(a, a); + s0 = _mm256_hadd_epi32(s0, s0); + __m128i s1 = _mm256_extracti128_si256(s0, 1);; + s1 = _mm_add_epi32(_mm256_castsi256_si128(s0), s1); + return _mm_cvtsi128_si32(s1); +} + +NPY_FINLINE npy_uint64 npyv_sum_u64(npyv_u64 a) +{ + __m256i two = _mm256_add_epi64(a, _mm256_shuffle_epi32(a, _MM_SHUFFLE(1, 0, 3, 2))); + __m128i one = _mm_add_epi64(_mm256_castsi256_si128(two), _mm256_extracti128_si256(two, 1)); + return (npy_uint64)npyv128_cvtsi128_si64(one); +} + +NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { __m256 sum_halves = _mm256_hadd_ps(a, a); sum_halves = _mm256_hadd_ps(sum_halves, sum_halves); @@ -128,7 +306,7 @@ NPY_FINLINE float npyv_sum_f32(__m256 a) return _mm_cvtss_f32(sum); } -NPY_FINLINE double npyv_sum_f64(__m256d a) +NPY_FINLINE double npyv_sum_f64(npyv_f64 a) { __m256d sum_halves = _mm256_hadd_pd(a, a); __m128d lo = _mm256_castpd256_pd128(sum_halves); @@ -137,6 +315,22 @@ NPY_FINLINE double npyv_sum_f64(__m256d a) return _mm_cvtsd_f64(sum); } -#endif // _NPY_SIMD_AVX2_ARITHMETIC_H +// expand the source vector and performs sum reduce +NPY_FINLINE npy_uint16 npyv_sumup_u8(npyv_u8 a) +{ + __m256i four = _mm256_sad_epu8(a, _mm256_setzero_si256()); + __m128i two = _mm_add_epi16(_mm256_castsi256_si128(four), _mm256_extracti128_si256(four, 1)); + __m128i one = _mm_add_epi16(two, _mm_unpackhi_epi64(two, two)); + return (npy_uint16)_mm_cvtsi128_si32(one); +} +NPY_FINLINE npy_uint32 npyv_sumup_u16(npyv_u16 a) +{ + const npyv_u16 even_mask = _mm256_set1_epi32(0x0000FFFF); + __m256i even = _mm256_and_si256(a, even_mask); + __m256i odd = _mm256_srli_epi32(a, 16); + __m256i eight = _mm256_add_epi32(even, odd); + return npyv_sum_u32(eight); +} +#endif // _NPY_SIMD_AVX2_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/avx2/avx2.h b/numpy/core/src/common/simd/avx2/avx2.h index 6f0d3c0d97da..02ff536fb15e 100644 --- a/numpy/core/src/common/simd/avx2/avx2.h +++ b/numpy/core/src/common/simd/avx2/avx2.h @@ -1,10 +1,14 @@ #ifndef _NPY_SIMD_H_ #error "Not a standalone header" #endif - #define NPY_SIMD 256 #define NPY_SIMD_WIDTH 32 #define NPY_SIMD_F64 1 +#ifdef NPY_HAVE_FMA3 + #define NPY_SIMD_FMA3 1 // native support +#else + #define NPY_SIMD_FMA3 0 // fast emulated +#endif // Enough limit to allow us to use _mm256_i32gather_* #define NPY_SIMD_MAXLOAD_STRIDE32 (0x7fffffff / 8) diff --git a/numpy/core/src/common/simd/avx2/conversion.h b/numpy/core/src/common/simd/avx2/conversion.h index 9fd86016d3d9..64e051686794 100644 --- a/numpy/core/src/common/simd/avx2/conversion.h +++ b/numpy/core/src/common/simd/avx2/conversion.h @@ -14,8 +14,8 @@ #define npyv_cvt_s32_b32(A) A #define npyv_cvt_u64_b64(A) A #define npyv_cvt_s64_b64(A) A -#define npyv_cvt_f32_b32(A) _mm256_castsi256_ps(A) -#define npyv_cvt_f64_b64(A) _mm256_castsi256_pd(A) +#define npyv_cvt_f32_b32 _mm256_castsi256_ps +#define npyv_cvt_f64_b64 _mm256_castsi256_pd // convert integer types to mask types #define npyv_cvt_b8_u8(BL) BL @@ -26,7 +26,44 @@ #define npyv_cvt_b32_s32(BL) BL #define npyv_cvt_b64_u64(BL) BL #define npyv_cvt_b64_s64(BL) BL -#define npyv_cvt_b32_f32(BL) _mm256_castps_si256(BL) -#define npyv_cvt_b64_f64(BL) _mm256_castpd_si256(BL) +#define npyv_cvt_b32_f32 _mm256_castps_si256 +#define npyv_cvt_b64_f64 _mm256_castpd_si256 + +// convert boolean vector to integer bitfield +NPY_FINLINE npy_uint64 npyv_tobits_b8(npyv_b8 a) +{ return (npy_uint32)_mm256_movemask_epi8(a); } + +NPY_FINLINE npy_uint64 npyv_tobits_b16(npyv_b16 a) +{ + __m128i pack = _mm_packs_epi16(_mm256_castsi256_si128(a), _mm256_extracti128_si256(a, 1)); + return (npy_uint16)_mm_movemask_epi8(pack); +} +NPY_FINLINE npy_uint64 npyv_tobits_b32(npyv_b32 a) +{ return (npy_uint8)_mm256_movemask_ps(_mm256_castsi256_ps(a)); } +NPY_FINLINE npy_uint64 npyv_tobits_b64(npyv_b64 a) +{ return (npy_uint8)_mm256_movemask_pd(_mm256_castsi256_pd(a)); } + +// expand +NPY_FINLINE npyv_u16x2 npyv_expand_u16_u8(npyv_u8 data) { + npyv_u16x2 r; + r.val[0] = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(data)); + r.val[1] = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(data, 1)); + return r; +} + +NPY_FINLINE npyv_u32x2 npyv_expand_u32_u16(npyv_u16 data) { + npyv_u32x2 r; + r.val[0] = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(data)); + r.val[1] = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(data, 1)); + return r; +} + +// round to nearest integer (assuming even) +#define npyv_round_s32_f32 _mm256_cvtps_epi32 +NPY_FINLINE npyv_s32 npyv_round_s32_f64(npyv_f64 a, npyv_f64 b) +{ + __m128i lo = _mm256_cvtpd_epi32(a), hi = _mm256_cvtpd_epi32(b); + return _mm256_inserti128_si256(_mm256_castsi128_si256(lo), hi, 1); +} #endif // _NPY_SIMD_AVX2_CVT_H diff --git a/numpy/core/src/common/simd/avx2/math.h b/numpy/core/src/common/simd/avx2/math.h index b3eba6f5ff79..9460183df5bb 100644 --- a/numpy/core/src/common/simd/avx2/math.h +++ b/numpy/core/src/common/simd/avx2/math.h @@ -37,4 +37,72 @@ NPY_FINLINE npyv_f32 npyv_square_f32(npyv_f32 a) NPY_FINLINE npyv_f64 npyv_square_f64(npyv_f64 a) { return _mm256_mul_pd(a, a); } -#endif +// Maximum, natively mapping with no guarantees to handle NaN. +#define npyv_max_f32 _mm256_max_ps +#define npyv_max_f64 _mm256_max_pd +// Maximum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +NPY_FINLINE npyv_f32 npyv_maxp_f32(npyv_f32 a, npyv_f32 b) +{ + __m256 nn = _mm256_cmp_ps(b, b, _CMP_ORD_Q); + __m256 max = _mm256_max_ps(a, b); + return _mm256_blendv_ps(a, max, nn); +} +NPY_FINLINE npyv_f64 npyv_maxp_f64(npyv_f64 a, npyv_f64 b) +{ + __m256d nn = _mm256_cmp_pd(b, b, _CMP_ORD_Q); + __m256d max = _mm256_max_pd(a, b); + return _mm256_blendv_pd(a, max, nn); +} +// Maximum, integer operations +#define npyv_max_u8 _mm256_max_epu8 +#define npyv_max_s8 _mm256_max_epi8 +#define npyv_max_u16 _mm256_max_epu16 +#define npyv_max_s16 _mm256_max_epi16 +#define npyv_max_u32 _mm256_max_epu32 +#define npyv_max_s32 _mm256_max_epi32 +NPY_FINLINE npyv_u64 npyv_max_u64(npyv_u64 a, npyv_u64 b) +{ + return _mm256_blendv_epi8(b, a, npyv_cmpgt_u64(a, b)); +} +NPY_FINLINE npyv_s64 npyv_max_s64(npyv_s64 a, npyv_s64 b) +{ + return _mm256_blendv_epi8(b, a, _mm256_cmpgt_epi64(a, b)); +} + +// Minimum, natively mapping with no guarantees to handle NaN. +#define npyv_min_f32 _mm256_min_ps +#define npyv_min_f64 _mm256_min_pd +// Minimum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +NPY_FINLINE npyv_f32 npyv_minp_f32(npyv_f32 a, npyv_f32 b) +{ + __m256 nn = _mm256_cmp_ps(b, b, _CMP_ORD_Q); + __m256 min = _mm256_min_ps(a, b); + return _mm256_blendv_ps(a, min, nn); +} +NPY_FINLINE npyv_f64 npyv_minp_f64(npyv_f64 a, npyv_f64 b) +{ + __m256d nn = _mm256_cmp_pd(b, b, _CMP_ORD_Q); + __m256d min = _mm256_min_pd(a, b); + return _mm256_blendv_pd(a, min, nn); +} +// Minimum, integer operations +#define npyv_min_u8 _mm256_min_epu8 +#define npyv_min_s8 _mm256_min_epi8 +#define npyv_min_u16 _mm256_min_epu16 +#define npyv_min_s16 _mm256_min_epi16 +#define npyv_min_u32 _mm256_min_epu32 +#define npyv_min_s32 _mm256_min_epi32 +NPY_FINLINE npyv_u64 npyv_min_u64(npyv_u64 a, npyv_u64 b) +{ + return _mm256_blendv_epi8(b, a, npyv_cmplt_u64(a, b)); +} +NPY_FINLINE npyv_s64 npyv_min_s64(npyv_s64 a, npyv_s64 b) +{ + return _mm256_blendv_epi8(a, b, _mm256_cmpgt_epi64(a, b)); +} + +#endif // _NPY_SIMD_AVX2_MATH_H diff --git a/numpy/core/src/common/simd/avx2/operators.h b/numpy/core/src/common/simd/avx2/operators.h index c1d30413fe5e..5fc7719e916d 100644 --- a/numpy/core/src/common/simd/avx2/operators.h +++ b/numpy/core/src/common/simd/avx2/operators.h @@ -61,6 +61,10 @@ NPY_FINLINE __m256i npyv_shr_s64(__m256i a, int c) #define npyv_and_s64 _mm256_and_si256 #define npyv_and_f32 _mm256_and_ps #define npyv_and_f64 _mm256_and_pd +#define npyv_and_b8 _mm256_and_si256 +#define npyv_and_b16 _mm256_and_si256 +#define npyv_and_b32 _mm256_and_si256 +#define npyv_and_b64 _mm256_and_si256 // OR #define npyv_or_u8 _mm256_or_si256 @@ -73,6 +77,10 @@ NPY_FINLINE __m256i npyv_shr_s64(__m256i a, int c) #define npyv_or_s64 _mm256_or_si256 #define npyv_or_f32 _mm256_or_ps #define npyv_or_f64 _mm256_or_pd +#define npyv_or_b8 _mm256_or_si256 +#define npyv_or_b16 _mm256_or_si256 +#define npyv_or_b32 _mm256_or_si256 +#define npyv_or_b64 _mm256_or_si256 // XOR #define npyv_xor_u8 _mm256_xor_si256 @@ -85,6 +93,10 @@ NPY_FINLINE __m256i npyv_shr_s64(__m256i a, int c) #define npyv_xor_s64 _mm256_xor_si256 #define npyv_xor_f32 _mm256_xor_ps #define npyv_xor_f64 _mm256_xor_pd +#define npyv_xor_b8 _mm256_xor_si256 +#define npyv_xor_b16 _mm256_xor_si256 +#define npyv_xor_b32 _mm256_xor_si256 +#define npyv_xor_b64 _mm256_xor_si256 // NOT #define npyv_not_u8(A) _mm256_xor_si256(A, _mm256_set1_epi32(-1)) @@ -97,6 +109,10 @@ NPY_FINLINE __m256i npyv_shr_s64(__m256i a, int c) #define npyv_not_s64 npyv_not_u8 #define npyv_not_f32(A) _mm256_xor_ps(A, _mm256_castsi256_ps(_mm256_set1_epi32(-1))) #define npyv_not_f64(A) _mm256_xor_pd(A, _mm256_castsi256_pd(_mm256_set1_epi32(-1))) +#define npyv_not_b8 npyv_not_u8 +#define npyv_not_b16 npyv_not_u8 +#define npyv_not_b32 npyv_not_u8 +#define npyv_not_b64 npyv_not_u8 /*************************** * Comparison @@ -197,4 +213,10 @@ NPY_FINLINE __m256i npyv_cmpge_u32(__m256i a, __m256i b) #define npyv_cmpge_f32(A, B) _mm256_castps_si256(_mm256_cmp_ps(A, B, _CMP_GE_OQ)) #define npyv_cmpge_f64(A, B) _mm256_castpd_si256(_mm256_cmp_pd(A, B, _CMP_GE_OQ)) +// check special cases +NPY_FINLINE npyv_b32 npyv_notnan_f32(npyv_f32 a) +{ return _mm256_castps_si256(_mm256_cmp_ps(a, a, _CMP_ORD_Q)); } +NPY_FINLINE npyv_b64 npyv_notnan_f64(npyv_f64 a) +{ return _mm256_castpd_si256(_mm256_cmp_pd(a, a, _CMP_ORD_Q)); } + #endif // _NPY_SIMD_AVX2_OPERATORS_H diff --git a/numpy/core/src/common/simd/avx2/reorder.h b/numpy/core/src/common/simd/avx2/reorder.h index 5a9e68e32286..4d6ec8f759b5 100644 --- a/numpy/core/src/common/simd/avx2/reorder.h +++ b/numpy/core/src/common/simd/avx2/reorder.h @@ -94,4 +94,36 @@ NPY_FINLINE npyv_f64x2 npyv_zip_f64(__m256d a, __m256d b) return npyv_combine_f64(ab0, ab1); } +// Reverse elements of each 64-bit lane +NPY_FINLINE npyv_u8 npyv_rev64_u8(npyv_u8 a) +{ + const __m256i idx = _mm256_setr_epi8( + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8 + ); + return _mm256_shuffle_epi8(a, idx); +} +#define npyv_rev64_s8 npyv_rev64_u8 + +NPY_FINLINE npyv_u16 npyv_rev64_u16(npyv_u16 a) +{ + const __m256i idx = _mm256_setr_epi8( + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9 + ); + return _mm256_shuffle_epi8(a, idx); +} +#define npyv_rev64_s16 npyv_rev64_u16 + +NPY_FINLINE npyv_u32 npyv_rev64_u32(npyv_u32 a) +{ + return _mm256_shuffle_epi32(a, _MM_SHUFFLE(2, 3, 0, 1)); +} +#define npyv_rev64_s32 npyv_rev64_u32 + +NPY_FINLINE npyv_f32 npyv_rev64_f32(npyv_f32 a) +{ + return _mm256_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); +} + #endif // _NPY_SIMD_AVX2_REORDER_H diff --git a/numpy/core/src/common/simd/avx512/arithmetic.h b/numpy/core/src/common/simd/avx512/arithmetic.h index 7372ca29e40d..f8632e701790 100644 --- a/numpy/core/src/common/simd/avx512/arithmetic.h +++ b/numpy/core/src/common/simd/avx512/arithmetic.h @@ -6,7 +6,7 @@ #define _NPY_SIMD_AVX512_ARITHMETIC_H #include "../avx2/utils.h" - +#include "../sse/utils.h" /*************************** * Addition ***************************/ @@ -39,7 +39,7 @@ NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_adds_u16, _mm256_adds_epu16) NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_adds_s16, _mm256_adds_epi16) #endif -// TODO: rest, after implment Packs intrins +// TODO: rest, after implement Packs intrins /*************************** * Subtraction @@ -73,7 +73,7 @@ NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_subs_u16, _mm256_subs_epu16) NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_subs_s16, _mm256_subs_epi16) #endif -// TODO: rest, after implment Packs intrins +// TODO: rest, after implement Packs intrins /*************************** * Multiplication @@ -104,8 +104,229 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) #define npyv_mul_f64 _mm512_mul_pd // saturated -// TODO: after implment Packs intrins +// TODO: after implement Packs intrins + +/*************************** + * Integer Division + ***************************/ +// See simd/intdiv.h for more clarification +// divide each unsigned 8-bit element by divisor +NPY_FINLINE npyv_u8 npyv_divc_u8(npyv_u8 a, const npyv_u8x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + const __m128i shf2 = _mm512_castsi512_si128(divisor.val[2]); +#ifdef NPY_HAVE_AVX512BW + const __m512i bmask = _mm512_set1_epi32(0x00FF00FF); + const __m512i shf1b = _mm512_set1_epi8(0xFFU >> _mm_cvtsi128_si32(shf1)); + const __m512i shf2b = _mm512_set1_epi8(0xFFU >> _mm_cvtsi128_si32(shf2)); + // high part of unsigned multiplication + __m512i mulhi_even = _mm512_mullo_epi16(_mm512_and_si512(a, bmask), divisor.val[0]); + mulhi_even = _mm512_srli_epi16(mulhi_even, 8); + __m512i mulhi_odd = _mm512_mullo_epi16(_mm512_srli_epi16(a, 8), divisor.val[0]); + __m512i mulhi = _mm512_mask_mov_epi8(mulhi_even, 0xAAAAAAAAAAAAAAAA, mulhi_odd); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m512i q = _mm512_sub_epi8(a, mulhi); + q = _mm512_and_si512(_mm512_srl_epi16(q, shf1), shf1b); + q = _mm512_add_epi8(mulhi, q); + q = _mm512_and_si512(_mm512_srl_epi16(q, shf2), shf2b); + return q; +#else + const __m256i bmask = _mm256_set1_epi32(0x00FF00FF); + const __m256i shf1b = _mm256_set1_epi8(0xFFU >> _mm_cvtsi128_si32(shf1)); + const __m256i shf2b = _mm256_set1_epi8(0xFFU >> _mm_cvtsi128_si32(shf2)); + const __m512i shf2bw= npyv512_combine_si256(shf2b, shf2b); + const __m256i mulc = npyv512_lower_si256(divisor.val[0]); + //// lower 256-bit + __m256i lo_a = npyv512_lower_si256(a); + // high part of unsigned multiplication + __m256i mulhi_even = _mm256_mullo_epi16(_mm256_and_si256(lo_a, bmask), mulc); + mulhi_even = _mm256_srli_epi16(mulhi_even, 8); + __m256i mulhi_odd = _mm256_mullo_epi16(_mm256_srli_epi16(lo_a, 8), mulc); + __m256i mulhi = _mm256_blendv_epi8(mulhi_odd, mulhi_even, bmask); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m256i lo_q = _mm256_sub_epi8(lo_a, mulhi); + lo_q = _mm256_and_si256(_mm256_srl_epi16(lo_q, shf1), shf1b); + lo_q = _mm256_add_epi8(mulhi, lo_q); + lo_q = _mm256_srl_epi16(lo_q, shf2); // no sign extend + + //// higher 256-bit + __m256i hi_a = npyv512_higher_si256(a); + // high part of unsigned multiplication + mulhi_even = _mm256_mullo_epi16(_mm256_and_si256(hi_a, bmask), mulc); + mulhi_even = _mm256_srli_epi16(mulhi_even, 8); + mulhi_odd = _mm256_mullo_epi16(_mm256_srli_epi16(hi_a, 8), mulc); + mulhi = _mm256_blendv_epi8(mulhi_odd, mulhi_even, bmask); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m256i hi_q = _mm256_sub_epi8(hi_a, mulhi); + hi_q = _mm256_and_si256(_mm256_srl_epi16(hi_q, shf1), shf1b); + hi_q = _mm256_add_epi8(mulhi, hi_q); + hi_q = _mm256_srl_epi16(hi_q, shf2); // no sign extend + return _mm512_and_si512(npyv512_combine_si256(lo_q, hi_q), shf2bw); // extend sign +#endif +} +// divide each signed 8-bit element by divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor); +NPY_FINLINE npyv_s8 npyv_divc_s8(npyv_s8 a, const npyv_s8x3 divisor) +{ + __m512i divc_even = npyv_divc_s16(npyv_shri_s16(npyv_shli_s16(a, 8), 8), divisor); + __m512i divc_odd = npyv_divc_s16(npyv_shri_s16(a, 8), divisor); + divc_odd = npyv_shli_s16(divc_odd, 8); +#ifdef NPY_HAVE_AVX512BW + return _mm512_mask_mov_epi8(divc_even, 0xAAAAAAAAAAAAAAAA, divc_odd); +#else + const __m512i bmask = _mm512_set1_epi32(0x00FF00FF); + return npyv_select_u8(bmask, divc_even, divc_odd); +#endif +} +// divide each unsigned 16-bit element by divisor +NPY_FINLINE npyv_u16 npyv_divc_u16(npyv_u16 a, const npyv_u16x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + const __m128i shf2 = _mm512_castsi512_si128(divisor.val[2]); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + #define NPYV__DIVC_U16(RLEN, A, MULC, R) \ + mulhi = _mm##RLEN##_mulhi_epu16(A, MULC); \ + R = _mm##RLEN##_sub_epi16(A, mulhi); \ + R = _mm##RLEN##_srl_epi16(R, shf1); \ + R = _mm##RLEN##_add_epi16(mulhi, R); \ + R = _mm##RLEN##_srl_epi16(R, shf2); + +#ifdef NPY_HAVE_AVX512BW + __m512i mulhi, q; + NPYV__DIVC_U16(512, a, divisor.val[0], q) + return q; +#else + const __m256i m = npyv512_lower_si256(divisor.val[0]); + __m256i lo_a = npyv512_lower_si256(a); + __m256i hi_a = npyv512_higher_si256(a); + + __m256i mulhi, lo_q, hi_q; + NPYV__DIVC_U16(256, lo_a, m, lo_q) + NPYV__DIVC_U16(256, hi_a, m, hi_q) + return npyv512_combine_si256(lo_q, hi_q); +#endif + #undef NPYV__DIVC_U16 +} +// divide each signed 16-bit element by divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + #define NPYV__DIVC_S16(RLEN, A, MULC, DSIGN, R) \ + mulhi = _mm##RLEN##_mulhi_epi16(A, MULC); \ + R = _mm##RLEN##_sra_epi16(_mm##RLEN##_add_epi16(A, mulhi), shf1); \ + R = _mm##RLEN##_sub_epi16(R, _mm##RLEN##_srai_epi16(A, 15)); \ + R = _mm##RLEN##_sub_epi16(_mm##RLEN##_xor_si##RLEN(R, DSIGN), DSIGN); + +#ifdef NPY_HAVE_AVX512BW + __m512i mulhi, q; + NPYV__DIVC_S16(512, a, divisor.val[0], divisor.val[2], q) + return q; +#else + const __m256i m = npyv512_lower_si256(divisor.val[0]); + const __m256i dsign = npyv512_lower_si256(divisor.val[2]); + __m256i lo_a = npyv512_lower_si256(a); + __m256i hi_a = npyv512_higher_si256(a); + + __m256i mulhi, lo_q, hi_q; + NPYV__DIVC_S16(256, lo_a, m, dsign, lo_q) + NPYV__DIVC_S16(256, hi_a, m, dsign, hi_q) + return npyv512_combine_si256(lo_q, hi_q); +#endif + #undef NPYV__DIVC_S16 +} +// divide each unsigned 32-bit element by divisor +NPY_FINLINE npyv_u32 npyv_divc_u32(npyv_u32 a, const npyv_u32x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + const __m128i shf2 = _mm512_castsi512_si128(divisor.val[2]); + // high part of unsigned multiplication + __m512i mulhi_even = _mm512_srli_epi64(_mm512_mul_epu32(a, divisor.val[0]), 32); + __m512i mulhi_odd = _mm512_mul_epu32(_mm512_srli_epi64(a, 32), divisor.val[0]); + __m512i mulhi = _mm512_mask_mov_epi32(mulhi_even, 0xAAAA, mulhi_odd); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m512i q = _mm512_sub_epi32(a, mulhi); + q = _mm512_srl_epi32(q, shf1); + q = _mm512_add_epi32(mulhi, q); + q = _mm512_srl_epi32(q, shf2); + return q; +} +// divide each signed 32-bit element by divisor (round towards zero) +NPY_FINLINE npyv_s32 npyv_divc_s32(npyv_s32 a, const npyv_s32x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + // high part of signed multiplication + __m512i mulhi_even = _mm512_srli_epi64(_mm512_mul_epi32(a, divisor.val[0]), 32); + __m512i mulhi_odd = _mm512_mul_epi32(_mm512_srli_epi64(a, 32), divisor.val[0]); + __m512i mulhi = _mm512_mask_mov_epi32(mulhi_even, 0xAAAA, mulhi_odd); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + __m512i q = _mm512_sra_epi32(_mm512_add_epi32(a, mulhi), shf1); + q = _mm512_sub_epi32(q, _mm512_srai_epi32(a, 31)); + q = _mm512_sub_epi32(_mm512_xor_si512(q, divisor.val[2]), divisor.val[2]); + return q; +} +// returns the high 64 bits of unsigned 64-bit multiplication +// xref https://stackoverflow.com/a/28827013 +NPY_FINLINE npyv_u64 npyv__mullhi_u64(npyv_u64 a, npyv_u64 b) +{ + __m512i lomask = npyv_setall_s64(0xffffffff); + __m512i a_hi = _mm512_srli_epi64(a, 32); // a0l, a0h, a1l, a1h + __m512i b_hi = _mm512_srli_epi64(b, 32); // b0l, b0h, b1l, b1h + // compute partial products + __m512i w0 = _mm512_mul_epu32(a, b); // a0l*b0l, a1l*b1l + __m512i w1 = _mm512_mul_epu32(a, b_hi); // a0l*b0h, a1l*b1h + __m512i w2 = _mm512_mul_epu32(a_hi, b); // a0h*b0l, a1h*b0l + __m512i w3 = _mm512_mul_epu32(a_hi, b_hi); // a0h*b0h, a1h*b1h + // sum partial products + __m512i w0h = _mm512_srli_epi64(w0, 32); + __m512i s1 = _mm512_add_epi64(w1, w0h); + __m512i s1l = _mm512_and_si512(s1, lomask); + __m512i s1h = _mm512_srli_epi64(s1, 32); + + __m512i s2 = _mm512_add_epi64(w2, s1l); + __m512i s2h = _mm512_srli_epi64(s2, 32); + __m512i hi = _mm512_add_epi64(w3, s1h); + hi = _mm512_add_epi64(hi, s2h); + return hi; +} +// divide each unsigned 64-bit element by a divisor +NPY_FINLINE npyv_u64 npyv_divc_u64(npyv_u64 a, const npyv_u64x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + const __m128i shf2 = _mm512_castsi512_si128(divisor.val[2]); + // high part of unsigned multiplication + __m512i mulhi = npyv__mullhi_u64(a, divisor.val[0]); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m512i q = _mm512_sub_epi64(a, mulhi); + q = _mm512_srl_epi64(q, shf1); + q = _mm512_add_epi64(mulhi, q); + q = _mm512_srl_epi64(q, shf2); + return q; +} +// divide each unsigned 64-bit element by a divisor (round towards zero) +NPY_FINLINE npyv_s64 npyv_divc_s64(npyv_s64 a, const npyv_s64x3 divisor) +{ + const __m128i shf1 = _mm512_castsi512_si128(divisor.val[1]); + // high part of unsigned multiplication + __m512i mulhi = npyv__mullhi_u64(a, divisor.val[0]); + // convert unsigned to signed high multiplication + // mulhi - ((a < 0) ? m : 0) - ((m < 0) ? a : 0); + __m512i asign = _mm512_srai_epi64(a, 63); + __m512i msign = _mm512_srai_epi64(divisor.val[0], 63); + __m512i m_asign = _mm512_and_si512(divisor.val[0], asign); + __m512i a_msign = _mm512_and_si512(a, msign); + mulhi = _mm512_sub_epi64(mulhi, m_asign); + mulhi = _mm512_sub_epi64(mulhi, a_msign); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + __m512i q = _mm512_sra_epi64(_mm512_add_epi64(a, mulhi), shf1); + q = _mm512_sub_epi64(q, asign); + q = _mm512_sub_epi64(_mm512_xor_si512(q, divisor.val[2]), divisor.val[2]); + return q; +} /*************************** * Division ***************************/ @@ -130,24 +351,43 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) #define npyv_nmulsub_f64 _mm512_fnmsub_pd /*************************** - * Reduce Sum + * Summation: Calculates the sum of all vector elements. * there are three ways to implement reduce sum for AVX512: * 1- split(256) /add /split(128) /add /hadd /hadd /extract * 2- shuff(cross) /add /shuff(cross) /add /shuff /add /shuff /add /extract * 3- _mm512_reduce_add_ps/pd * The first one is been widely used by many projects - * + * * the second one is used by Intel Compiler, maybe because the * latency of hadd increased by (2-3) starting from Skylake-X which makes two * extra shuffles(non-cross) cheaper. check https://godbolt.org/z/s3G9Er for more info. - * + * * The third one is almost the same as the second one but only works for * intel compiler/GCC 7.1/Clang 4, we still need to support older GCC. ***************************/ +// reduce sum across vector #ifdef NPY_HAVE_AVX512F_REDUCE + #define npyv_sum_u32 _mm512_reduce_add_epi32 + #define npyv_sum_u64 _mm512_reduce_add_epi64 #define npyv_sum_f32 _mm512_reduce_add_ps #define npyv_sum_f64 _mm512_reduce_add_pd #else + NPY_FINLINE npy_uint32 npyv_sum_u32(npyv_u32 a) + { + __m256i half = _mm256_add_epi32(npyv512_lower_si256(a), npyv512_higher_si256(a)); + __m128i quarter = _mm_add_epi32(_mm256_castsi256_si128(half), _mm256_extracti128_si256(half, 1)); + quarter = _mm_hadd_epi32(quarter, quarter); + return _mm_cvtsi128_si32(_mm_hadd_epi32(quarter, quarter)); + } + + NPY_FINLINE npy_uint64 npyv_sum_u64(npyv_u64 a) + { + __m256i four = _mm256_add_epi64(npyv512_lower_si256(a), npyv512_higher_si256(a)); + __m256i two = _mm256_add_epi64(four, _mm256_shuffle_epi32(four, _MM_SHUFFLE(1, 0, 3, 2))); + __m128i one = _mm_add_epi64(_mm256_castsi256_si128(two), _mm256_extracti128_si256(two, 1)); + return (npy_uint64)npyv128_cvtsi128_si64(one); + } + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { __m512 h64 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 3, 2)); @@ -160,6 +400,7 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) __m512 sum4 = _mm512_add_ps(sum8, h4); return _mm_cvtss_f32(_mm512_castps512_ps128(sum4)); } + NPY_FINLINE double npyv_sum_f64(npyv_f64 a) { __m512d h64 = _mm512_shuffle_f64x2(a, a, _MM_SHUFFLE(3, 2, 3, 2)); @@ -172,4 +413,29 @@ NPY_FINLINE __m512i npyv_mul_u8(__m512i a, __m512i b) } #endif +// expand the source vector and performs sum reduce +NPY_FINLINE npy_uint16 npyv_sumup_u8(npyv_u8 a) +{ +#ifdef NPY_HAVE_AVX512BW + __m512i eight = _mm512_sad_epu8(a, _mm512_setzero_si512()); + __m256i four = _mm256_add_epi16(npyv512_lower_si256(eight), npyv512_higher_si256(eight)); +#else + __m256i lo_four = _mm256_sad_epu8(npyv512_lower_si256(a), _mm256_setzero_si256()); + __m256i hi_four = _mm256_sad_epu8(npyv512_higher_si256(a), _mm256_setzero_si256()); + __m256i four = _mm256_add_epi16(lo_four, hi_four); +#endif + __m128i two = _mm_add_epi16(_mm256_castsi256_si128(four), _mm256_extracti128_si256(four, 1)); + __m128i one = _mm_add_epi16(two, _mm_unpackhi_epi64(two, two)); + return (npy_uint16)_mm_cvtsi128_si32(one); +} + +NPY_FINLINE npy_uint32 npyv_sumup_u16(npyv_u16 a) +{ + const npyv_u16 even_mask = _mm512_set1_epi32(0x0000FFFF); + __m512i even = _mm512_and_si512(a, even_mask); + __m512i odd = _mm512_srli_epi32(a, 16); + __m512i ff = _mm512_add_epi32(even, odd); + return npyv_sum_u32(ff); +} + #endif // _NPY_SIMD_AVX512_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/avx512/avx512.h b/numpy/core/src/common/simd/avx512/avx512.h index 2de33765a97c..f38686834cfb 100644 --- a/numpy/core/src/common/simd/avx512/avx512.h +++ b/numpy/core/src/common/simd/avx512/avx512.h @@ -4,6 +4,7 @@ #define NPY_SIMD 512 #define NPY_SIMD_WIDTH 64 #define NPY_SIMD_F64 1 +#define NPY_SIMD_FMA3 1 // native support // Enough limit to allow us to use _mm512_i32gather_* and _mm512_i32scatter_* #define NPY_SIMD_MAXLOAD_STRIDE32 (0x7fffffff / 16) #define NPY_SIMD_MAXSTORE_STRIDE32 (0x7fffffff / 16) @@ -73,3 +74,4 @@ typedef struct { __m512d val[3]; } npyv_f64x3; #include "conversion.h" #include "arithmetic.h" #include "math.h" +#include "maskop.h" diff --git a/numpy/core/src/common/simd/avx512/conversion.h b/numpy/core/src/common/simd/avx512/conversion.h index 0f7e27de3a0a..0bd44179b332 100644 --- a/numpy/core/src/common/simd/avx512/conversion.h +++ b/numpy/core/src/common/simd/avx512/conversion.h @@ -51,4 +51,88 @@ #define npyv_cvt_b32_f32(A) npyv_cvt_b32_u32(_mm512_castps_si512(A)) #define npyv_cvt_b64_f64(A) npyv_cvt_b64_u64(_mm512_castpd_si512(A)) +// expand +NPY_FINLINE npyv_u16x2 npyv_expand_u16_u8(npyv_u8 data) +{ + npyv_u16x2 r; + __m256i lo = npyv512_lower_si256(data); + __m256i hi = npyv512_higher_si256(data); +#ifdef NPY_HAVE_AVX512BW + r.val[0] = _mm512_cvtepu8_epi16(lo); + r.val[1] = _mm512_cvtepu8_epi16(hi); +#else + __m256i loelo = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(lo)); + __m256i loehi = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(lo, 1)); + __m256i hielo = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(hi)); + __m256i hiehi = _mm256_cvtepu8_epi16(_mm256_extracti128_si256(hi, 1)); + r.val[0] = npyv512_combine_si256(loelo, loehi); + r.val[1] = npyv512_combine_si256(hielo, hiehi); +#endif + return r; +} + +NPY_FINLINE npyv_u32x2 npyv_expand_u32_u16(npyv_u16 data) +{ + npyv_u32x2 r; + __m256i lo = npyv512_lower_si256(data); + __m256i hi = npyv512_higher_si256(data); +#ifdef NPY_HAVE_AVX512BW + r.val[0] = _mm512_cvtepu16_epi32(lo); + r.val[1] = _mm512_cvtepu16_epi32(hi); +#else + __m256i loelo = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(lo)); + __m256i loehi = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(lo, 1)); + __m256i hielo = _mm256_cvtepu16_epi32(_mm256_castsi256_si128(hi)); + __m256i hiehi = _mm256_cvtepu16_epi32(_mm256_extracti128_si256(hi, 1)); + r.val[0] = npyv512_combine_si256(loelo, loehi); + r.val[1] = npyv512_combine_si256(hielo, hiehi); +#endif + return r; +} + +// convert boolean vectors to integer bitfield +NPY_FINLINE npy_uint64 npyv_tobits_b8(npyv_b8 a) +{ +#ifdef NPY_HAVE_AVX512BW_MASK + return (npy_uint64)_cvtmask64_u64(a); +#elif defined(NPY_HAVE_AVX512BW) + return (npy_uint64)a; +#else + int mask_lo = _mm256_movemask_epi8(npyv512_lower_si256(a)); + int mask_hi = _mm256_movemask_epi8(npyv512_higher_si256(a)); + return (unsigned)mask_lo | ((npy_uint64)(unsigned)mask_hi << 32); +#endif +} +NPY_FINLINE npy_uint64 npyv_tobits_b16(npyv_b16 a) +{ +#ifdef NPY_HAVE_AVX512BW_MASK + return (npy_uint32)_cvtmask32_u32(a); +#elif defined(NPY_HAVE_AVX512BW) + return (npy_uint32)a; +#else + __m256i pack = _mm256_packs_epi16( + npyv512_lower_si256(a), npyv512_higher_si256(a) + ); + return (npy_uint32)_mm256_movemask_epi8(_mm256_permute4x64_epi64(pack, _MM_SHUFFLE(3, 1, 2, 0))); +#endif +} +NPY_FINLINE npy_uint64 npyv_tobits_b32(npyv_b32 a) +{ return (npy_uint16)a; } +NPY_FINLINE npy_uint64 npyv_tobits_b64(npyv_b64 a) +{ +#ifdef NPY_HAVE_AVX512DQ_MASK + return _cvtmask8_u32(a); +#else + return (npy_uint8)a; +#endif +} + +// round to nearest integer (assuming even) +#define npyv_round_s32_f32 _mm512_cvtps_epi32 +NPY_FINLINE npyv_s32 npyv_round_s32_f64(npyv_f64 a, npyv_f64 b) +{ + __m256i lo = _mm512_cvtpd_epi32(a), hi = _mm512_cvtpd_epi32(b); + return npyv512_combine_si256(lo, hi); +} + #endif // _NPY_SIMD_AVX512_CVT_H diff --git a/numpy/core/src/common/simd/avx512/maskop.h b/numpy/core/src/common/simd/avx512/maskop.h new file mode 100644 index 000000000000..d1c188390a11 --- /dev/null +++ b/numpy/core/src/common/simd/avx512/maskop.h @@ -0,0 +1,54 @@ +#ifndef NPY_SIMD + #error "Not a standalone header, use simd/simd.h instead" +#endif + +#ifndef _NPY_SIMD_AVX512_MASKOP_H +#define _NPY_SIMD_AVX512_MASKOP_H + +/** + * Implements conditional addition and subtraction. + * e.g. npyv_ifadd_f32(m, a, b, c) -> m ? a + b : c + * e.g. npyv_ifsub_f32(m, a, b, c) -> m ? a - b : c + */ +#define NPYV_IMPL_AVX512_EMULATE_MASK_ADDSUB(SFX, BSFX) \ + NPY_FINLINE npyv_##SFX npyv_ifadd_##SFX \ + (npyv_##BSFX m, npyv_##SFX a, npyv_##SFX b, npyv_##SFX c) \ + { \ + npyv_##SFX add = npyv_add_##SFX(a, b); \ + return npyv_select_##SFX(m, add, c); \ + } \ + NPY_FINLINE npyv_##SFX npyv_ifsub_##SFX \ + (npyv_##BSFX m, npyv_##SFX a, npyv_##SFX b, npyv_##SFX c) \ + { \ + npyv_##SFX sub = npyv_sub_##SFX(a, b); \ + return npyv_select_##SFX(m, sub, c); \ + } + +#define NPYV_IMPL_AVX512_MASK_ADDSUB(SFX, BSFX, ZSFX) \ + NPY_FINLINE npyv_##SFX npyv_ifadd_##SFX \ + (npyv_##BSFX m, npyv_##SFX a, npyv_##SFX b, npyv_##SFX c) \ + { return _mm512_mask_add_##ZSFX(c, m, a, b); } \ + NPY_FINLINE npyv_##SFX npyv_ifsub_##SFX \ + (npyv_##BSFX m, npyv_##SFX a, npyv_##SFX b, npyv_##SFX c) \ + { return _mm512_mask_sub_##ZSFX(c, m, a, b); } + +#ifdef NPY_HAVE_AVX512BW + NPYV_IMPL_AVX512_MASK_ADDSUB(u8, b8, epi8) + NPYV_IMPL_AVX512_MASK_ADDSUB(s8, b8, epi8) + NPYV_IMPL_AVX512_MASK_ADDSUB(u16, b16, epi16) + NPYV_IMPL_AVX512_MASK_ADDSUB(s16, b16, epi16) +#else + NPYV_IMPL_AVX512_EMULATE_MASK_ADDSUB(u8, b8) + NPYV_IMPL_AVX512_EMULATE_MASK_ADDSUB(s8, b8) + NPYV_IMPL_AVX512_EMULATE_MASK_ADDSUB(u16, b16) + NPYV_IMPL_AVX512_EMULATE_MASK_ADDSUB(s16, b16) +#endif + +NPYV_IMPL_AVX512_MASK_ADDSUB(u32, b32, epi32) +NPYV_IMPL_AVX512_MASK_ADDSUB(s32, b32, epi32) +NPYV_IMPL_AVX512_MASK_ADDSUB(u64, b64, epi64) +NPYV_IMPL_AVX512_MASK_ADDSUB(s64, b64, epi64) +NPYV_IMPL_AVX512_MASK_ADDSUB(f32, b32, ps) +NPYV_IMPL_AVX512_MASK_ADDSUB(f64, b64, pd) + +#endif // _NPY_SIMD_AVX512_MASKOP_H diff --git a/numpy/core/src/common/simd/avx512/math.h b/numpy/core/src/common/simd/avx512/math.h index 1db710670208..0141396d06a3 100644 --- a/numpy/core/src/common/simd/avx512/math.h +++ b/numpy/core/src/common/simd/avx512/math.h @@ -46,4 +46,70 @@ NPY_FINLINE npyv_f32 npyv_square_f32(npyv_f32 a) NPY_FINLINE npyv_f64 npyv_square_f64(npyv_f64 a) { return _mm512_mul_pd(a, a); } +// Maximum, natively mapping with no guarantees to handle NaN. +#define npyv_max_f32 _mm512_max_ps +#define npyv_max_f64 _mm512_max_pd +// Maximum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +NPY_FINLINE npyv_f32 npyv_maxp_f32(npyv_f32 a, npyv_f32 b) +{ + __mmask16 nn = _mm512_cmp_ps_mask(b, b, _CMP_ORD_Q); + return _mm512_mask_max_ps(a, nn, a, b); +} +NPY_FINLINE npyv_f64 npyv_maxp_f64(npyv_f64 a, npyv_f64 b) +{ + __mmask8 nn = _mm512_cmp_pd_mask(b, b, _CMP_ORD_Q); + return _mm512_mask_max_pd(a, nn, a, b); +} +// Maximum, integer operations +#ifdef NPY_HAVE_AVX512BW + #define npyv_max_u8 _mm512_max_epu8 + #define npyv_max_s8 _mm512_max_epi8 + #define npyv_max_u16 _mm512_max_epu16 + #define npyv_max_s16 _mm512_max_epi16 +#else + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_max_u8, _mm256_max_epu8) + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_max_s8, _mm256_max_epi8) + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_max_u16, _mm256_max_epu16) + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_max_s16, _mm256_max_epi16) #endif +#define npyv_max_u32 _mm512_max_epu32 +#define npyv_max_s32 _mm512_max_epi32 +#define npyv_max_u64 _mm512_max_epu64 +#define npyv_max_s64 _mm512_max_epi64 + +// Minimum, natively mapping with no guarantees to handle NaN. +#define npyv_min_f32 _mm512_min_ps +#define npyv_min_f64 _mm512_min_pd +// Minimum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +NPY_FINLINE npyv_f32 npyv_minp_f32(npyv_f32 a, npyv_f32 b) +{ + __mmask16 nn = _mm512_cmp_ps_mask(b, b, _CMP_ORD_Q); + return _mm512_mask_min_ps(a, nn, a, b); +} +NPY_FINLINE npyv_f64 npyv_minp_f64(npyv_f64 a, npyv_f64 b) +{ + __mmask8 nn = _mm512_cmp_pd_mask(b, b, _CMP_ORD_Q); + return _mm512_mask_min_pd(a, nn, a, b); +} +// Minimum, integer operations +#ifdef NPY_HAVE_AVX512BW + #define npyv_min_u8 _mm512_min_epu8 + #define npyv_min_s8 _mm512_min_epi8 + #define npyv_min_u16 _mm512_min_epu16 + #define npyv_min_s16 _mm512_min_epi16 +#else + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_min_u8, _mm256_min_epu8) + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_min_s8, _mm256_min_epi8) + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_min_u16, _mm256_min_epu16) + NPYV_IMPL_AVX512_FROM_AVX2_2ARG(npyv_min_s16, _mm256_min_epi16) +#endif +#define npyv_min_u32 _mm512_min_epu32 +#define npyv_min_s32 _mm512_min_epi32 +#define npyv_min_u64 _mm512_min_epu64 +#define npyv_min_s64 _mm512_min_epi64 + +#endif // _NPY_SIMD_AVX512_MATH_H diff --git a/numpy/core/src/common/simd/avx512/operators.h b/numpy/core/src/common/simd/avx512/operators.h index f76ea5e2d6c5..d53932fa8726 100644 --- a/numpy/core/src/common/simd/avx512/operators.h +++ b/numpy/core/src/common/simd/avx512/operators.h @@ -90,7 +90,6 @@ NPYV_IMPL_AVX512_FROM_SI512_PS_2ARG(npyv_and_f32, _mm512_and_si512) NPYV_IMPL_AVX512_FROM_SI512_PD_2ARG(npyv_and_f64, _mm512_and_si512) #endif - // OR #define npyv_or_u8 _mm512_or_si512 #define npyv_or_s8 _mm512_or_si512 @@ -124,7 +123,6 @@ NPYV_IMPL_AVX512_FROM_SI512_PS_2ARG(npyv_xor_f32, _mm512_xor_si512) NPYV_IMPL_AVX512_FROM_SI512_PD_2ARG(npyv_xor_f64, _mm512_xor_si512) #endif - // NOT #define npyv_not_u8(A) _mm512_xor_si512(A, _mm512_set1_epi32(-1)) #define npyv_not_s8 npyv_not_u8 @@ -142,6 +140,67 @@ #define npyv_not_f64(A) _mm512_castsi512_pd(npyv_not_u64(_mm512_castpd_si512(A))) #endif +/*************************** + * Logical (boolean) + ***************************/ +#ifdef NPY_HAVE_AVX512BW_MASK + #define npyv_and_b8 _kand_mask64 + #define npyv_and_b16 _kand_mask32 + #define npyv_or_b8 _kor_mask64 + #define npyv_or_b16 _kor_mask32 + #define npyv_xor_b8 _kxor_mask64 + #define npyv_xor_b16 _kxor_mask32 + #define npyv_not_b8 _knot_mask64 + #define npyv_not_b16 _knot_mask32 +#elif defined(NPY_HAVE_AVX512BW) + NPY_FINLINE npyv_b8 npyv_and_b8(npyv_b8 a, npyv_b8 b) + { return a & b; } + NPY_FINLINE npyv_b16 npyv_and_b16(npyv_b16 a, npyv_b16 b) + { return a & b; } + NPY_FINLINE npyv_b8 npyv_or_b8(npyv_b8 a, npyv_b8 b) + { return a | b; } + NPY_FINLINE npyv_b16 npyv_or_b16(npyv_b16 a, npyv_b16 b) + { return a | b; } + NPY_FINLINE npyv_b8 npyv_xor_b8(npyv_b8 a, npyv_b8 b) + { return a ^ b; } + NPY_FINLINE npyv_b16 npyv_xor_b16(npyv_b16 a, npyv_b16 b) + { return a ^ b; } + NPY_FINLINE npyv_b8 npyv_not_b8(npyv_b8 a) + { return ~a; } + NPY_FINLINE npyv_b16 npyv_not_b16(npyv_b16 a) + { return ~a; } +#else + #define npyv_and_b8 _mm512_and_si512 + #define npyv_and_b16 _mm512_and_si512 + #define npyv_or_b8 _mm512_or_si512 + #define npyv_or_b16 _mm512_or_si512 + #define npyv_xor_b8 _mm512_xor_si512 + #define npyv_xor_b16 _mm512_xor_si512 + #define npyv_not_b8 npyv_not_u8 + #define npyv_not_b16 npyv_not_u8 +#endif + +#define npyv_and_b32 _mm512_kand +#define npyv_or_b32 _mm512_kor +#define npyv_xor_b32 _mm512_kxor +#define npyv_not_b32 _mm512_knot + +#ifdef NPY_HAVE_AVX512DQ_MASK + #define npyv_and_b64 _kand_mask8 + #define npyv_or_b64 _kor_mask8 + #define npyv_xor_b64 _kxor_mask8 + #define npyv_not_b64 _knot_mask8 +#else + NPY_FINLINE npyv_b64 npyv_and_b64(npyv_b64 a, npyv_b64 b) + { return (npyv_b64)_mm512_kand((npyv_b32)a, (npyv_b32)b); } + NPY_FINLINE npyv_b64 npyv_or_b64(npyv_b64 a, npyv_b64 b) + { return (npyv_b64)_mm512_kor((npyv_b32)a, (npyv_b32)b); } + NPY_FINLINE npyv_b64 npyv_xor_b64(npyv_b64 a, npyv_b64 b) + { return (npyv_b64)_mm512_kxor((npyv_b32)a, (npyv_b32)b); } + NPY_FINLINE npyv_b64 npyv_not_b64(npyv_b64 a) + { return (npyv_b64)_mm512_knot((npyv_b32)a); } +#endif + /*************************** * Comparison ***************************/ @@ -256,4 +315,10 @@ #define npyv_cmpge_f32(A, B) _mm512_cmp_ps_mask(A, B, _CMP_GE_OQ) #define npyv_cmpge_f64(A, B) _mm512_cmp_pd_mask(A, B, _CMP_GE_OQ) +// check special cases +NPY_FINLINE npyv_b32 npyv_notnan_f32(npyv_f32 a) +{ return _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q); } +NPY_FINLINE npyv_b64 npyv_notnan_f64(npyv_f64 a) +{ return _mm512_cmp_pd_mask(a, a, _CMP_ORD_Q); } + #endif // _NPY_SIMD_AVX512_OPERATORS_H diff --git a/numpy/core/src/common/simd/avx512/reorder.h b/numpy/core/src/common/simd/avx512/reorder.h index cdbae7aaccee..f043004ecc45 100644 --- a/numpy/core/src/common/simd/avx512/reorder.h +++ b/numpy/core/src/common/simd/avx512/reorder.h @@ -167,4 +167,60 @@ NPY_FINLINE npyv_f64x2 npyv_zip_f64(__m512d a, __m512d b) return r; } +// Reverse elements of each 64-bit lane +NPY_FINLINE npyv_u8 npyv_rev64_u8(npyv_u8 a) +{ +#ifdef NPY_HAVE_AVX512BW + const __m512i idx = npyv_set_u8( + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8 + ); + return _mm512_shuffle_epi8(a, idx); +#else + const __m256i idx = _mm256_setr_epi8( + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 + ); + __m256i lo = _mm256_shuffle_epi8(npyv512_lower_si256(a), idx); + __m256i hi = _mm256_shuffle_epi8(npyv512_higher_si256(a), idx); + return npyv512_combine_si256(lo, hi); +#endif +} +#define npyv_rev64_s8 npyv_rev64_u8 + +NPY_FINLINE npyv_u16 npyv_rev64_u16(npyv_u16 a) +{ +#ifdef NPY_HAVE_AVX512BW + const __m512i idx = npyv_set_u8( + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9 + ); + return _mm512_shuffle_epi8(a, idx); +#else + const __m256i idx = _mm256_setr_epi8( + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9 + ); + __m256i lo = _mm256_shuffle_epi8(npyv512_lower_si256(a), idx); + __m256i hi = _mm256_shuffle_epi8(npyv512_higher_si256(a), idx); + return npyv512_combine_si256(lo, hi); +#endif +} +#define npyv_rev64_s16 npyv_rev64_u16 + +NPY_FINLINE npyv_u32 npyv_rev64_u32(npyv_u32 a) +{ + return _mm512_shuffle_epi32(a, _MM_SHUFFLE(2, 3, 0, 1)); +} +#define npyv_rev64_s32 npyv_rev64_u32 + +NPY_FINLINE npyv_f32 npyv_rev64_f32(npyv_f32 a) +{ + return _mm512_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); +} + #endif // _NPY_SIMD_AVX512_REORDER_H diff --git a/numpy/core/src/common/simd/emulate_maskop.h b/numpy/core/src/common/simd/emulate_maskop.h new file mode 100644 index 000000000000..7e7446bc56ef --- /dev/null +++ b/numpy/core/src/common/simd/emulate_maskop.h @@ -0,0 +1,44 @@ +/** + * This header is used internaly by all current supported SIMD extention, + * execpt for AVX512. + */ +#ifndef NPY_SIMD + #error "Not a standalone header, use simd/simd.h instead" +#endif + +#ifndef _NPY_SIMD_EMULATE_MASKOP_H +#define _NPY_SIMD_EMULATE_MASKOP_H + +/** + * Implements conditional addition and subtraction. + * e.g. npyv_ifadd_f32(mask, a, b, c) -> mask ? a + b : c + * e.g. npyv_ifsub_f32(mask, a, b, c) -> mask ? a - b : c + */ +#define NPYV_IMPL_EMULATE_MASK_ADDSUB(SFX, BSFX) \ + NPY_FINLINE npyv_##SFX npyv_ifadd_##SFX \ + (npyv_##BSFX m, npyv_##SFX a, npyv_##SFX b, npyv_##SFX c) \ + { \ + npyv_##SFX add = npyv_add_##SFX(a, b); \ + return npyv_select_##SFX(m, add, c); \ + } \ + NPY_FINLINE npyv_##SFX npyv_ifsub_##SFX \ + (npyv_##BSFX m, npyv_##SFX a, npyv_##SFX b, npyv_##SFX c) \ + { \ + npyv_##SFX sub = npyv_sub_##SFX(a, b); \ + return npyv_select_##SFX(m, sub, c); \ + } + +NPYV_IMPL_EMULATE_MASK_ADDSUB(u8, b8) +NPYV_IMPL_EMULATE_MASK_ADDSUB(s8, b8) +NPYV_IMPL_EMULATE_MASK_ADDSUB(u16, b16) +NPYV_IMPL_EMULATE_MASK_ADDSUB(s16, b16) +NPYV_IMPL_EMULATE_MASK_ADDSUB(u32, b32) +NPYV_IMPL_EMULATE_MASK_ADDSUB(s32, b32) +NPYV_IMPL_EMULATE_MASK_ADDSUB(u64, b64) +NPYV_IMPL_EMULATE_MASK_ADDSUB(s64, b64) +NPYV_IMPL_EMULATE_MASK_ADDSUB(f32, b32) +#if NPY_SIMD_F64 + NPYV_IMPL_EMULATE_MASK_ADDSUB(f64, b64) +#endif + +#endif // _NPY_SIMD_EMULATE_MASKOP_H diff --git a/numpy/core/src/common/simd/intdiv.h b/numpy/core/src/common/simd/intdiv.h new file mode 100644 index 000000000000..f6ea9abf254e --- /dev/null +++ b/numpy/core/src/common/simd/intdiv.h @@ -0,0 +1,475 @@ +/** + * This header implements `npyv_divisor_*` intrinsics used for computing the parameters + * of fast integer division, while division intrinsics `npyv_divc_*` are defined in + * {extension}/arithmetic.h. + */ +#ifndef NPY_SIMD + #error "Not a standalone header, use simd/simd.h instead" +#endif +#ifndef _NPY_SIMD_INTDIV_H +#define _NPY_SIMD_INTDIV_H +/********************************************************************************** + ** Integer division + ********************************************************************************** + * Almost all architecture (except Power10) doesn't support integer vector division, + * also the cost of scalar division in architectures like x86 is too high it can take + * 30 to 40 cycles on modern chips and up to 100 on old ones. + * + * Therefore we are using division by multiplying with precomputed reciprocal technique, + * the method that been used in this implementation is based on T. Granlund and P. L. Montgomery + * “Division by invariant integers using multiplication(see [Figure 4.1, 5.1] + * http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.1.2556) + * + * It shows a good impact for all architectures especially on X86, + * however computing divisor parameters is kind of expensive so this implementation + * should only works when divisor is a scalar and used multiple of times. + * + * The division process is separated into two intrinsics for each data type + * + * 1- npyv_{dtype}x3 npyv_divisor_{dtype} ({dtype} divisor); + * For computing the divisor parameters (multiplier + shifters + sign of divisor(signed only)) + * + * 2- npyv_{dtype} npyv_divisor_{dtype} (npyv_{dtype} dividend, npyv_{dtype}x3 divisor_parms); + * For performing the final division. + * + ** For example: + * int vstep = npyv_nlanes_s32; // number of lanes + * int x = 0x6e70; + * npyv_s32x3 divisor = npyv_divisor_s32(x); // init divisor params + * for (; len >= vstep; src += vstep, dst += vstep, len -= vstep) { + * npyv_s32 a = npyv_load_s32(*src); // load s32 vector from memory + * a = npyv_divc_s32(a, divisor); // divide all elements by x + * npyv_store_s32(dst, a); // store s32 vector into memroy + * } + * + ** NOTES: + * - For 64-bit division on Aarch64 and IBM/Power, we fall-back to the scalar division + * since emulating multiply-high is expensive and both architectures have very fast dividers. + * + ** TODO: + * - Add support for Power10(VSX4) + * + *************************************************************** + ** Figure 4.1: Unsigned division by run–time invariant divisor + *************************************************************** + * Initialization (given uword d with 1 ≤ d < 2^N): + * int l = ceil(log2(d)); + * uword m = 2^N * (2^l− d) / d + 1; + * int sh1 = min(l, 1); + * int sh2 = max(l − 1, 0); + * + * For q = FLOOR(a/d), all uword: + * uword t1 = MULUH(m, a); + * q = SRL(t1 + SRL(a − t1, sh1), sh2); + * + ************************************************************************************ + ** Figure 5.1: Signed division by run–time invariant divisor, rounded towards zero + ************************************************************************************ + * Initialization (given constant sword d with d !=0): + * int l = max(ceil(log2(abs(d))), 1); + * udword m0 = 1 + (2^(N+l-1)) / abs(d); + * sword m = m0 − 2^N; + * sword dsign = XSIGN(d); + * int sh = l − 1; + * + * For q = TRUNC(a/d), all sword: + * sword q0 = a + MULSH(m, a); + * q0 = SRA(q0, sh) − XSIGN(a); + * q = EOR(q0, dsign) − dsign; + */ +/** + * bit-scan reverse for non-zeros. returns the index of the highest set bit. + * equivalent to floor(log2(a)) + */ +#ifdef _MSC_VER + #include // _BitScanReverse +#endif +NPY_FINLINE unsigned npyv__bitscan_revnz_u32(npy_uint32 a) +{ + assert(a > 0); // due to use __builtin_clz + unsigned r; +#if defined(NPY_HAVE_SSE2) && defined(_MSC_VER) + unsigned long rl; + (void)_BitScanReverse(&rl, (unsigned long)a); + r = (unsigned)rl; +#elif defined(NPY_HAVE_SSE2) && (defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER)) + __asm__("bsr %1, %0" : "=r" (r) : "r"(a)); +#elif defined(__GNUC__) || defined(__clang__) + r = 31 - __builtin_clz(a); // performs on arm -> clz, ppc -> cntlzw +#else + r = 0; + while (a >>= 1) { + r++; + } +#endif + return r; +} +NPY_FINLINE unsigned npyv__bitscan_revnz_u64(npy_uint64 a) +{ + assert(a > 0); // due to use __builtin_clzll +#if defined(_M_AMD64) && defined(_MSC_VER) + unsigned long rl; + (void)_BitScanReverse64(&rl, a); + return (unsigned)rl; +#elif defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER)) + npy_uint64 r; + __asm__("bsrq %1, %0" : "=r"(r) : "r"(a)); + return (unsigned)r; +#elif defined(__GNUC__) || defined(__clang__) + return 63 - __builtin_clzll(a); +#else + npy_uint64 a_hi = a >> 32; + if (a_hi == 0) { + return npyv__bitscan_revnz_u32((npy_uint32)a); + } + return 32 + npyv__bitscan_revnz_u32((npy_uint32)a_hi); +#endif +} +/** + * Divides 128-bit unsigned integer by a 64-bit when the lower + * 64-bit of the dividend is zero. + * + * This function is needed to calculate the multiplier of 64-bit integer division + * see npyv_divisor_u64/npyv_divisor_s64. + */ +NPY_FINLINE npy_uint64 npyv__divh128_u64(npy_uint64 high, npy_uint64 divisor) +{ + assert(divisor > 1); + npy_uint64 quotient; +#if defined(_M_X64) && defined(_MSC_VER) && _MSC_VER >= 1920 + npy_uint64 remainder; + quotient = _udiv128(high, 0, divisor, &remainder); + (void)remainder; +#elif defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER)) + __asm__("divq %[d]" : "=a"(quotient) : [d] "r"(divisor), "a"(0), "d"(high)); +#elif defined(__SIZEOF_INT128__) + quotient = (npy_uint64)((((__uint128_t)high) << 64) / divisor); +#else + /** + * Minified version based on Donald Knuth’s Algorithm D (Division of nonnegative integers), + * and Generic implementation in Hacker’s Delight. + * + * See https://skanthak.homepage.t-online.de/division.html + * with respect to the license of the Hacker's Delight book + * (https://web.archive.org/web/20190408122508/http://www.hackersdelight.org/permissions.htm) + */ + // shift amount for normalize + unsigned ldz = 63 - npyv__bitscan_revnz_u64(divisor); + // normalize divisor + divisor <<= ldz; + high <<= ldz; + // break divisor up into two 32-bit digits + npy_uint32 divisor_hi = divisor >> 32; + npy_uint32 divisor_lo = divisor & 0xFFFFFFFF; + // compute high quotient digit + npy_uint32 quotient_hi = (npy_uint32)(high / divisor_hi); + npy_uint64 remainder = high - divisor_hi * quotient_hi; + npy_uint64 base32 = 1ULL << 32; + while (quotient_hi >= base32 || quotient_hi*divisor_lo > base32*remainder) { + remainder += --divisor_hi; + if (remainder >= base32) { + break; + } + } + // compute dividend digit pairs + npy_uint64 dividend_pairs = base32*high - divisor*quotient_hi; + // compute second quotient digit for lower zeros + npy_uint32 quotient_lo = (npy_uint32)(dividend_pairs / divisor_hi); + quotient = base32*quotient_hi + quotient_lo; +#endif + return quotient; +} +// Initializing divisor parameters for unsigned 8-bit division +NPY_FINLINE npyv_u8x3 npyv_divisor_u8(npy_uint8 d) +{ + unsigned l, l2, sh1, sh2, m; + switch (d) { + case 0: // LCOV_EXCL_LINE + // for potential divide by zero, On x86 GCC inserts `ud2` instruction + // instead of letting the HW/CPU trap it which leads to illegal instruction exception. + // 'volatile' should suppress this behavior and allow us to raise HW/CPU + // arithmetic exception. + m = sh1 = sh2 = 1 / ((npy_uint8 volatile *)&d)[0]; + break; + case 1: + m = 1; sh1 = sh2 = 0; + break; + case 2: + m = 1; sh1 = 1; sh2 = 0; + break; + default: + l = npyv__bitscan_revnz_u32(d - 1) + 1; // ceil(log2(d)) + l2 = (npy_uint8)(1 << l); // 2^l, overflow to 0 if l = 8 + m = ((l2 - d) << 8) / d + 1; // multiplier + sh1 = 1; sh2 = l - 1; // shift counts + } + npyv_u8x3 divisor; +#ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[0] = npyv_setall_u16(m); + divisor.val[1] = npyv_set_u8(sh1); + divisor.val[2] = npyv_set_u8(sh2); +#elif defined(NPY_HAVE_VSX2) + divisor.val[0] = npyv_setall_u8(m); + divisor.val[1] = npyv_setall_u8(sh1); + divisor.val[2] = npyv_setall_u8(sh2); +#elif defined(NPY_HAVE_NEON) + divisor.val[0] = npyv_setall_u8(m); + divisor.val[1] = npyv_reinterpret_u8_s8(npyv_setall_s8(-sh1)); + divisor.val[2] = npyv_reinterpret_u8_s8(npyv_setall_s8(-sh2)); +#else + #error "please initialize the shifting operand for the new architecture" +#endif + return divisor; +} +// Initializing divisor parameters for signed 8-bit division +NPY_FINLINE npyv_s16x3 npyv_divisor_s16(npy_int16 d); +NPY_FINLINE npyv_s8x3 npyv_divisor_s8(npy_int8 d) +{ +#ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + npyv_s16x3 p = npyv_divisor_s16(d); + npyv_s8x3 r; + r.val[0] = npyv_reinterpret_s8_s16(p.val[0]); + r.val[1] = npyv_reinterpret_s8_s16(p.val[1]); + r.val[2] = npyv_reinterpret_s8_s16(p.val[2]); + return r; +#else + int d1 = abs(d); + int sh, m; + if (d1 > 1) { + sh = (int)npyv__bitscan_revnz_u32(d1-1); // ceil(log2(abs(d))) - 1 + m = (1 << (8 + sh)) / d1 + 1; // multiplier + } + else if (d1 == 1) { + sh = 0; m = 1; + } + else { + // raise arithmetic exception for d == 0 + sh = m = 1 / ((npy_int8 volatile *)&d)[0]; // LCOV_EXCL_LINE + } + npyv_s8x3 divisor; + divisor.val[0] = npyv_setall_s8(m); + divisor.val[2] = npyv_setall_s8(d < 0 ? -1 : 0); + #ifdef NPY_HAVE_VSX2 + divisor.val[1] = npyv_setall_s8(sh); + #elif defined(NPY_HAVE_NEON) + divisor.val[1] = npyv_setall_s8(-sh); + #else + #error "please initialize the shifting operand for the new architecture" + #endif + return divisor; +#endif +} +// Initializing divisor parameters for unsigned 16-bit division +NPY_FINLINE npyv_u16x3 npyv_divisor_u16(npy_uint16 d) +{ + unsigned l, l2, sh1, sh2, m; + switch (d) { + case 0: // LCOV_EXCL_LINE + // raise arithmetic exception for d == 0 + m = sh1 = sh2 = 1 / ((npy_uint16 volatile *)&d)[0]; + break; + case 1: + m = 1; sh1 = sh2 = 0; + break; + case 2: + m = 1; sh1 = 1; sh2 = 0; + break; + default: + l = npyv__bitscan_revnz_u32(d - 1) + 1; // ceil(log2(d)) + l2 = (npy_uint16)(1 << l); // 2^l, overflow to 0 if l = 16 + m = ((l2 - d) << 16) / d + 1; // multiplier + sh1 = 1; sh2 = l - 1; // shift counts + } + npyv_u16x3 divisor; + divisor.val[0] = npyv_setall_u16(m); +#ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[1] = npyv_set_u16(sh1); + divisor.val[2] = npyv_set_u16(sh2); +#elif defined(NPY_HAVE_VSX2) + divisor.val[1] = npyv_setall_u16(sh1); + divisor.val[2] = npyv_setall_u16(sh2); +#elif defined(NPY_HAVE_NEON) + divisor.val[1] = npyv_reinterpret_u16_s16(npyv_setall_s16(-sh1)); + divisor.val[2] = npyv_reinterpret_u16_s16(npyv_setall_s16(-sh2)); +#else + #error "please initialize the shifting operand for the new architecture" +#endif + return divisor; +} +// Initializing divisor parameters for signed 16-bit division +NPY_FINLINE npyv_s16x3 npyv_divisor_s16(npy_int16 d) +{ + int d1 = abs(d); + int sh, m; + if (d1 > 1) { + sh = (int)npyv__bitscan_revnz_u32(d1 - 1); // ceil(log2(abs(d))) - 1 + m = (1 << (16 + sh)) / d1 + 1; // multiplier + } + else if (d1 == 1) { + sh = 0; m = 1; + } + else { + // raise arithmetic exception for d == 0 + sh = m = 1 / ((npy_int16 volatile *)&d)[0]; // LCOV_EXCL_LINE + } + npyv_s16x3 divisor; + divisor.val[0] = npyv_setall_s16(m); + divisor.val[2] = npyv_setall_s16(d < 0 ? -1 : 0); // sign of divisor +#ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[1] = npyv_set_s16(sh); +#elif defined(NPY_HAVE_VSX2) + divisor.val[1] = npyv_setall_s16(sh); +#elif defined(NPY_HAVE_NEON) + divisor.val[1] = npyv_setall_s16(-sh); +#else + #error "please initialize the shifting operand for the new architecture" +#endif + return divisor; +} +// Initializing divisor parameters for unsigned 32-bit division +NPY_FINLINE npyv_u32x3 npyv_divisor_u32(npy_uint32 d) +{ + npy_uint32 l, l2, sh1, sh2, m; + switch (d) { + case 0: // LCOV_EXCL_LINE + // raise arithmetic exception for d == 0 + m = sh1 = sh2 = 1 / ((npy_uint32 volatile *)&d)[0]; // LCOV_EXCL_LINE + break; + case 1: + m = 1; sh1 = sh2 = 0; + break; + case 2: + m = 1; sh1 = 1; sh2 = 0; + break; + default: + l = npyv__bitscan_revnz_u32(d - 1) + 1; // ceil(log2(d)) + l2 = (npy_uint32)(1ULL << l); // 2^l, overflow to 0 if l = 32 + m = ((npy_uint64)(l2 - d) << 32) / d + 1; // multiplier + sh1 = 1; sh2 = l - 1; // shift counts + } + npyv_u32x3 divisor; + divisor.val[0] = npyv_setall_u32(m); +#ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[1] = npyv_set_u32(sh1); + divisor.val[2] = npyv_set_u32(sh2); +#elif defined(NPY_HAVE_VSX2) + divisor.val[1] = npyv_setall_u32(sh1); + divisor.val[2] = npyv_setall_u32(sh2); +#elif defined(NPY_HAVE_NEON) + divisor.val[1] = npyv_reinterpret_u32_s32(npyv_setall_s32(-sh1)); + divisor.val[2] = npyv_reinterpret_u32_s32(npyv_setall_s32(-sh2)); +#else + #error "please initialize the shifting operand for the new architecture" +#endif + return divisor; +} +// Initializing divisor parameters for signed 32-bit division +NPY_FINLINE npyv_s32x3 npyv_divisor_s32(npy_int32 d) +{ + npy_int32 d1 = abs(d); + npy_int32 sh, m; + // Handel abs overflow + if ((npy_uint32)d == 0x80000000U) { + m = 0x80000001; + sh = 30; + } + else if (d1 > 1) { + sh = npyv__bitscan_revnz_u32(d1 - 1); // ceil(log2(abs(d))) - 1 + m = (1ULL << (32 + sh)) / d1 + 1; // multiplier + } + else if (d1 == 1) { + sh = 0; m = 1; + } + else { + // raise arithmetic exception for d == 0 + sh = m = 1 / ((npy_int32 volatile *)&d)[0]; // LCOV_EXCL_LINE + } + npyv_s32x3 divisor; + divisor.val[0] = npyv_setall_s32(m); + divisor.val[2] = npyv_setall_s32(d < 0 ? -1 : 0); // sign of divisor +#ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[1] = npyv_set_s32(sh); +#elif defined(NPY_HAVE_VSX2) + divisor.val[1] = npyv_setall_s32(sh); +#elif defined(NPY_HAVE_NEON) + divisor.val[1] = npyv_setall_s32(-sh); +#else + #error "please initialize the shifting operand for the new architecture" +#endif + return divisor; +} +// Initializing divisor parameters for unsigned 64-bit division +NPY_FINLINE npyv_u64x3 npyv_divisor_u64(npy_uint64 d) +{ + npyv_u64x3 divisor; +#if defined(NPY_HAVE_VSX2) || defined(NPY_HAVE_NEON) + divisor.val[0] = npyv_setall_u64(d); +#else + npy_uint64 l, l2, sh1, sh2, m; + switch (d) { + case 0: // LCOV_EXCL_LINE + // raise arithmetic exception for d == 0 + m = sh1 = sh2 = 1 / ((npy_uint64 volatile *)&d)[0]; // LCOV_EXCL_LINE + break; + case 1: + m = 1; sh1 = sh2 = 0; + break; + case 2: + m = 1; sh1 = 1; sh2 = 0; + break; + default: + l = npyv__bitscan_revnz_u64(d - 1) + 1; // ceil(log2(d)) + l2 = l < 64 ? 1ULL << l : 0; // 2^l + m = npyv__divh128_u64(l2 - d, d) + 1; // multiplier + sh1 = 1; sh2 = l - 1; // shift counts + } + divisor.val[0] = npyv_setall_u64(m); + #ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[1] = npyv_set_u64(sh1); + divisor.val[2] = npyv_set_u64(sh2); + #else + #error "please initialize the shifting operand for the new architecture" + #endif +#endif + return divisor; +} +// Initializing divisor parameters for signed 64-bit division +NPY_FINLINE npyv_s64x3 npyv_divisor_s64(npy_int64 d) +{ + npyv_s64x3 divisor; +#if defined(NPY_HAVE_VSX2) || defined(NPY_HAVE_NEON) + divisor.val[0] = npyv_setall_s64(d); + divisor.val[1] = npyv_cvt_s64_b64( + npyv_cmpeq_s64(npyv_setall_s64(-1), divisor.val[0]) + ); +#else + npy_int64 d1 = llabs(d); + npy_int64 sh, m; + // Handel abs overflow + if ((npy_uint64)d == 0x8000000000000000ULL) { + m = 0x8000000000000001LL; + sh = 62; + } + else if (d1 > 1) { + sh = npyv__bitscan_revnz_u64(d1 - 1); // ceil(log2(abs(d))) - 1 + m = npyv__divh128_u64(1ULL << sh, d1) + 1; // multiplier + } + else if (d1 == 1) { + sh = 0; m = 1; + } + else { + // raise arithmetic exception for d == 0 + sh = m = 1 / ((npy_int64 volatile *)&d)[0]; // LCOV_EXCL_LINE + } + divisor.val[0] = npyv_setall_s64(m); + divisor.val[2] = npyv_setall_s64(d < 0 ? -1 : 0); // sign of divisor + #ifdef NPY_HAVE_SSE2 // SSE/AVX2/AVX512 + divisor.val[1] = npyv_set_s64(sh); + #else + #error "please initialize the shifting operand for the new architecture" + #endif +#endif + return divisor; +} + +#endif // _NPY_SIMD_INTDIV_H diff --git a/numpy/core/src/common/simd/neon/arithmetic.h b/numpy/core/src/common/simd/neon/arithmetic.h index 87e00d5d14e4..00994806df68 100644 --- a/numpy/core/src/common/simd/neon/arithmetic.h +++ b/numpy/core/src/common/simd/neon/arithmetic.h @@ -60,6 +60,154 @@ #define npyv_mul_f32 vmulq_f32 #define npyv_mul_f64 vmulq_f64 +/*************************** + * Integer Division + ***************************/ +// See simd/intdiv.h for more clarification +// divide each unsigned 8-bit element by a precomputed divisor +NPY_FINLINE npyv_u8 npyv_divc_u8(npyv_u8 a, const npyv_u8x3 divisor) +{ + const uint8x8_t mulc_lo = vget_low_u8(divisor.val[0]); + // high part of unsigned multiplication + uint16x8_t mull_lo = vmull_u8(vget_low_u8(a), mulc_lo); +#if NPY_SIMD_F64 + uint16x8_t mull_hi = vmull_high_u8(a, divisor.val[0]); + // get the high unsigned bytes + uint8x16_t mulhi = vuzp2q_u8(vreinterpretq_u8_u16(mull_lo), vreinterpretq_u8_u16(mull_hi)); +#else + const uint8x8_t mulc_hi = vget_high_u8(divisor.val[0]); + uint16x8_t mull_hi = vmull_u8(vget_high_u8(a), mulc_hi); + uint8x16_t mulhi = vuzpq_u8(vreinterpretq_u8_u16(mull_lo), vreinterpretq_u8_u16(mull_hi)).val[1]; +#endif + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + uint8x16_t q = vsubq_u8(a, mulhi); + q = vshlq_u8(q, vreinterpretq_s8_u8(divisor.val[1])); + q = vaddq_u8(mulhi, q); + q = vshlq_u8(q, vreinterpretq_s8_u8(divisor.val[2])); + return q; +} +// divide each signed 8-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s8 npyv_divc_s8(npyv_s8 a, const npyv_s8x3 divisor) +{ + const int8x8_t mulc_lo = vget_low_s8(divisor.val[0]); + // high part of signed multiplication + int16x8_t mull_lo = vmull_s8(vget_low_s8(a), mulc_lo); +#if NPY_SIMD_F64 + int16x8_t mull_hi = vmull_high_s8(a, divisor.val[0]); + // get the high unsigned bytes + int8x16_t mulhi = vuzp2q_s8(vreinterpretq_s8_s16(mull_lo), vreinterpretq_s8_s16(mull_hi)); +#else + const int8x8_t mulc_hi = vget_high_s8(divisor.val[0]); + int16x8_t mull_hi = vmull_s8(vget_high_s8(a), mulc_hi); + int8x16_t mulhi = vuzpq_s8(vreinterpretq_s8_s16(mull_lo), vreinterpretq_s8_s16(mull_hi)).val[1]; +#endif + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + int8x16_t q = vshlq_s8(vaddq_s8(a, mulhi), divisor.val[1]); + q = vsubq_s8(q, vshrq_n_s8(a, 7)); + q = vsubq_s8(veorq_s8(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 16-bit element by a precomputed divisor +NPY_FINLINE npyv_u16 npyv_divc_u16(npyv_u16 a, const npyv_u16x3 divisor) +{ + const uint16x4_t mulc_lo = vget_low_u16(divisor.val[0]); + // high part of unsigned multiplication + uint32x4_t mull_lo = vmull_u16(vget_low_u16(a), mulc_lo); +#if NPY_SIMD_F64 + uint32x4_t mull_hi = vmull_high_u16(a, divisor.val[0]); + // get the high unsigned bytes + uint16x8_t mulhi = vuzp2q_u16(vreinterpretq_u16_u32(mull_lo), vreinterpretq_u16_u32(mull_hi)); +#else + const uint16x4_t mulc_hi = vget_high_u16(divisor.val[0]); + uint32x4_t mull_hi = vmull_u16(vget_high_u16(a), mulc_hi); + uint16x8_t mulhi = vuzpq_u16(vreinterpretq_u16_u32(mull_lo), vreinterpretq_u16_u32(mull_hi)).val[1]; +#endif + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + uint16x8_t q = vsubq_u16(a, mulhi); + q = vshlq_u16(q, vreinterpretq_s16_u16(divisor.val[1])); + q = vaddq_u16(mulhi, q); + q = vshlq_u16(q, vreinterpretq_s16_u16(divisor.val[2])); + return q; +} +// divide each signed 16-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor) +{ + const int16x4_t mulc_lo = vget_low_s16(divisor.val[0]); + // high part of signed multiplication + int32x4_t mull_lo = vmull_s16(vget_low_s16(a), mulc_lo); +#if NPY_SIMD_F64 + int32x4_t mull_hi = vmull_high_s16(a, divisor.val[0]); + // get the high unsigned bytes + int16x8_t mulhi = vuzp2q_s16(vreinterpretq_s16_s32(mull_lo), vreinterpretq_s16_s32(mull_hi)); +#else + const int16x4_t mulc_hi = vget_high_s16(divisor.val[0]); + int32x4_t mull_hi = vmull_s16(vget_high_s16(a), mulc_hi); + int16x8_t mulhi = vuzpq_s16(vreinterpretq_s16_s32(mull_lo), vreinterpretq_s16_s32(mull_hi)).val[1]; +#endif + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + int16x8_t q = vshlq_s16(vaddq_s16(a, mulhi), divisor.val[1]); + q = vsubq_s16(q, vshrq_n_s16(a, 15)); + q = vsubq_s16(veorq_s16(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 32-bit element by a precomputed divisor +NPY_FINLINE npyv_u32 npyv_divc_u32(npyv_u32 a, const npyv_u32x3 divisor) +{ + const uint32x2_t mulc_lo = vget_low_u32(divisor.val[0]); + // high part of unsigned multiplication + uint64x2_t mull_lo = vmull_u32(vget_low_u32(a), mulc_lo); +#if NPY_SIMD_F64 + uint64x2_t mull_hi = vmull_high_u32(a, divisor.val[0]); + // get the high unsigned bytes + uint32x4_t mulhi = vuzp2q_u32(vreinterpretq_u32_u64(mull_lo), vreinterpretq_u32_u64(mull_hi)); +#else + const uint32x2_t mulc_hi = vget_high_u32(divisor.val[0]); + uint64x2_t mull_hi = vmull_u32(vget_high_u32(a), mulc_hi); + uint32x4_t mulhi = vuzpq_u32(vreinterpretq_u32_u64(mull_lo), vreinterpretq_u32_u64(mull_hi)).val[1]; +#endif + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + uint32x4_t q = vsubq_u32(a, mulhi); + q = vshlq_u32(q, vreinterpretq_s32_u32(divisor.val[1])); + q = vaddq_u32(mulhi, q); + q = vshlq_u32(q, vreinterpretq_s32_u32(divisor.val[2])); + return q; +} +// divide each signed 32-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s32 npyv_divc_s32(npyv_s32 a, const npyv_s32x3 divisor) +{ + const int32x2_t mulc_lo = vget_low_s32(divisor.val[0]); + // high part of signed multiplication + int64x2_t mull_lo = vmull_s32(vget_low_s32(a), mulc_lo); +#if NPY_SIMD_F64 + int64x2_t mull_hi = vmull_high_s32(a, divisor.val[0]); + // get the high unsigned bytes + int32x4_t mulhi = vuzp2q_s32(vreinterpretq_s32_s64(mull_lo), vreinterpretq_s32_s64(mull_hi)); +#else + const int32x2_t mulc_hi = vget_high_s32(divisor.val[0]); + int64x2_t mull_hi = vmull_s32(vget_high_s32(a), mulc_hi); + int32x4_t mulhi = vuzpq_s32(vreinterpretq_s32_s64(mull_lo), vreinterpretq_s32_s64(mull_hi)).val[1]; +#endif + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + int32x4_t q = vshlq_s32(vaddq_s32(a, mulhi), divisor.val[1]); + q = vsubq_s32(q, vshrq_n_s32(a, 31)); + q = vsubq_s32(veorq_s32(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 64-bit element by a divisor +NPY_FINLINE npyv_u64 npyv_divc_u64(npyv_u64 a, const npyv_u64x3 divisor) +{ + const uint64_t d = vgetq_lane_u64(divisor.val[0], 0); + return npyv_set_u64(vgetq_lane_u64(a, 0) / d, vgetq_lane_u64(a, 1) / d); +} +// returns the high 64 bits of signed 64-bit multiplication +NPY_FINLINE npyv_s64 npyv_divc_s64(npyv_s64 a, const npyv_s64x3 divisor) +{ + const int64_t d = vgetq_lane_s64(divisor.val[0], 0); + return npyv_set_s64(vgetq_lane_s64(a, 0) / d, vgetq_lane_s64(a, 1) / d); +} /*************************** * Division ***************************/ @@ -131,11 +279,27 @@ { return vfmsq_f64(vnegq_f64(c), a, b); } #endif // NPY_SIMD_F64 -// Horizontal add: Calculates the sum of all vector elements. +/*************************** + * Summation + ***************************/ +// reduce sum across vector #if NPY_SIMD_F64 + #define npyv_sum_u32 vaddvq_u32 + #define npyv_sum_u64 vaddvq_u64 #define npyv_sum_f32 vaddvq_f32 #define npyv_sum_f64 vaddvq_f64 #else + NPY_FINLINE npy_uint64 npyv_sum_u64(npyv_u64 a) + { + return vget_lane_u64(vadd_u64(vget_low_u64(a), vget_high_u64(a)),0); + } + + NPY_FINLINE npy_uint32 npyv_sum_u32(npyv_u32 a) + { + uint32x2_t a0 = vpadd_u32(vget_low_u32(a), vget_high_u32(a)); + return (unsigned)vget_lane_u32(vpadd_u32(a0, vget_high_u32(a)),0); + } + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { float32x2_t r = vadd_f32(vget_high_f32(a), vget_low_f32(a)); @@ -143,4 +307,24 @@ } #endif +// expand the source vector and performs sum reduce +#if NPY_SIMD_F64 + #define npyv_sumup_u8 vaddlvq_u8 + #define npyv_sumup_u16 vaddlvq_u16 +#else + NPY_FINLINE npy_uint16 npyv_sumup_u8(npyv_u8 a) + { + uint32x4_t t0 = vpaddlq_u16(vpaddlq_u8(a)); + uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); + return vget_lane_u32(vpadd_u32(t1, t1), 0); + } + + NPY_FINLINE npy_uint32 npyv_sumup_u16(npyv_u16 a) + { + uint32x4_t t0 = vpaddlq_u16(a); + uint32x2_t t1 = vpadd_u32(vget_low_u32(t0), vget_high_u32(t0)); + return vget_lane_u32(vpadd_u32(t1, t1), 0); + } +#endif + #endif // _NPY_SIMD_NEON_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/neon/conversion.h b/numpy/core/src/common/simd/neon/conversion.h index b286931d1404..7487559d1c30 100644 --- a/numpy/core/src/common/simd/neon/conversion.h +++ b/numpy/core/src/common/simd/neon/conversion.h @@ -7,26 +7,103 @@ // convert boolean vectors to integer vectors #define npyv_cvt_u8_b8(A) A -#define npyv_cvt_s8_b8(A) vreinterpretq_s8_u8(A) +#define npyv_cvt_s8_b8 vreinterpretq_s8_u8 #define npyv_cvt_u16_b16(A) A -#define npyv_cvt_s16_b16(A) vreinterpretq_s16_u16(A) +#define npyv_cvt_s16_b16 vreinterpretq_s16_u16 #define npyv_cvt_u32_b32(A) A -#define npyv_cvt_s32_b32(A) vreinterpretq_s32_u32(A) +#define npyv_cvt_s32_b32 vreinterpretq_s32_u32 #define npyv_cvt_u64_b64(A) A -#define npyv_cvt_s64_b64(A) vreinterpretq_s64_u64(A) -#define npyv_cvt_f32_b32(A) vreinterpretq_f32_u32(A) -#define npyv_cvt_f64_b64(A) vreinterpretq_f64_u64(A) +#define npyv_cvt_s64_b64 vreinterpretq_s64_u64 +#define npyv_cvt_f32_b32 vreinterpretq_f32_u32 +#define npyv_cvt_f64_b64 vreinterpretq_f64_u64 // convert integer vectors to boolean vectors #define npyv_cvt_b8_u8(BL) BL -#define npyv_cvt_b8_s8(BL) vreinterpretq_u8_s8(BL) +#define npyv_cvt_b8_s8 vreinterpretq_u8_s8 #define npyv_cvt_b16_u16(BL) BL -#define npyv_cvt_b16_s16(BL) vreinterpretq_u16_s16(BL) +#define npyv_cvt_b16_s16 vreinterpretq_u16_s16 #define npyv_cvt_b32_u32(BL) BL -#define npyv_cvt_b32_s32(BL) vreinterpretq_u32_s32(BL) +#define npyv_cvt_b32_s32 vreinterpretq_u32_s32 #define npyv_cvt_b64_u64(BL) BL -#define npyv_cvt_b64_s64(BL) vreinterpretq_u64_s64(BL) -#define npyv_cvt_b32_f32(BL) vreinterpretq_u32_f32(BL) -#define npyv_cvt_b64_f64(BL) vreinterpretq_u64_f64(BL) +#define npyv_cvt_b64_s64 vreinterpretq_u64_s64 +#define npyv_cvt_b32_f32 vreinterpretq_u32_f32 +#define npyv_cvt_b64_f64 vreinterpretq_u64_f64 + +// convert boolean vector to integer bitfield +NPY_FINLINE npy_uint64 npyv_tobits_b8(npyv_b8 a) +{ + const npyv_u8 scale = npyv_set_u8(1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128); + npyv_u8 seq_scale = vandq_u8(a, scale); +#if NPY_SIMD_F64 + npy_uint8 sumlo = vaddv_u8(vget_low_u8(seq_scale)); + npy_uint8 sumhi = vaddv_u8(vget_high_u8(seq_scale)); + return sumlo + ((int)sumhi << 8); +#else + npyv_u64 sumh = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(seq_scale))); + return vgetq_lane_u64(sumh, 0) + ((int)vgetq_lane_u64(sumh, 1) << 8); +#endif +} +NPY_FINLINE npy_uint64 npyv_tobits_b16(npyv_b16 a) +{ + const npyv_u16 scale = npyv_set_u16(1, 2, 4, 8, 16, 32, 64, 128); + npyv_u16 seq_scale = vandq_u16(a, scale); +#if NPY_SIMD_F64 + return vaddvq_u16(seq_scale); +#else + npyv_u64 sumh = vpaddlq_u32(vpaddlq_u16(seq_scale)); + return vgetq_lane_u64(sumh, 0) + vgetq_lane_u64(sumh, 1); +#endif +} +NPY_FINLINE npy_uint64 npyv_tobits_b32(npyv_b32 a) +{ + const npyv_u32 scale = npyv_set_u32(1, 2, 4, 8); + npyv_u32 seq_scale = vandq_u32(a, scale); +#if NPY_SIMD_F64 + return vaddvq_u32(seq_scale); +#else + npyv_u64 sumh = vpaddlq_u32(seq_scale); + return vgetq_lane_u64(sumh, 0) + vgetq_lane_u64(sumh, 1); +#endif +} +NPY_FINLINE npy_uint64 npyv_tobits_b64(npyv_b64 a) +{ + npyv_u64 bit = vshrq_n_u64(a, 63); + return vgetq_lane_u64(bit, 0) | ((int)vgetq_lane_u64(bit, 1) << 1); +} + +//expand +NPY_FINLINE npyv_u16x2 npyv_expand_u16_u8(npyv_u8 data) { + npyv_u16x2 r; + r.val[0] = vmovl_u8(vget_low_u8(data)); + r.val[1] = vmovl_u8(vget_high_u8(data)); + return r; +} + +NPY_FINLINE npyv_u32x2 npyv_expand_u32_u16(npyv_u16 data) { + npyv_u32x2 r; + r.val[0] = vmovl_u16(vget_low_u16(data)); + r.val[1] = vmovl_u16(vget_high_u16(data)); + return r; +} + +// round to nearest integer +#if NPY_SIMD_F64 + #define npyv_round_s32_f32 vcvtnq_s32_f32 + NPY_FINLINE npyv_s32 npyv_round_s32_f64(npyv_f64 a, npyv_f64 b) + { + npyv_s64 lo = vcvtnq_s64_f64(a), hi = vcvtnq_s64_f64(b); + return vcombine_s32(vmovn_s64(lo), vmovn_s64(hi)); + } +#else + NPY_FINLINE npyv_s32 npyv_round_s32_f32(npyv_f32 a) + { + // halves will be rounded up. it's very costly + // to obey IEEE standard on arm7. tests should pass +-1 difference + const npyv_u32 sign = vdupq_n_u32(0x80000000); + const npyv_f32 half = vdupq_n_f32(0.5f); + npyv_f32 sign_half = vbslq_f32(sign, a, half); + return vcvtq_s32_f32(vaddq_f32(a, sign_half)); + } +#endif #endif // _NPY_SIMD_NEON_CVT_H diff --git a/numpy/core/src/common/simd/neon/math.h b/numpy/core/src/common/simd/neon/math.h index a2bbdf2a5d30..ced82d1de65f 100644 --- a/numpy/core/src/common/simd/neon/math.h +++ b/numpy/core/src/common/simd/neon/math.h @@ -83,4 +83,74 @@ NPY_FINLINE npyv_f32 npyv_recip_f32(npyv_f32 a) } #endif // NPY_SIMD_F64 -#endif // _NPY_SIMD_SSE_MATH_H +// Maximum, natively mapping with no guarantees to handle NaN. +#define npyv_max_f32 vmaxq_f32 +#define npyv_max_f64 vmaxq_f64 +// Maximum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +#ifdef NPY_HAVE_ASIMD + #define npyv_maxp_f32 vmaxnmq_f32 +#else + NPY_FINLINE npyv_f32 npyv_maxp_f32(npyv_f32 a, npyv_f32 b) + { + npyv_u32 nn_a = vceqq_f32(a, a); + npyv_u32 nn_b = vceqq_f32(b, b); + return vmaxq_f32(vbslq_f32(nn_a, a, b), vbslq_f32(nn_b, b, a)); + } +#endif +#if NPY_SIMD_F64 + #define npyv_maxp_f64 vmaxnmq_f64 +#endif // NPY_SIMD_F64 +// Maximum, integer operations +#define npyv_max_u8 vmaxq_u8 +#define npyv_max_s8 vmaxq_s8 +#define npyv_max_u16 vmaxq_u16 +#define npyv_max_s16 vmaxq_s16 +#define npyv_max_u32 vmaxq_u32 +#define npyv_max_s32 vmaxq_s32 +NPY_FINLINE npyv_u64 npyv_max_u64(npyv_u64 a, npyv_u64 b) +{ + return vbslq_u64(npyv_cmpgt_u64(a, b), a, b); +} +NPY_FINLINE npyv_s64 npyv_max_s64(npyv_s64 a, npyv_s64 b) +{ + return vbslq_s64(npyv_cmpgt_s64(a, b), a, b); +} + +// Minimum, natively mapping with no guarantees to handle NaN. +#define npyv_min_f32 vminq_f32 +#define npyv_min_f64 vminq_f64 +// Minimum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +#ifdef NPY_HAVE_ASIMD + #define npyv_minp_f32 vminnmq_f32 +#else + NPY_FINLINE npyv_f32 npyv_minp_f32(npyv_f32 a, npyv_f32 b) + { + npyv_u32 nn_a = vceqq_f32(a, a); + npyv_u32 nn_b = vceqq_f32(b, b); + return vminq_f32(vbslq_f32(nn_a, a, b), vbslq_f32(nn_b, b, a)); + } +#endif +#if NPY_SIMD_F64 + #define npyv_minp_f64 vminnmq_f64 +#endif // NPY_SIMD_F64 +// Minimum, integer operations +#define npyv_min_u8 vminq_u8 +#define npyv_min_s8 vminq_s8 +#define npyv_min_u16 vminq_u16 +#define npyv_min_s16 vminq_s16 +#define npyv_min_u32 vminq_u32 +#define npyv_min_s32 vminq_s32 +NPY_FINLINE npyv_u64 npyv_min_u64(npyv_u64 a, npyv_u64 b) +{ + return vbslq_u64(npyv_cmplt_u64(a, b), a, b); +} +NPY_FINLINE npyv_s64 npyv_min_s64(npyv_s64 a, npyv_s64 b) +{ + return vbslq_s64(npyv_cmplt_s64(a, b), a, b); +} + +#endif // _NPY_SIMD_NEON_MATH_H diff --git a/numpy/core/src/common/simd/neon/neon.h b/numpy/core/src/common/simd/neon/neon.h index c8ddc92add01..e6f6a732478a 100644 --- a/numpy/core/src/common/simd/neon/neon.h +++ b/numpy/core/src/common/simd/neon/neon.h @@ -10,6 +10,11 @@ #else #define NPY_SIMD_F64 0 #endif +#ifdef NPY_HAVE_NEON_VFPV4 + #define NPY_SIMD_FMA3 1 // native support +#else + #define NPY_SIMD_FMA3 0 // HW emulated +#endif typedef uint8x16_t npyv_u8; typedef int8x16_t npyv_s8; diff --git a/numpy/core/src/common/simd/neon/operators.h b/numpy/core/src/common/simd/neon/operators.h index c1ad4ba12a6f..b43ba36537e9 100644 --- a/numpy/core/src/common/simd/neon/operators.h +++ b/numpy/core/src/common/simd/neon/operators.h @@ -34,12 +34,12 @@ #define npyv_shr_s64(A, C) vshlq_s64(A, npyv_setall_s64(-(C))) // right by an immediate constant -#define npyv_shri_u16(VEC, C) ((C) == 0 ? VEC : vshrq_n_u16(VEC, C)) -#define npyv_shri_s16(VEC, C) ((C) == 0 ? VEC : vshrq_n_s16(VEC, C)) -#define npyv_shri_u32(VEC, C) ((C) == 0 ? VEC : vshrq_n_u32(VEC, C)) -#define npyv_shri_s32(VEC, C) ((C) == 0 ? VEC : vshrq_n_s32(VEC, C)) -#define npyv_shri_u64(VEC, C) ((C) == 0 ? VEC : vshrq_n_u64(VEC, C)) -#define npyv_shri_s64(VEC, C) ((C) == 0 ? VEC : vshrq_n_s64(VEC, C)) +#define npyv_shri_u16 vshrq_n_u16 +#define npyv_shri_s16 vshrq_n_s16 +#define npyv_shri_u32 vshrq_n_u32 +#define npyv_shri_s32 vshrq_n_s32 +#define npyv_shri_u64 vshrq_n_u64 +#define npyv_shri_s64 vshrq_n_s64 /*************************** * Logical @@ -58,6 +58,10 @@ vreinterpretq_f32_u8(vandq_u8(vreinterpretq_u8_f32(A), vreinterpretq_u8_f32(B))) #define npyv_and_f64(A, B) \ vreinterpretq_f64_u8(vandq_u8(vreinterpretq_u8_f64(A), vreinterpretq_u8_f64(B))) +#define npyv_and_b8 vandq_u8 +#define npyv_and_b16 vandq_u16 +#define npyv_and_b32 vandq_u32 +#define npyv_and_b64 vandq_u64 // OR #define npyv_or_u8 vorrq_u8 @@ -72,6 +76,11 @@ vreinterpretq_f32_u8(vorrq_u8(vreinterpretq_u8_f32(A), vreinterpretq_u8_f32(B))) #define npyv_or_f64(A, B) \ vreinterpretq_f64_u8(vorrq_u8(vreinterpretq_u8_f64(A), vreinterpretq_u8_f64(B))) +#define npyv_or_b8 vorrq_u8 +#define npyv_or_b16 vorrq_u16 +#define npyv_or_b32 vorrq_u32 +#define npyv_or_b64 vorrq_u64 + // XOR #define npyv_xor_u8 veorq_u8 @@ -86,6 +95,10 @@ vreinterpretq_f32_u8(veorq_u8(vreinterpretq_u8_f32(A), vreinterpretq_u8_f32(B))) #define npyv_xor_f64(A, B) \ vreinterpretq_f64_u8(veorq_u8(vreinterpretq_u8_f64(A), vreinterpretq_u8_f64(B))) +#define npyv_xor_b8 veorq_u8 +#define npyv_xor_b16 veorq_u16 +#define npyv_xor_b32 veorq_u32 +#define npyv_xor_b64 veorq_u64 // NOT #define npyv_not_u8 vmvnq_u8 @@ -98,6 +111,10 @@ #define npyv_not_s64(A) vreinterpretq_s64_u8(vmvnq_u8(vreinterpretq_u8_s64(A))) #define npyv_not_f32(A) vreinterpretq_f32_u8(vmvnq_u8(vreinterpretq_u8_f32(A))) #define npyv_not_f64(A) vreinterpretq_f64_u8(vmvnq_u8(vreinterpretq_u8_f64(A))) +#define npyv_not_b8 vmvnq_u8 +#define npyv_not_b16 vmvnq_u16 +#define npyv_not_b32 vmvnq_u32 +#define npyv_not_b64 npyv_not_u64 /*************************** * Comparison @@ -215,4 +232,12 @@ #define npyv_cmple_f32(A, B) npyv_cmpge_f32(B, A) #define npyv_cmple_f64(A, B) npyv_cmpge_f64(B, A) +// check special cases +NPY_FINLINE npyv_b32 npyv_notnan_f32(npyv_f32 a) +{ return vceqq_f32(a, a); } +#if NPY_SIMD_F64 + NPY_FINLINE npyv_b64 npyv_notnan_f64(npyv_f64 a) + { return vceqq_f64(a, a); } +#endif + #endif // _NPY_SIMD_NEON_OPERATORS_H diff --git a/numpy/core/src/common/simd/neon/reorder.h b/numpy/core/src/common/simd/neon/reorder.h index 712a77982ee6..50b06ed11c01 100644 --- a/numpy/core/src/common/simd/neon/reorder.h +++ b/numpy/core/src/common/simd/neon/reorder.h @@ -107,4 +107,13 @@ NPYV_IMPL_NEON_COMBINE(npyv_f64, f64) #define npyv_zip_u64 npyv_combine_u64 #define npyv_zip_s64 npyv_combine_s64 +// Reverse elements of each 64-bit lane +#define npyv_rev64_u8 vrev64q_u8 +#define npyv_rev64_s8 vrev64q_s8 +#define npyv_rev64_u16 vrev64q_u16 +#define npyv_rev64_s16 vrev64q_s16 +#define npyv_rev64_u32 vrev64q_u32 +#define npyv_rev64_s32 vrev64q_s32 +#define npyv_rev64_f32 vrev64q_f32 + #endif // _NPY_SIMD_NEON_REORDER_H diff --git a/numpy/core/src/common/simd/simd.h b/numpy/core/src/common/simd/simd.h index 8804223c9fef..a3e2b95de9bd 100644 --- a/numpy/core/src/common/simd/simd.h +++ b/numpy/core/src/common/simd/simd.h @@ -48,7 +48,19 @@ typedef double npyv_lanetype_f64; #define NPY_SIMD 0 #define NPY_SIMD_WIDTH 0 #define NPY_SIMD_F64 0 + #define NPY_SIMD_FMA3 0 #endif + +// enable emulated mask operations for all SIMD extension except for AVX512 +#if !defined(NPY_HAVE_AVX512F) && NPY_SIMD && NPY_SIMD < 512 + #include "emulate_maskop.h" +#endif + +// enable integer divisor generator for all SIMD extensions +#if NPY_SIMD + #include "intdiv.h" +#endif + /** * Some SIMD extensions currently(AVX2, AVX512F) require (de facto) * a maximum number of strides sizes when dealing with non-contiguous memory access. diff --git a/numpy/core/src/common/simd/sse/arithmetic.h b/numpy/core/src/common/simd/sse/arithmetic.h index 8440cc52e5c8..bced35108116 100644 --- a/numpy/core/src/common/simd/sse/arithmetic.h +++ b/numpy/core/src/common/simd/sse/arithmetic.h @@ -25,7 +25,7 @@ #define npyv_adds_s8 _mm_adds_epi8 #define npyv_adds_u16 _mm_adds_epu16 #define npyv_adds_s16 _mm_adds_epi16 -// TODO: rest, after implment Packs intrins +// TODO: rest, after implement Packs intrins /*************************** * Subtraction @@ -47,7 +47,7 @@ #define npyv_subs_s8 _mm_subs_epi8 #define npyv_subs_u16 _mm_subs_epu16 #define npyv_subs_s16 _mm_subs_epi16 -// TODO: rest, after implment Packs intrins +// TODO: rest, after implement Packs intrins /*************************** * Multiplication @@ -83,8 +83,183 @@ NPY_FINLINE __m128i npyv_mul_u8(__m128i a, __m128i b) #define npyv_mul_f64 _mm_mul_pd // saturated -// TODO: after implment Packs intrins +// TODO: after implement Packs intrins +/*************************** + * Integer Division + ***************************/ +// See simd/intdiv.h for more clarification +// divide each unsigned 8-bit element by a precomputed divisor +NPY_FINLINE npyv_u8 npyv_divc_u8(npyv_u8 a, const npyv_u8x3 divisor) +{ + const __m128i bmask = _mm_set1_epi32(0x00FF00FF); + const __m128i shf1b = _mm_set1_epi8(0xFFU >> _mm_cvtsi128_si32(divisor.val[1])); + const __m128i shf2b = _mm_set1_epi8(0xFFU >> _mm_cvtsi128_si32(divisor.val[2])); + // high part of unsigned multiplication + __m128i mulhi_even = _mm_mullo_epi16(_mm_and_si128(a, bmask), divisor.val[0]); + __m128i mulhi_odd = _mm_mullo_epi16(_mm_srli_epi16(a, 8), divisor.val[0]); + mulhi_even = _mm_srli_epi16(mulhi_even, 8); + __m128i mulhi = npyv_select_u8(bmask, mulhi_even, mulhi_odd); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m128i q = _mm_sub_epi8(a, mulhi); + q = _mm_and_si128(_mm_srl_epi16(q, divisor.val[1]), shf1b); + q = _mm_add_epi8(mulhi, q); + q = _mm_and_si128(_mm_srl_epi16(q, divisor.val[2]), shf2b); + return q; +} +// divide each signed 8-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor); +NPY_FINLINE npyv_s8 npyv_divc_s8(npyv_s8 a, const npyv_s8x3 divisor) +{ + const __m128i bmask = _mm_set1_epi32(0x00FF00FF); + // instead of _mm_cvtepi8_epi16/_mm_packs_epi16 to wrap around overflow + __m128i divc_even = npyv_divc_s16(_mm_srai_epi16(_mm_slli_epi16(a, 8), 8), divisor); + __m128i divc_odd = npyv_divc_s16(_mm_srai_epi16(a, 8), divisor); + divc_odd = _mm_slli_epi16(divc_odd, 8); + return npyv_select_u8(bmask, divc_even, divc_odd); +} +// divide each unsigned 16-bit element by a precomputed divisor +NPY_FINLINE npyv_u16 npyv_divc_u16(npyv_u16 a, const npyv_u16x3 divisor) +{ + // high part of unsigned multiplication + __m128i mulhi = _mm_mulhi_epu16(a, divisor.val[0]); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m128i q = _mm_sub_epi16(a, mulhi); + q = _mm_srl_epi16(q, divisor.val[1]); + q = _mm_add_epi16(mulhi, q); + q = _mm_srl_epi16(q, divisor.val[2]); + return q; +} +// divide each signed 16-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor) +{ + // high part of signed multiplication + __m128i mulhi = _mm_mulhi_epi16(a, divisor.val[0]); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + __m128i q = _mm_sra_epi16(_mm_add_epi16(a, mulhi), divisor.val[1]); + q = _mm_sub_epi16(q, _mm_srai_epi16(a, 15)); + q = _mm_sub_epi16(_mm_xor_si128(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 32-bit element by a precomputed divisor +NPY_FINLINE npyv_u32 npyv_divc_u32(npyv_u32 a, const npyv_u32x3 divisor) +{ + // high part of unsigned multiplication + __m128i mulhi_even = _mm_srli_epi64(_mm_mul_epu32(a, divisor.val[0]), 32); + __m128i mulhi_odd = _mm_mul_epu32(_mm_srli_epi64(a, 32), divisor.val[0]); +#ifdef NPY_HAVE_SSE41 + __m128i mulhi = _mm_blend_epi16(mulhi_even, mulhi_odd, 0xCC); +#else + __m128i mask_13 = _mm_setr_epi32(0, -1, 0, -1); + mulhi_odd = _mm_and_si128(mulhi_odd, mask_13); + __m128i mulhi = _mm_or_si128(mulhi_even, mulhi_odd); +#endif + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m128i q = _mm_sub_epi32(a, mulhi); + q = _mm_srl_epi32(q, divisor.val[1]); + q = _mm_add_epi32(mulhi, q); + q = _mm_srl_epi32(q, divisor.val[2]); + return q; +} +// divide each signed 32-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s32 npyv_divc_s32(npyv_s32 a, const npyv_s32x3 divisor) +{ + __m128i asign = _mm_srai_epi32(a, 31); +#ifdef NPY_HAVE_SSE41 + // high part of signed multiplication + __m128i mulhi_even = _mm_srli_epi64(_mm_mul_epi32(a, divisor.val[0]), 32); + __m128i mulhi_odd = _mm_mul_epi32(_mm_srli_epi64(a, 32), divisor.val[0]); + __m128i mulhi = _mm_blend_epi16(mulhi_even, mulhi_odd, 0xCC); +#else // not SSE4.1 + // high part of "unsigned" multiplication + __m128i mulhi_even = _mm_srli_epi64(_mm_mul_epu32(a, divisor.val[0]), 32); + __m128i mulhi_odd = _mm_mul_epu32(_mm_srli_epi64(a, 32), divisor.val[0]); + __m128i mask_13 = _mm_setr_epi32(0, -1, 0, -1); + mulhi_odd = _mm_and_si128(mulhi_odd, mask_13); + __m128i mulhi = _mm_or_si128(mulhi_even, mulhi_odd); + // convert unsigned to signed high multiplication + // mulhi - ((a < 0) ? m : 0) - ((m < 0) ? a : 0); + const __m128i msign= _mm_srai_epi32(divisor.val[0], 31); + __m128i m_asign = _mm_and_si128(divisor.val[0], asign); + __m128i a_msign = _mm_and_si128(a, msign); + mulhi = _mm_sub_epi32(mulhi, m_asign); + mulhi = _mm_sub_epi32(mulhi, a_msign); +#endif + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + __m128i q = _mm_sra_epi32(_mm_add_epi32(a, mulhi), divisor.val[1]); + q = _mm_sub_epi32(q, asign); + q = _mm_sub_epi32(_mm_xor_si128(q, divisor.val[2]), divisor.val[2]); + return q; +} +// returns the high 64 bits of unsigned 64-bit multiplication +// xref https://stackoverflow.com/a/28827013 +NPY_FINLINE npyv_u64 npyv__mullhi_u64(npyv_u64 a, npyv_u64 b) +{ + __m128i lomask = npyv_setall_s64(0xffffffff); + __m128i a_hi = _mm_srli_epi64(a, 32); // a0l, a0h, a1l, a1h + __m128i b_hi = _mm_srli_epi64(b, 32); // b0l, b0h, b1l, b1h + // compute partial products + __m128i w0 = _mm_mul_epu32(a, b); // a0l*b0l, a1l*b1l + __m128i w1 = _mm_mul_epu32(a, b_hi); // a0l*b0h, a1l*b1h + __m128i w2 = _mm_mul_epu32(a_hi, b); // a0h*b0l, a1h*b0l + __m128i w3 = _mm_mul_epu32(a_hi, b_hi); // a0h*b0h, a1h*b1h + // sum partial products + __m128i w0h = _mm_srli_epi64(w0, 32); + __m128i s1 = _mm_add_epi64(w1, w0h); + __m128i s1l = _mm_and_si128(s1, lomask); + __m128i s1h = _mm_srli_epi64(s1, 32); + + __m128i s2 = _mm_add_epi64(w2, s1l); + __m128i s2h = _mm_srli_epi64(s2, 32); + + __m128i hi = _mm_add_epi64(w3, s1h); + hi = _mm_add_epi64(hi, s2h); + return hi; +} +// divide each unsigned 64-bit element by a precomputed divisor +NPY_FINLINE npyv_u64 npyv_divc_u64(npyv_u64 a, const npyv_u64x3 divisor) +{ + // high part of unsigned multiplication + __m128i mulhi = npyv__mullhi_u64(a, divisor.val[0]); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + __m128i q = _mm_sub_epi64(a, mulhi); + q = _mm_srl_epi64(q, divisor.val[1]); + q = _mm_add_epi64(mulhi, q); + q = _mm_srl_epi64(q, divisor.val[2]); + return q; +} +// divide each signed 64-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s64 npyv_divc_s64(npyv_s64 a, const npyv_s64x3 divisor) +{ + // high part of unsigned multiplication + __m128i mulhi = npyv__mullhi_u64(a, divisor.val[0]); + // convert unsigned to signed high multiplication + // mulhi - ((a < 0) ? m : 0) - ((m < 0) ? a : 0); +#ifdef NPY_HAVE_SSE42 + const __m128i msign= _mm_cmpgt_epi64(_mm_setzero_si128(), divisor.val[0]); + __m128i asign = _mm_cmpgt_epi64(_mm_setzero_si128(), a); +#else + const __m128i msign= _mm_srai_epi32(_mm_shuffle_epi32(divisor.val[0], _MM_SHUFFLE(3, 3, 1, 1)), 31); + __m128i asign = _mm_srai_epi32(_mm_shuffle_epi32(a, _MM_SHUFFLE(3, 3, 1, 1)), 31); +#endif + __m128i m_asign = _mm_and_si128(divisor.val[0], asign); + __m128i a_msign = _mm_and_si128(a, msign); + mulhi = _mm_sub_epi64(mulhi, m_asign); + mulhi = _mm_sub_epi64(mulhi, a_msign); + // q = (a + mulhi) >> sh + __m128i q = _mm_add_epi64(a, mulhi); + // emulate arithmetic right shift + const __m128i sigb = npyv_setall_s64(1LL << 63); + q = _mm_srl_epi64(_mm_add_epi64(q, sigb), divisor.val[1]); + q = _mm_sub_epi64(q, _mm_srl_epi64(sigb, divisor.val[1])); + // q = q - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + q = _mm_sub_epi64(q, asign); + q = _mm_sub_epi64(_mm_xor_si128(q, divisor.val[2]), divisor.val[2]); + return q; +} /*************************** * Division ***************************/ @@ -148,8 +323,24 @@ NPY_FINLINE __m128i npyv_mul_u8(__m128i a, __m128i b) } #endif // !NPY_HAVE_FMA3 -// Horizontal add: Calculates the sum of all vector elements. -NPY_FINLINE float npyv_sum_f32(__m128 a) +/*************************** + * Summation + ***************************/ +// reduce sum across vector +NPY_FINLINE npy_uint32 npyv_sum_u32(npyv_u32 a) +{ + __m128i t = _mm_add_epi32(a, _mm_srli_si128(a, 8)); + t = _mm_add_epi32(t, _mm_srli_si128(t, 4)); + return (unsigned)_mm_cvtsi128_si32(t); +} + +NPY_FINLINE npy_uint64 npyv_sum_u64(npyv_u64 a) +{ + __m128i one = _mm_add_epi64(a, _mm_unpackhi_epi64(a, a)); + return (npy_uint64)npyv128_cvtsi128_si64(one); +} + +NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { #ifdef NPY_HAVE_SSE3 __m128 sum_halves = _mm_hadd_ps(a, a); @@ -159,11 +350,11 @@ NPY_FINLINE float npyv_sum_f32(__m128 a) __m128 t2 = _mm_add_ps(a, t1); __m128 t3 = _mm_shuffle_ps(t2, t2, 1); __m128 t4 = _mm_add_ss(t2, t3); - return _mm_cvtss_f32(t4); + return _mm_cvtss_f32(t4); #endif } -NPY_FINLINE double npyv_sum_f64(__m128d a) +NPY_FINLINE double npyv_sum_f64(npyv_f64 a) { #ifdef NPY_HAVE_SSE3 return _mm_cvtsd_f64(_mm_hadd_pd(a, a)); @@ -172,6 +363,23 @@ NPY_FINLINE double npyv_sum_f64(__m128d a) #endif } +// expand the source vector and performs sum reduce +NPY_FINLINE npy_uint16 npyv_sumup_u8(npyv_u8 a) +{ + __m128i two = _mm_sad_epu8(a, _mm_setzero_si128()); + __m128i one = _mm_add_epi16(two, _mm_unpackhi_epi64(two, two)); + return (npy_uint16)_mm_cvtsi128_si32(one); +} + +NPY_FINLINE npy_uint32 npyv_sumup_u16(npyv_u16 a) +{ + const __m128i even_mask = _mm_set1_epi32(0x0000FFFF); + __m128i even = _mm_and_si128(a, even_mask); + __m128i odd = _mm_srli_epi32(a, 16); + __m128i four = _mm_add_epi32(even, odd); + return npyv_sum_u32(four); +} + #endif // _NPY_SIMD_SSE_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/sse/conversion.h b/numpy/core/src/common/simd/sse/conversion.h index ea9660d13a30..ab7eb490727b 100644 --- a/numpy/core/src/common/simd/sse/conversion.h +++ b/numpy/core/src/common/simd/sse/conversion.h @@ -14,8 +14,8 @@ #define npyv_cvt_s32_b32(BL) BL #define npyv_cvt_u64_b64(BL) BL #define npyv_cvt_s64_b64(BL) BL -#define npyv_cvt_f32_b32(BL) _mm_castsi128_ps(BL) -#define npyv_cvt_f64_b64(BL) _mm_castsi128_pd(BL) +#define npyv_cvt_f32_b32 _mm_castsi128_ps +#define npyv_cvt_f64_b64 _mm_castsi128_pd // convert integer types to mask types #define npyv_cvt_b8_u8(A) A @@ -26,7 +26,45 @@ #define npyv_cvt_b32_s32(A) A #define npyv_cvt_b64_u64(A) A #define npyv_cvt_b64_s64(A) A -#define npyv_cvt_b32_f32(A) _mm_castps_si128(A) -#define npyv_cvt_b64_f64(A) _mm_castpd_si128(A) +#define npyv_cvt_b32_f32 _mm_castps_si128 +#define npyv_cvt_b64_f64 _mm_castpd_si128 + +// convert boolean vector to integer bitfield +NPY_FINLINE npy_uint64 npyv_tobits_b8(npyv_b8 a) +{ return (npy_uint16)_mm_movemask_epi8(a); } +NPY_FINLINE npy_uint64 npyv_tobits_b16(npyv_b16 a) +{ + __m128i pack = _mm_packs_epi16(a, a); + return (npy_uint8)_mm_movemask_epi8(pack); +} +NPY_FINLINE npy_uint64 npyv_tobits_b32(npyv_b32 a) +{ return (npy_uint8)_mm_movemask_ps(_mm_castsi128_ps(a)); } +NPY_FINLINE npy_uint64 npyv_tobits_b64(npyv_b64 a) +{ return (npy_uint8)_mm_movemask_pd(_mm_castsi128_pd(a)); } + +// expand +NPY_FINLINE npyv_u16x2 npyv_expand_u16_u8(npyv_u8 data) { + npyv_u16x2 r; + const __m128i z = _mm_setzero_si128(); + r.val[0] = _mm_unpacklo_epi8(data, z); + r.val[1] = _mm_unpackhi_epi8(data, z); + return r; +} + +NPY_FINLINE npyv_u32x2 npyv_expand_u32_u16(npyv_u16 data) { + npyv_u32x2 r; + const __m128i z = _mm_setzero_si128(); + r.val[0] = _mm_unpacklo_epi16(data, z); + r.val[1] = _mm_unpackhi_epi16(data, z); + return r; +} + +// round to nearest integer (assuming even) +#define npyv_round_s32_f32 _mm_cvtps_epi32 +NPY_FINLINE npyv_s32 npyv_round_s32_f64(npyv_f64 a, npyv_f64 b) +{ + __m128i lo = _mm_cvtpd_epi32(a), hi = _mm_cvtpd_epi32(b); + return _mm_unpacklo_epi64(lo, hi); +} #endif // _NPY_SIMD_SSE_CVT_H diff --git a/numpy/core/src/common/simd/sse/math.h b/numpy/core/src/common/simd/sse/math.h index b7203cd8993d..97d35afc5e04 100644 --- a/numpy/core/src/common/simd/sse/math.h +++ b/numpy/core/src/common/simd/sse/math.h @@ -37,4 +37,110 @@ NPY_FINLINE npyv_f32 npyv_square_f32(npyv_f32 a) NPY_FINLINE npyv_f64 npyv_square_f64(npyv_f64 a) { return _mm_mul_pd(a, a); } +// Maximum, natively mapping with no guarantees to handle NaN. +#define npyv_max_f32 _mm_max_ps +#define npyv_max_f64 _mm_max_pd +// Maximum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +NPY_FINLINE npyv_f32 npyv_maxp_f32(npyv_f32 a, npyv_f32 b) +{ + __m128 nn = _mm_cmpord_ps(b, b); + __m128 max = _mm_max_ps(a, b); + return npyv_select_f32(_mm_castps_si128(nn), max, a); +} +NPY_FINLINE npyv_f64 npyv_maxp_f64(npyv_f64 a, npyv_f64 b) +{ + __m128d nn = _mm_cmpord_pd(b, b); + __m128d max = _mm_max_pd(a, b); + return npyv_select_f64(_mm_castpd_si128(nn), max, a); +} +// Maximum, integer operations +#ifdef NPY_HAVE_SSE41 + #define npyv_max_s8 _mm_max_epi8 + #define npyv_max_u16 _mm_max_epu16 + #define npyv_max_u32 _mm_max_epu32 + #define npyv_max_s32 _mm_max_epi32 +#else + NPY_FINLINE npyv_s8 npyv_max_s8(npyv_s8 a, npyv_s8 b) + { + return npyv_select_s8(npyv_cmpgt_s8(a, b), a, b); + } + NPY_FINLINE npyv_u16 npyv_max_u16(npyv_u16 a, npyv_u16 b) + { + return npyv_select_u16(npyv_cmpgt_u16(a, b), a, b); + } + NPY_FINLINE npyv_u32 npyv_max_u32(npyv_u32 a, npyv_u32 b) + { + return npyv_select_u32(npyv_cmpgt_u32(a, b), a, b); + } + NPY_FINLINE npyv_s32 npyv_max_s32(npyv_s32 a, npyv_s32 b) + { + return npyv_select_s32(npyv_cmpgt_s32(a, b), a, b); + } #endif +#define npyv_max_u8 _mm_max_epu8 +#define npyv_max_s16 _mm_max_epi16 +NPY_FINLINE npyv_u64 npyv_max_u64(npyv_u64 a, npyv_u64 b) +{ + return npyv_select_u64(npyv_cmpgt_u64(a, b), a, b); +} +NPY_FINLINE npyv_s64 npyv_max_s64(npyv_s64 a, npyv_s64 b) +{ + return npyv_select_s64(npyv_cmpgt_s64(a, b), a, b); +} + +// Minimum, natively mapping with no guarantees to handle NaN. +#define npyv_min_f32 _mm_min_ps +#define npyv_min_f64 _mm_min_pd +// Minimum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +NPY_FINLINE npyv_f32 npyv_minp_f32(npyv_f32 a, npyv_f32 b) +{ + __m128 nn = _mm_cmpord_ps(b, b); + __m128 min = _mm_min_ps(a, b); + return npyv_select_f32(_mm_castps_si128(nn), min, a); +} +NPY_FINLINE npyv_f64 npyv_minp_f64(npyv_f64 a, npyv_f64 b) +{ + __m128d nn = _mm_cmpord_pd(b, b); + __m128d min = _mm_min_pd(a, b); + return npyv_select_f64(_mm_castpd_si128(nn), min, a); +} +// Minimum, integer operations +#ifdef NPY_HAVE_SSE41 + #define npyv_min_s8 _mm_min_epi8 + #define npyv_min_u16 _mm_min_epu16 + #define npyv_min_u32 _mm_min_epu32 + #define npyv_min_s32 _mm_min_epi32 +#else + NPY_FINLINE npyv_s8 npyv_min_s8(npyv_s8 a, npyv_s8 b) + { + return npyv_select_s8(npyv_cmplt_s8(a, b), a, b); + } + NPY_FINLINE npyv_u16 npyv_min_u16(npyv_u16 a, npyv_u16 b) + { + return npyv_select_u16(npyv_cmplt_u16(a, b), a, b); + } + NPY_FINLINE npyv_u32 npyv_min_u32(npyv_u32 a, npyv_u32 b) + { + return npyv_select_u32(npyv_cmplt_u32(a, b), a, b); + } + NPY_FINLINE npyv_s32 npyv_min_s32(npyv_s32 a, npyv_s32 b) + { + return npyv_select_s32(npyv_cmplt_s32(a, b), a, b); + } +#endif +#define npyv_min_u8 _mm_min_epu8 +#define npyv_min_s16 _mm_min_epi16 +NPY_FINLINE npyv_u64 npyv_min_u64(npyv_u64 a, npyv_u64 b) +{ + return npyv_select_u64(npyv_cmplt_u64(a, b), a, b); +} +NPY_FINLINE npyv_s64 npyv_min_s64(npyv_s64 a, npyv_s64 b) +{ + return npyv_select_s64(npyv_cmplt_s64(a, b), a, b); +} + +#endif // _NPY_SIMD_SSE_MATH_H diff --git a/numpy/core/src/common/simd/sse/misc.h b/numpy/core/src/common/simd/sse/misc.h index 7ba47bc688df..1099c491d072 100644 --- a/numpy/core/src/common/simd/sse/misc.h +++ b/numpy/core/src/common/simd/sse/misc.h @@ -18,21 +18,16 @@ #define npyv_zero_f64 _mm_setzero_pd // vector with a specific value set to all lanes -#define npyv_setall_u8(VAL) _mm_set1_epi8((char)VAL) -#define npyv_setall_s8(VAL) _mm_set1_epi8((char)VAL) -#define npyv_setall_u16(VAL) _mm_set1_epi16((short)VAL) -#define npyv_setall_s16(VAL) _mm_set1_epi16((short)VAL) -#define npyv_setall_u32(VAL) _mm_set1_epi32((int)VAL) -#define npyv_setall_s32(VAL) _mm_set1_epi32(VAL) -#if !defined(__x86_64__) && !defined(_M_X64) - #define npyv_setall_u64(VAL) _mm_set_epi32((int)(VAL >> 32), (int)VAL, (int)(VAL >> 32), (int)VAL) - #define npyv_setall_s64 npyv_setall_u64 -#else - #define npyv_setall_u64(VAL) _mm_set1_epi64x(VAL) - #define npyv_setall_s64(VAL) _mm_set1_epi64x(VAL) -#endif -#define npyv_setall_f32(VAL) _mm_set1_ps(VAL) -#define npyv_setall_f64(VAL) _mm_set1_pd(VAL) +#define npyv_setall_u8(VAL) _mm_set1_epi8((char)(VAL)) +#define npyv_setall_s8(VAL) _mm_set1_epi8((char)(VAL)) +#define npyv_setall_u16(VAL) _mm_set1_epi16((short)(VAL)) +#define npyv_setall_s16(VAL) _mm_set1_epi16((short)(VAL)) +#define npyv_setall_u32(VAL) _mm_set1_epi32((int)(VAL)) +#define npyv_setall_s32(VAL) _mm_set1_epi32((int)(VAL)) +#define npyv_setall_u64(VAL) _mm_set1_epi64x((npy_int64)(VAL)) +#define npyv_setall_s64(VAL) _mm_set1_epi64x((npy_int64)(VAL)) +#define npyv_setall_f32 _mm_set1_ps +#define npyv_setall_f64 _mm_set1_pd /** * vector with specific values set to each lane and diff --git a/numpy/core/src/common/simd/sse/operators.h b/numpy/core/src/common/simd/sse/operators.h index 6e32ca4fd9b3..51c84fb4e9d9 100644 --- a/numpy/core/src/common/simd/sse/operators.h +++ b/numpy/core/src/common/simd/sse/operators.h @@ -62,6 +62,10 @@ NPY_FINLINE __m128i npyv_shr_s64(__m128i a, int c) #define npyv_and_s64 _mm_and_si128 #define npyv_and_f32 _mm_and_ps #define npyv_and_f64 _mm_and_pd +#define npyv_and_b8 _mm_and_si128 +#define npyv_and_b16 _mm_and_si128 +#define npyv_and_b32 _mm_and_si128 +#define npyv_and_b64 _mm_and_si128 // OR #define npyv_or_u8 _mm_or_si128 @@ -74,6 +78,10 @@ NPY_FINLINE __m128i npyv_shr_s64(__m128i a, int c) #define npyv_or_s64 _mm_or_si128 #define npyv_or_f32 _mm_or_ps #define npyv_or_f64 _mm_or_pd +#define npyv_or_b8 _mm_or_si128 +#define npyv_or_b16 _mm_or_si128 +#define npyv_or_b32 _mm_or_si128 +#define npyv_or_b64 _mm_or_si128 // XOR #define npyv_xor_u8 _mm_xor_si128 @@ -86,6 +94,10 @@ NPY_FINLINE __m128i npyv_shr_s64(__m128i a, int c) #define npyv_xor_s64 _mm_xor_si128 #define npyv_xor_f32 _mm_xor_ps #define npyv_xor_f64 _mm_xor_pd +#define npyv_xor_b8 _mm_xor_si128 +#define npyv_xor_b16 _mm_xor_si128 +#define npyv_xor_b32 _mm_xor_si128 +#define npyv_xor_b64 _mm_xor_si128 // NOT #define npyv_not_u8(A) _mm_xor_si128(A, _mm_set1_epi32(-1)) @@ -98,6 +110,10 @@ NPY_FINLINE __m128i npyv_shr_s64(__m128i a, int c) #define npyv_not_s64 npyv_not_u8 #define npyv_not_f32(A) _mm_xor_ps(A, _mm_castsi128_ps(_mm_set1_epi32(-1))) #define npyv_not_f64(A) _mm_xor_pd(A, _mm_castsi128_pd(_mm_set1_epi32(-1))) +#define npyv_not_b8 npyv_not_u8 +#define npyv_not_b16 npyv_not_u8 +#define npyv_not_b32 npyv_not_u8 +#define npyv_not_b64 npyv_not_u8 /*************************** * Comparison @@ -255,4 +271,10 @@ NPY_FINLINE __m128i npyv_shr_s64(__m128i a, int c) #define npyv_cmpge_f32(a, b) _mm_castps_si128(_mm_cmpge_ps(a, b)) #define npyv_cmpge_f64(a, b) _mm_castpd_si128(_mm_cmpge_pd(a, b)) +// check special cases +NPY_FINLINE npyv_b32 npyv_notnan_f32(npyv_f32 a) +{ return _mm_castps_si128(_mm_cmpord_ps(a, a)); } +NPY_FINLINE npyv_b64 npyv_notnan_f64(npyv_f64 a) +{ return _mm_castpd_si128(_mm_cmpord_pd(a, a)); } + #endif // _NPY_SIMD_SSE_OPERATORS_H diff --git a/numpy/core/src/common/simd/sse/reorder.h b/numpy/core/src/common/simd/sse/reorder.h index 3f68b4ad70e4..d96ab9c5688b 100644 --- a/numpy/core/src/common/simd/sse/reorder.h +++ b/numpy/core/src/common/simd/sse/reorder.h @@ -81,4 +81,45 @@ NPYV_IMPL_SSE_ZIP(npyv_s64, s64, epi64) NPYV_IMPL_SSE_ZIP(npyv_f32, f32, ps) NPYV_IMPL_SSE_ZIP(npyv_f64, f64, pd) +// Reverse elements of each 64-bit lane +NPY_FINLINE npyv_u16 npyv_rev64_u16(npyv_u16 a) +{ +#ifdef NPY_HAVE_SSSE3 + const __m128i idx = _mm_setr_epi8( + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9 + ); + return _mm_shuffle_epi8(a, idx); +#else + __m128i lo = _mm_shufflelo_epi16(a, _MM_SHUFFLE(0, 1, 2, 3)); + return _mm_shufflehi_epi16(lo, _MM_SHUFFLE(0, 1, 2, 3)); +#endif +} +#define npyv_rev64_s16 npyv_rev64_u16 + +NPY_FINLINE npyv_u8 npyv_rev64_u8(npyv_u8 a) +{ +#ifdef NPY_HAVE_SSSE3 + const __m128i idx = _mm_setr_epi8( + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8 + ); + return _mm_shuffle_epi8(a, idx); +#else + __m128i rev16 = npyv_rev64_u16(a); + // swap 8bit pairs + return _mm_or_si128(_mm_slli_epi16(rev16, 8), _mm_srli_epi16(rev16, 8)); +#endif +} +#define npyv_rev64_s8 npyv_rev64_u8 + +NPY_FINLINE npyv_u32 npyv_rev64_u32(npyv_u32 a) +{ + return _mm_shuffle_epi32(a, _MM_SHUFFLE(2, 3, 0, 1)); +} +#define npyv_rev64_s32 npyv_rev64_u32 + +NPY_FINLINE npyv_f32 npyv_rev64_f32(npyv_f32 a) +{ + return _mm_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); +} + #endif // _NPY_SIMD_SSE_REORDER_H diff --git a/numpy/core/src/common/simd/sse/sse.h b/numpy/core/src/common/simd/sse/sse.h index 132d3d347a5d..0bb404312867 100644 --- a/numpy/core/src/common/simd/sse/sse.h +++ b/numpy/core/src/common/simd/sse/sse.h @@ -5,7 +5,11 @@ #define NPY_SIMD 128 #define NPY_SIMD_WIDTH 16 #define NPY_SIMD_F64 1 - +#if defined(NPY_HAVE_FMA3) || defined(NPY_HAVE_FMA4) + #define NPY_SIMD_FMA3 1 // native support +#else + #define NPY_SIMD_FMA3 0 // fast emulated +#endif typedef __m128i npyv_u8; typedef __m128i npyv_s8; typedef __m128i npyv_u16; @@ -58,6 +62,7 @@ typedef struct { __m128d val[3]; } npyv_f64x3; #define npyv_nlanes_f32 4 #define npyv_nlanes_f64 2 +#include "utils.h" #include "memory.h" #include "misc.h" #include "reorder.h" diff --git a/numpy/core/src/common/simd/sse/utils.h b/numpy/core/src/common/simd/sse/utils.h new file mode 100644 index 000000000000..c23def11d44c --- /dev/null +++ b/numpy/core/src/common/simd/sse/utils.h @@ -0,0 +1,19 @@ +#ifndef NPY_SIMD + #error "Not a standalone header" +#endif + +#ifndef _NPY_SIMD_SSE_UTILS_H +#define _NPY_SIMD_SSE_UTILS_H + +#if !defined(__x86_64__) && !defined(_M_X64) +NPY_FINLINE npy_int64 npyv128_cvtsi128_si64(__m128i a) +{ + npy_int64 NPY_DECL_ALIGNED(16) idx[2]; + _mm_store_si128((__m128i *)idx, a); + return idx[0]; +} +#else + #define npyv128_cvtsi128_si64 _mm_cvtsi128_si64 +#endif + +#endif // _NPY_SIMD_SSE_UTILS_H diff --git a/numpy/core/src/common/simd/vsx/arithmetic.h b/numpy/core/src/common/simd/vsx/arithmetic.h index 2f6762e636b4..eaca536201fb 100644 --- a/numpy/core/src/common/simd/vsx/arithmetic.h +++ b/numpy/core/src/common/simd/vsx/arithmetic.h @@ -94,6 +94,138 @@ #define npyv_mul_f32 vec_mul #define npyv_mul_f64 vec_mul +/*************************** + * Integer Division + ***************************/ +/*** + * TODO: Add support for VSX4(Power10) + */ +// See simd/intdiv.h for more clarification +// divide each unsigned 8-bit element by a precomputed divisor +NPY_FINLINE npyv_u8 npyv_divc_u8(npyv_u8 a, const npyv_u8x3 divisor) +{ + const npyv_u8 mergeo_perm = { + 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31 + }; + // high part of unsigned multiplication + npyv_u16 mul_even = vec_mule(a, divisor.val[0]); + npyv_u16 mul_odd = vec_mulo(a, divisor.val[0]); + npyv_u8 mulhi = (npyv_u8)vec_perm(mul_even, mul_odd, mergeo_perm); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + npyv_u8 q = vec_sub(a, mulhi); + q = vec_sr(q, divisor.val[1]); + q = vec_add(mulhi, q); + q = vec_sr(q, divisor.val[2]); + return q; +} +// divide each signed 8-bit element by a precomputed divisor +NPY_FINLINE npyv_s8 npyv_divc_s8(npyv_s8 a, const npyv_s8x3 divisor) +{ + const npyv_u8 mergeo_perm = { + 1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31 + }; + // high part of signed multiplication + npyv_s16 mul_even = vec_mule(a, divisor.val[0]); + npyv_s16 mul_odd = vec_mulo(a, divisor.val[0]); + npyv_s8 mulhi = (npyv_s8)vec_perm(mul_even, mul_odd, mergeo_perm); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + npyv_s8 q = vec_sra(vec_add(a, mulhi), (npyv_u8)divisor.val[1]); + q = vec_sub(q, vec_sra(a, npyv_setall_u8(7))); + q = vec_sub(vec_xor(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 16-bit element by a precomputed divisor +NPY_FINLINE npyv_u16 npyv_divc_u16(npyv_u16 a, const npyv_u16x3 divisor) +{ + const npyv_u8 mergeo_perm = { + 2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31 + }; + // high part of unsigned multiplication + npyv_u32 mul_even = vec_mule(a, divisor.val[0]); + npyv_u32 mul_odd = vec_mulo(a, divisor.val[0]); + npyv_u16 mulhi = (npyv_u16)vec_perm(mul_even, mul_odd, mergeo_perm); + // floor(a/d) = (mulhi + ((a-mulhi) >> sh1)) >> sh2 + npyv_u16 q = vec_sub(a, mulhi); + q = vec_sr(q, divisor.val[1]); + q = vec_add(mulhi, q); + q = vec_sr(q, divisor.val[2]); + return q; +} +// divide each signed 16-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s16 npyv_divc_s16(npyv_s16 a, const npyv_s16x3 divisor) +{ + const npyv_u8 mergeo_perm = { + 2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31 + }; + // high part of signed multiplication + npyv_s32 mul_even = vec_mule(a, divisor.val[0]); + npyv_s32 mul_odd = vec_mulo(a, divisor.val[0]); + npyv_s16 mulhi = (npyv_s16)vec_perm(mul_even, mul_odd, mergeo_perm); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + npyv_s16 q = vec_sra(vec_add(a, mulhi), (npyv_u16)divisor.val[1]); + q = vec_sub(q, vec_sra(a, npyv_setall_u16(15))); + q = vec_sub(vec_xor(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 32-bit element by a precomputed divisor +NPY_FINLINE npyv_u32 npyv_divc_u32(npyv_u32 a, const npyv_u32x3 divisor) +{ +#if defined(__GNUC__) && __GNUC__ < 8 + // Doubleword integer wide multiplication supported by GCC 8+ + npyv_u64 mul_even, mul_odd; + __asm__ ("vmulouw %0,%1,%2" : "=v" (mul_even) : "v" (a), "v" (divisor.val[0])); + __asm__ ("vmuleuw %0,%1,%2" : "=v" (mul_odd) : "v" (a), "v" (divisor.val[0])); +#else + // Doubleword integer wide multiplication supported by GCC 8+ + npyv_u64 mul_even = vec_mule(a, divisor.val[0]); + npyv_u64 mul_odd = vec_mulo(a, divisor.val[0]); +#endif + // high part of unsigned multiplication + npyv_u32 mulhi = vec_mergeo((npyv_u32)mul_even, (npyv_u32)mul_odd); + // floor(x/d) = (((a-mulhi) >> sh1) + mulhi) >> sh2 + npyv_u32 q = vec_sub(a, mulhi); + q = vec_sr(q, divisor.val[1]); + q = vec_add(mulhi, q); + q = vec_sr(q, divisor.val[2]); + return q; +} +// divide each signed 32-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s32 npyv_divc_s32(npyv_s32 a, const npyv_s32x3 divisor) +{ +#if defined(__GNUC__) && __GNUC__ < 8 + // Doubleword integer wide multiplication supported by GCC8+ + npyv_s64 mul_even, mul_odd; + __asm__ ("vmulosw %0,%1,%2" : "=v" (mul_even) : "v" (a), "v" (divisor.val[0])); + __asm__ ("vmulesw %0,%1,%2" : "=v" (mul_odd) : "v" (a), "v" (divisor.val[0])); +#else + // Doubleword integer wide multiplication supported by GCC8+ + npyv_s64 mul_even = vec_mule(a, divisor.val[0]); + npyv_s64 mul_odd = vec_mulo(a, divisor.val[0]); +#endif + // high part of signed multiplication + npyv_s32 mulhi = vec_mergeo((npyv_s32)mul_even, (npyv_s32)mul_odd); + // q = ((a + mulhi) >> sh1) - XSIGN(a) + // trunc(a/d) = (q ^ dsign) - dsign + npyv_s32 q = vec_sra(vec_add(a, mulhi), (npyv_u32)divisor.val[1]); + q = vec_sub(q, vec_sra(a, npyv_setall_u32(31))); + q = vec_sub(vec_xor(q, divisor.val[2]), divisor.val[2]); + return q; +} +// divide each unsigned 64-bit element by a precomputed divisor +NPY_FINLINE npyv_u64 npyv_divc_u64(npyv_u64 a, const npyv_u64x3 divisor) +{ + const npy_uint64 d = vec_extract(divisor.val[0], 0); + return npyv_set_u64(vec_extract(a, 0) / d, vec_extract(a, 1) / d); +} +// divide each signed 64-bit element by a precomputed divisor (round towards zero) +NPY_FINLINE npyv_s64 npyv_divc_s64(npyv_s64 a, const npyv_s64x3 divisor) +{ + npyv_b64 overflow = npyv_and_b64(vec_cmpeq(a, npyv_setall_s64(-1LL << 63)), (npyv_b64)divisor.val[1]); + npyv_s64 d = vec_sel(divisor.val[0], npyv_setall_s64(1), overflow); + return vec_div(a, d); +} /*************************** * Division ***************************/ @@ -116,7 +248,21 @@ #define npyv_nmulsub_f32 vec_nmadd // equivalent to -(a*b + c) #define npyv_nmulsub_f64 vec_nmadd -// Horizontal add: Calculates the sum of all vector elements. +/*************************** + * Summation + ***************************/ +// reduce sum across vector +NPY_FINLINE npy_uint64 npyv_sum_u64(npyv_u64 a) +{ + return vec_extract(vec_add(a, vec_mergel(a, a)), 0); +} + +NPY_FINLINE npy_uint32 npyv_sum_u32(npyv_u32 a) +{ + const npyv_u32 rs = vec_add(a, vec_sld(a, a, 8)); + return vec_extract(vec_add(rs, vec_sld(rs, rs, 4)), 0); +} + NPY_FINLINE float npyv_sum_f32(npyv_f32 a) { npyv_f32 sum = vec_add(a, npyv_combineh_f32(a, a)); @@ -128,4 +274,22 @@ NPY_FINLINE double npyv_sum_f64(npyv_f64 a) return vec_extract(a, 0) + vec_extract(a, 1); } +// expand the source vector and performs sum reduce +NPY_FINLINE npy_uint16 npyv_sumup_u8(npyv_u8 a) +{ + const npyv_u32 zero = npyv_zero_u32(); + npyv_u32 four = vec_sum4s(a, zero); + npyv_s32 one = vec_sums((npyv_s32)four, (npyv_s32)zero); + return (npy_uint16)vec_extract(one, 3); +} + +NPY_FINLINE npy_uint32 npyv_sumup_u16(npyv_u16 a) +{ + const npyv_s32 zero = npyv_zero_s32(); + npyv_u32x2 eight = npyv_expand_u32_u16(a); + npyv_u32 four = vec_add(eight.val[0], eight.val[1]); + npyv_s32 one = vec_sums((npyv_s32)four, zero); + return (npy_uint32)vec_extract(one, 3); +} + #endif // _NPY_SIMD_VSX_ARITHMETIC_H diff --git a/numpy/core/src/common/simd/vsx/conversion.h b/numpy/core/src/common/simd/vsx/conversion.h index 6ed135990ccc..36bea7bbaddf 100644 --- a/numpy/core/src/common/simd/vsx/conversion.h +++ b/numpy/core/src/common/simd/vsx/conversion.h @@ -29,4 +29,95 @@ #define npyv_cvt_b32_f32(A) ((npyv_b32) A) #define npyv_cvt_b64_f64(A) ((npyv_b64) A) +//expand +NPY_FINLINE npyv_u16x2 npyv_expand_u16_u8(npyv_u8 data) +{ + npyv_u16x2 r; + npyv_u8 zero = npyv_zero_u8(); + r.val[0] = (npyv_u16)vec_mergeh(data, zero); + r.val[1] = (npyv_u16)vec_mergel(data, zero); + return r; +} + +NPY_FINLINE npyv_u32x2 npyv_expand_u32_u16(npyv_u16 data) +{ + npyv_u32x2 r; + npyv_u16 zero = npyv_zero_u16(); + r.val[0] = (npyv_u32)vec_mergeh(data, zero); + r.val[1] = (npyv_u32)vec_mergel(data, zero); + return r; +} + +// convert boolean vector to integer bitfield +NPY_FINLINE npy_uint64 npyv_tobits_b8(npyv_b8 a) +{ + const npyv_u8 qperm = npyv_set_u8(120, 112, 104, 96, 88, 80, 72, 64, 56, 48, 40, 32, 24, 16, 8, 0); + return vec_extract((npyv_u32)vec_vbpermq((npyv_u8)a, qperm), 2); +} +NPY_FINLINE npy_uint64 npyv_tobits_b16(npyv_b16 a) +{ + const npyv_u8 qperm = npyv_setf_u8(128, 112, 96, 80, 64, 48, 32, 16, 0); + return vec_extract((npyv_u32)vec_vbpermq((npyv_u8)a, qperm), 2); +} +NPY_FINLINE npy_uint64 npyv_tobits_b32(npyv_b32 a) +{ + const npyv_u8 qperm = npyv_setf_u8(128, 96, 64, 32, 0); + return vec_extract((npyv_u32)vec_vbpermq((npyv_u8)a, qperm), 2); +} +NPY_FINLINE npy_uint64 npyv_tobits_b64(npyv_b64 a) +{ + npyv_u64 bit = npyv_shri_u64((npyv_u64)a, 63); + return vec_extract(bit, 0) | (int)vec_extract(bit, 1) << 1; +} + +// truncate compatible with all compilers(internal use for now) +NPY_FINLINE npyv_s32 npyv__trunc_s32_f32(npyv_f32 a) +{ +#ifdef __IBMC__ + return vec_cts(a, 0); +#elif defined(__clang__) + /** + * old versions of CLANG doesn't support %x in the inline asm template + * which fixes register number when using any of the register constraints wa, wd, wf. + * therefore, we count on built-in functions. + */ + return __builtin_convertvector(a, npyv_s32); +#else // gcc + npyv_s32 ret; + __asm__ ("xvcvspsxws %x0,%x1" : "=wa" (ret) : "wa" (a)); + return ret; +#endif +} +NPY_FINLINE npyv_s32 npyv__trunc_s32_f64(npyv_f64 a, npyv_f64 b) +{ +#ifdef __IBMC__ + const npyv_u8 seq_even = npyv_set_u8(0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27); + // unfortunately, XLC missing asm register vsx fixer + // hopefully, xlc can optimize around big-endian compatibility + npyv_s32 lo_even = vec_cts(a, 0); + npyv_s32 hi_even = vec_cts(b, 0); + return vec_perm(lo_even, hi_even, seq_even); +#else + const npyv_u8 seq_odd = npyv_set_u8(4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31); + #ifdef __clang__ + // __builtin_convertvector doesn't support this conversion on wide range of versions + // fortunately, almost all versions have direct builtin of 'xvcvdpsxws' + npyv_s32 lo_odd = __builtin_vsx_xvcvdpsxws(a); + npyv_s32 hi_odd = __builtin_vsx_xvcvdpsxws(b); + #else // gcc + npyv_s32 lo_odd, hi_odd; + __asm__ ("xvcvdpsxws %x0,%x1" : "=wa" (lo_odd) : "wa" (a)); + __asm__ ("xvcvdpsxws %x0,%x1" : "=wa" (hi_odd) : "wa" (b)); + #endif + return vec_perm(lo_odd, hi_odd, seq_odd); +#endif +} + +// round to nearest integer (assuming even) +NPY_FINLINE npyv_s32 npyv_round_s32_f32(npyv_f32 a) +{ return npyv__trunc_s32_f32(vec_rint(a)); } + +NPY_FINLINE npyv_s32 npyv_round_s32_f64(npyv_f64 a, npyv_f64 b) +{ return npyv__trunc_s32_f64(vec_rint(a), vec_rint(b)); } + #endif // _NPY_SIMD_VSX_CVT_H diff --git a/numpy/core/src/common/simd/vsx/math.h b/numpy/core/src/common/simd/vsx/math.h index 7c8610b197eb..b2e393c7cf77 100644 --- a/numpy/core/src/common/simd/vsx/math.h +++ b/numpy/core/src/common/simd/vsx/math.h @@ -33,4 +33,40 @@ NPY_FINLINE npyv_f32 npyv_square_f32(npyv_f32 a) NPY_FINLINE npyv_f64 npyv_square_f64(npyv_f64 a) { return vec_mul(a, a); } +// Maximum, natively mapping with no guarantees to handle NaN. +#define npyv_max_f32 vec_max +#define npyv_max_f64 vec_max +// Maximum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +#define npyv_maxp_f32 vec_max +#define npyv_maxp_f64 vec_max +// Maximum, integer operations +#define npyv_max_u8 vec_max +#define npyv_max_s8 vec_max +#define npyv_max_u16 vec_max +#define npyv_max_s16 vec_max +#define npyv_max_u32 vec_max +#define npyv_max_s32 vec_max +#define npyv_max_u64 vec_max +#define npyv_max_s64 vec_max + +// Minimum, natively mapping with no guarantees to handle NaN. +#define npyv_min_f32 vec_min +#define npyv_min_f64 vec_min +// Minimum, supports IEEE floating-point arithmetic (IEC 60559), +// - If one of the two vectors contains NaN, the equivalent element of the other vector is set +// - Only if both corresponded elements are NaN, NaN is set. +#define npyv_minp_f32 vec_min +#define npyv_minp_f64 vec_min +// Minimum, integer operations +#define npyv_min_u8 vec_min +#define npyv_min_s8 vec_min +#define npyv_min_u16 vec_min +#define npyv_min_s16 vec_min +#define npyv_min_u32 vec_min +#define npyv_min_s32 vec_min +#define npyv_min_u64 vec_min +#define npyv_min_s64 vec_min + #endif // _NPY_SIMD_VSX_MATH_H diff --git a/numpy/core/src/common/simd/vsx/operators.h b/numpy/core/src/common/simd/vsx/operators.h index ca020d9e0e8c..23c5d0dbe70c 100644 --- a/numpy/core/src/common/simd/vsx/operators.h +++ b/numpy/core/src/common/simd/vsx/operators.h @@ -44,7 +44,16 @@ /*************************** * Logical ***************************/ +#define NPYV_IMPL_VSX_BIN_CAST(INTRIN, SFX, CAST) \ + NPY_FINLINE npyv_##SFX npyv_##INTRIN##_##SFX(npyv_##SFX a, npyv_##SFX b) \ + { return (npyv_##SFX)vec_##INTRIN((CAST)a, (CAST)b); } +// Up to GCC 6 logical intrinsics don't support bool long long +#if defined(__GNUC__) && __GNUC__ <= 6 + #define NPYV_IMPL_VSX_BIN_B64(INTRIN) NPYV_IMPL_VSX_BIN_CAST(INTRIN, b64, npyv_u64) +#else + #define NPYV_IMPL_VSX_BIN_B64(INTRIN) NPYV_IMPL_VSX_BIN_CAST(INTRIN, b64, npyv_b64) +#endif // AND #define npyv_and_u8 vec_and #define npyv_and_s8 vec_and @@ -56,6 +65,10 @@ #define npyv_and_s64 vec_and #define npyv_and_f32 vec_and #define npyv_and_f64 vec_and +#define npyv_and_b8 vec_and +#define npyv_and_b16 vec_and +#define npyv_and_b32 vec_and +NPYV_IMPL_VSX_BIN_B64(and) // OR #define npyv_or_u8 vec_or @@ -68,6 +81,10 @@ #define npyv_or_s64 vec_or #define npyv_or_f32 vec_or #define npyv_or_f64 vec_or +#define npyv_or_b8 vec_or +#define npyv_or_b16 vec_or +#define npyv_or_b32 vec_or +NPYV_IMPL_VSX_BIN_B64(or) // XOR #define npyv_xor_u8 vec_xor @@ -80,6 +97,10 @@ #define npyv_xor_s64 vec_xor #define npyv_xor_f32 vec_xor #define npyv_xor_f64 vec_xor +#define npyv_xor_b8 vec_xor +#define npyv_xor_b16 vec_xor +#define npyv_xor_b32 vec_xor +NPYV_IMPL_VSX_BIN_B64(xor) // NOT // note: we implement npyv_not_b*(boolen types) for internal use*/ @@ -129,7 +150,8 @@ NPY_FINLINE npyv_f64 npyv_not_f64(npyv_f64 a) #define npyv_cmpeq_f64 vec_cmpeq // Int Not Equal -#ifdef NPY_HAVE_VSX3 +#if defined(NPY_HAVE_VSX3) && (!defined(__GNUC__) || defined(vec_cmpne)) + // vec_cmpne supported by gcc since version 7 #define npyv_cmpneq_u8 vec_cmpne #define npyv_cmpneq_s8 vec_cmpne #define npyv_cmpneq_u16 vec_cmpne @@ -213,4 +235,10 @@ NPY_FINLINE npyv_f64 npyv_not_f64(npyv_f64 a) #define npyv_cmple_f32(A, B) npyv_cmpge_f32(B, A) #define npyv_cmple_f64(A, B) npyv_cmpge_f64(B, A) +// check special cases +NPY_FINLINE npyv_b32 npyv_notnan_f32(npyv_f32 a) +{ return vec_cmpeq(a, a); } +NPY_FINLINE npyv_b64 npyv_notnan_f64(npyv_f64 a) +{ return vec_cmpeq(a, a); } + #endif // _NPY_SIMD_VSX_OPERATORS_H diff --git a/numpy/core/src/common/simd/vsx/reorder.h b/numpy/core/src/common/simd/vsx/reorder.h index bfb9115fac81..6533e50933d2 100644 --- a/numpy/core/src/common/simd/vsx/reorder.h +++ b/numpy/core/src/common/simd/vsx/reorder.h @@ -62,4 +62,45 @@ NPYV_IMPL_VSX_COMBINE_ZIP(npyv_s64, s64) NPYV_IMPL_VSX_COMBINE_ZIP(npyv_f32, f32) NPYV_IMPL_VSX_COMBINE_ZIP(npyv_f64, f64) +// Reverse elements of each 64-bit lane +NPY_FINLINE npyv_u8 npyv_rev64_u8(npyv_u8 a) +{ +#if defined(NPY_HAVE_VSX3) && ((defined(__GNUC__) && __GNUC__ > 7) || defined(__IBMC__)) + return (npyv_u8)vec_revb((npyv_u64)a); +#elif defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX_ASM) + npyv_u8 ret; + __asm__ ("xxbrd %x0,%x1" : "=wa" (ret) : "wa" (a)); + return ret; +#else + const npyv_u8 idx = npyv_set_u8( + 7, 6, 5, 4, 3, 2, 1, 0,/*64*/15, 14, 13, 12, 11, 10, 9, 8 + ); + return vec_perm(a, a, idx); +#endif +} +NPY_FINLINE npyv_s8 npyv_rev64_s8(npyv_s8 a) +{ return (npyv_s8)npyv_rev64_u8((npyv_u8)a); } + +NPY_FINLINE npyv_u16 npyv_rev64_u16(npyv_u16 a) +{ + const npyv_u8 idx = npyv_set_u8( + 6, 7, 4, 5, 2, 3, 0, 1,/*64*/14, 15, 12, 13, 10, 11, 8, 9 + ); + return vec_perm(a, a, idx); +} +NPY_FINLINE npyv_s16 npyv_rev64_s16(npyv_s16 a) +{ return (npyv_s16)npyv_rev64_u16((npyv_u16)a); } + +NPY_FINLINE npyv_u32 npyv_rev64_u32(npyv_u32 a) +{ + const npyv_u8 idx = npyv_set_u8( + 4, 5, 6, 7, 0, 1, 2, 3,/*64*/12, 13, 14, 15, 8, 9, 10, 11 + ); + return vec_perm(a, a, idx); +} +NPY_FINLINE npyv_s32 npyv_rev64_s32(npyv_s32 a) +{ return (npyv_s32)npyv_rev64_u32((npyv_u32)a); } +NPY_FINLINE npyv_f32 npyv_rev64_f32(npyv_f32 a) +{ return (npyv_f32)npyv_rev64_u32((npyv_u32)a); } + #endif // _NPY_SIMD_VSX_REORDER_H diff --git a/numpy/core/src/common/simd/vsx/vsx.h b/numpy/core/src/common/simd/vsx/vsx.h index 27dde98e7c8a..66b76208f042 100644 --- a/numpy/core/src/common/simd/vsx/vsx.h +++ b/numpy/core/src/common/simd/vsx/vsx.h @@ -2,9 +2,20 @@ #error "Not a standalone header" #endif +#if defined(__GNUC__) && __GNUC__ <= 7 + /** + * GCC <= 7 produces ambiguous warning caused by -Werror=maybe-uninitialized, + * when certain intrinsics involved. `vec_ld` is one of them but it seemed to work fine, + * and suppressing the warning wouldn't affect its functionality. + */ + #pragma GCC diagnostic ignored "-Wuninitialized" + #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + #define NPY_SIMD 128 #define NPY_SIMD_WIDTH 16 #define NPY_SIMD_F64 1 +#define NPY_SIMD_FMA3 1 // native support typedef __vector unsigned char npyv_u8; typedef __vector signed char npyv_s8; diff --git a/numpy/core/src/multiarray/_multiarray_tests.c.src b/numpy/core/src/multiarray/_multiarray_tests.c.src index 3811e87a8e34..bfdeae07932e 100644 --- a/numpy/core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/core/src/multiarray/_multiarray_tests.c.src @@ -7,6 +7,7 @@ #include "numpy/npy_math.h" #include "numpy/halffloat.h" #include "common.h" +#include "npy_argparse.h" #include "mem_overlap.h" #include "npy_extint128.h" #include "array_method.h" @@ -19,6 +20,25 @@ #define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) + +static PyObject * +argparse_example_function(PyObject *NPY_UNUSED(mod), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) +{ + NPY_PREPARE_ARGPARSER; + int arg1; + PyObject *arg2, *arg3, *arg4; + if (npy_parse_arguments("func", args, len_args, kwnames, + "", &PyArray_PythonPyIntFromInt, &arg1, + "arg2", NULL, &arg2, + "|arg3", NULL, &arg3, + "$arg3", NULL, &arg4, + NULL, NULL, NULL) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + /* test PyArray_IsPythonScalar, before including private py3 compat header */ static PyObject * IsPythonScalar(PyObject * dummy, PyObject *args) @@ -387,7 +407,7 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) niterx2 = (PyArrayNeighborhoodIterObject*)PyArray_NeighborhoodIterNew( (PyArrayIterObject*)niterx1, bounds, mode2, NULL); - if (niterx1 == NULL) { + if (niterx2 == NULL) { goto clean_niterx1; } @@ -677,11 +697,12 @@ create_custom_field_dtype(PyObject *NPY_UNUSED(mod), PyObject *args) "invalid error argument to test function."); } if (PyArray_RegisterDataType(dtype) < 0) { - /* Fix original type in the error_path == 2 case. */ + /* Fix original type in the error_path == 2 case and delete it */ Py_SET_TYPE(dtype, original_type); + Py_DECREF(dtype); return NULL; } - Py_INCREF(dtype); + Py_INCREF(dtype); /* hold on to the original (leaks a reference) */ return (PyObject *)dtype; } @@ -2119,17 +2140,6 @@ getset_numericops(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) } -static PyObject * -uses_new_casts(PyObject* NPY_UNUSED(self), PyObject* NPY_UNUSED(args)) -{ -#if NPY_USE_NEW_CASTINGIMPL - Py_RETURN_TRUE; -#else - Py_RETURN_FALSE; -#endif -} - - static PyObject * run_byteorder_converter(PyObject* NPY_UNUSED(self), PyObject *args) { @@ -2254,6 +2264,9 @@ run_intp_converter(PyObject* NPY_UNUSED(self), PyObject *args) } static PyMethodDef Multiarray_TestsMethods[] = { + {"argparse_example_function", + (PyCFunction)argparse_example_function, + METH_KEYWORDS | METH_FASTCALL, NULL}, {"IsPythonScalar", IsPythonScalar, METH_VARARGS, NULL}, @@ -2383,9 +2396,6 @@ static PyMethodDef Multiarray_TestsMethods[] = { {"getset_numericops", getset_numericops, METH_NOARGS, NULL}, - {"uses_new_casts", - uses_new_casts, - METH_NOARGS, NULL}, /**begin repeat * #name = cabs, carg# */ diff --git a/numpy/core/src/multiarray/abstractdtypes.c b/numpy/core/src/multiarray/abstractdtypes.c index 02c0eac53d7b..587d91c49cda 100644 --- a/numpy/core/src/multiarray/abstractdtypes.c +++ b/numpy/core/src/multiarray/abstractdtypes.c @@ -13,6 +13,12 @@ #include "common.h" +static NPY_INLINE PyArray_Descr * +int_default_descriptor(PyArray_DTypeMeta* NPY_UNUSED(cls)) +{ + return PyArray_DescrFromType(NPY_LONG); +} + static PyArray_Descr * discover_descriptor_from_pyint( PyArray_DTypeMeta *NPY_UNUSED(cls), PyObject *obj) @@ -45,6 +51,13 @@ discover_descriptor_from_pyint( } +static NPY_INLINE PyArray_Descr * +float_default_descriptor(PyArray_DTypeMeta* NPY_UNUSED(cls)) +{ + return PyArray_DescrFromType(NPY_DOUBLE); +} + + static PyArray_Descr* discover_descriptor_from_pyfloat( PyArray_DTypeMeta* NPY_UNUSED(cls), PyObject *obj) @@ -53,6 +66,11 @@ discover_descriptor_from_pyfloat( return PyArray_DescrFromType(NPY_DOUBLE); } +static NPY_INLINE PyArray_Descr * +complex_default_descriptor(PyArray_DTypeMeta* NPY_UNUSED(cls)) +{ + return PyArray_DescrFromType(NPY_CDOUBLE); +} static PyArray_Descr* discover_descriptor_from_pycomplex( @@ -66,21 +84,17 @@ discover_descriptor_from_pycomplex( NPY_NO_EXPORT int initialize_and_map_pytypes_to_dtypes() { - PyArrayAbstractObjDTypeMeta_Type.tp_base = &PyArrayDTypeMeta_Type; - if (PyType_Ready(&PyArrayAbstractObjDTypeMeta_Type) < 0) { - return -1; - } - ((PyTypeObject *)&PyArray_PyIntAbstractDType)->tp_base = &PyArrayDTypeMeta_Type; + ((PyTypeObject *)&PyArray_PyIntAbstractDType)->tp_base = &PyArrayDescr_Type; PyArray_PyIntAbstractDType.scalar_type = &PyLong_Type; if (PyType_Ready((PyTypeObject *)&PyArray_PyIntAbstractDType) < 0) { return -1; } - ((PyTypeObject *)&PyArray_PyFloatAbstractDType)->tp_base = &PyArrayDTypeMeta_Type; + ((PyTypeObject *)&PyArray_PyFloatAbstractDType)->tp_base = &PyArrayDescr_Type; PyArray_PyFloatAbstractDType.scalar_type = &PyFloat_Type; if (PyType_Ready((PyTypeObject *)&PyArray_PyFloatAbstractDType) < 0) { return -1; } - ((PyTypeObject *)&PyArray_PyComplexAbstractDType)->tp_base = &PyArrayDTypeMeta_Type; + ((PyTypeObject *)&PyArray_PyComplexAbstractDType)->tp_base = &PyArrayDescr_Type; PyArray_PyComplexAbstractDType.scalar_type = &PyComplex_Type; if (PyType_Ready((PyTypeObject *)&PyArray_PyComplexAbstractDType) < 0) { return -1; @@ -126,43 +140,147 @@ initialize_and_map_pytypes_to_dtypes() } +/* + * The following functions define the "common DType" for the abstract dtypes. + * + * Note that the logic with respect to the "higher" dtypes such as floats + * could likely be more logically defined for them, but since NumPy dtypes + * largely "know" each other, that is not necessary. + */ +static PyArray_DTypeMeta * +int_common_dtype(PyArray_DTypeMeta *NPY_UNUSED(cls), PyArray_DTypeMeta *other) +{ + if (other->legacy && other->type_num < NPY_NTYPES) { + if (other->type_num == NPY_BOOL) { + /* Use the default integer for bools: */ + return PyArray_DTypeFromTypeNum(NPY_LONG); + } + else if (PyTypeNum_ISNUMBER(other->type_num) || + other->type_num == NPY_TIMEDELTA) { + /* All other numeric types (ant timdelta) are preserved: */ + Py_INCREF(other); + return other; + } + } + else if (other->legacy) { + /* This is a back-compat fallback to usually do the right thing... */ + return PyArray_DTypeFromTypeNum(NPY_UINT8); + } + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} -/* Note: This is currently largely not used, but will be required eventually. */ -NPY_NO_EXPORT PyTypeObject PyArrayAbstractObjDTypeMeta_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "numpy._AbstractObjDTypeMeta", - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_doc = "Helper MetaClass for value based casting AbstractDTypes.", -}; +static PyArray_DTypeMeta * +float_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + if (other->legacy && other->type_num < NPY_NTYPES) { + if (other->type_num == NPY_BOOL || PyTypeNum_ISINTEGER(other->type_num)) { + /* Use the default integer for bools and ints: */ + return PyArray_DTypeFromTypeNum(NPY_DOUBLE); + } + else if (PyTypeNum_ISNUMBER(other->type_num)) { + /* All other numeric types (float+complex) are preserved: */ + Py_INCREF(other); + return other; + } + } + else if (other == &PyArray_PyIntAbstractDType) { + Py_INCREF(cls); + return cls; + } + else if (other->legacy) { + /* This is a back-compat fallback to usually do the right thing... */ + return PyArray_DTypeFromTypeNum(NPY_HALF); + } + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} + + +static PyArray_DTypeMeta * +complex_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) +{ + if (other->legacy && other->type_num < NPY_NTYPES) { + if (other->type_num == NPY_BOOL || + PyTypeNum_ISINTEGER(other->type_num)) { + /* Use the default integer for bools and ints: */ + return PyArray_DTypeFromTypeNum(NPY_CDOUBLE); + } + else if (PyTypeNum_ISFLOAT(other->type_num)) { + /* + * For floats we choose the equivalent precision complex, although + * there is no CHALF, so half also goes to CFLOAT. + */ + if (other->type_num == NPY_HALF || other->type_num == NPY_FLOAT) { + return PyArray_DTypeFromTypeNum(NPY_CFLOAT); + } + if (other->type_num == NPY_DOUBLE) { + return PyArray_DTypeFromTypeNum(NPY_CDOUBLE); + } + assert(other->type_num == NPY_LONGDOUBLE); + return PyArray_DTypeFromTypeNum(NPY_CLONGDOUBLE); + } + else if (PyTypeNum_ISCOMPLEX(other->type_num)) { + /* All other numeric types are preserved: */ + Py_INCREF(other); + return other; + } + } + else if (other->legacy) { + /* This is a back-compat fallback to usually do the right thing... */ + return PyArray_DTypeFromTypeNum(NPY_CFLOAT); + } + else if (other == &PyArray_PyIntAbstractDType || + other == &PyArray_PyFloatAbstractDType) { + Py_INCREF(cls); + return cls; + } + Py_INCREF(Py_NotImplemented); + return (PyArray_DTypeMeta *)Py_NotImplemented; +} + + +/* + * TODO: These abstract DTypes also carry the dual role of representing + * `Floating`, `Complex`, and `Integer` (both signed and unsigned). + * They will have to be renamed and exposed in that capacity. + */ NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyIntAbstractDType = {{{ - PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0) - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_name = "numpy._PyIntBaseAbstractDType", + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_name = "numpy._IntegerAbstractDType", },}, .abstract = 1, + .default_descr = int_default_descriptor, .discover_descr_from_pyobject = discover_descriptor_from_pyint, + .common_dtype = int_common_dtype, .kind = 'i', }; NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyFloatAbstractDType = {{{ - PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0) - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_name = "numpy._PyFloatBaseAbstractDType", + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_name = "numpy._FloatAbstractDType", },}, .abstract = 1, + .default_descr = float_default_descriptor, .discover_descr_from_pyobject = discover_descriptor_from_pyfloat, + .common_dtype = float_common_dtype, .kind = 'f', }; NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyComplexAbstractDType = {{{ - PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0) - .tp_basicsize = sizeof(PyArray_DTypeMeta), - .tp_name = "numpy._PyComplexBaseAbstractDType", + PyVarObject_HEAD_INIT(&PyArrayDTypeMeta_Type, 0) + .tp_basicsize = sizeof(PyArray_Descr), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_name = "numpy._ComplexAbstractDType", },}, .abstract = 1, + .default_descr = complex_default_descriptor, .discover_descr_from_pyobject = discover_descriptor_from_pycomplex, + .common_dtype = complex_common_dtype, .kind = 'c', }; - diff --git a/numpy/core/src/multiarray/abstractdtypes.h b/numpy/core/src/multiarray/abstractdtypes.h index 3a982cd38a0e..a6c526717032 100644 --- a/numpy/core/src/multiarray/abstractdtypes.h +++ b/numpy/core/src/multiarray/abstractdtypes.h @@ -3,12 +3,12 @@ #include "dtypemeta.h" + /* * These are mainly needed for value based promotion in ufuncs. It * may be necessary to make them (partially) public, to allow user-defined * dtypes to perform value based casting. */ -NPY_NO_EXPORT extern PyTypeObject PyArrayAbstractObjDTypeMeta_Type; NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyIntAbstractDType; NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyFloatAbstractDType; NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyComplexAbstractDType; diff --git a/numpy/core/src/multiarray/array_assign_array.c b/numpy/core/src/multiarray/array_assign_array.c index 361964a5cdf3..665dadfbfb64 100644 --- a/numpy/core/src/multiarray/array_assign_array.c +++ b/numpy/core/src/multiarray/array_assign_array.c @@ -23,6 +23,7 @@ #include "lowlevel_strided_loops.h" #include "array_assign.h" +#include "dtype_transfer.h" /* * Check that array data is both uint-aligned and true-aligned for all array @@ -82,10 +83,7 @@ raw_array_assign_array(int ndim, npy_intp const *shape, npy_intp src_strides_it[NPY_MAXDIMS]; npy_intp coord[NPY_MAXDIMS]; - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; int aligned, needs_api = 0; - npy_intp src_itemsize = src_dtype->elsize; NPY_BEGIN_THREADS_DEF; @@ -117,12 +115,12 @@ raw_array_assign_array(int ndim, npy_intp const *shape, } /* Get the function to do the casting */ + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction(aligned, src_strides_it[0], dst_strides_it[0], src_dtype, dst_dtype, 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { + &cast_info, &needs_api) != NPY_SUCCEED) { return -1; } @@ -130,11 +128,13 @@ raw_array_assign_array(int ndim, npy_intp const *shape, NPY_BEGIN_THREADS; } + npy_intp strides[2] = {src_strides_it[0], dst_strides_it[0]}; + NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ - if (stransfer( - dst_data, dst_strides_it[0], src_data, src_strides_it[0], - shape_it[0], src_itemsize, transferdata) < 0) { + char *args[2] = {src_data, dst_data}; + if (cast_info.func(&cast_info.context, + args, &shape_it[0], strides, cast_info.auxdata) < 0) { goto fail; } } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, @@ -142,11 +142,11 @@ raw_array_assign_array(int ndim, npy_intp const *shape, src_data, src_strides_it); NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return 0; fail: NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } @@ -170,10 +170,7 @@ raw_array_wheremasked_assign_array(int ndim, npy_intp const *shape, npy_intp wheremask_strides_it[NPY_MAXDIMS]; npy_intp coord[NPY_MAXDIMS]; - PyArray_MaskedStridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; int aligned, needs_api = 0; - npy_intp src_itemsize = src_dtype->elsize; NPY_BEGIN_THREADS_DEF; @@ -209,35 +206,41 @@ raw_array_wheremasked_assign_array(int ndim, npy_intp const *shape, } /* Get the function to do the casting */ + NPY_cast_info cast_info; if (PyArray_GetMaskedDTypeTransferFunction(aligned, src_strides_it[0], dst_strides_it[0], wheremask_strides_it[0], src_dtype, dst_dtype, wheremask_dtype, 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { + &cast_info, &needs_api) != NPY_SUCCEED) { return -1; } if (!needs_api) { NPY_BEGIN_THREADS; } + npy_intp strides[2] = {src_strides_it[0], dst_strides_it[0]}; NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { + PyArray_MaskedStridedUnaryOp *stransfer; + stransfer = (PyArray_MaskedStridedUnaryOp *)cast_info.func; + /* Process the innermost dimension */ - stransfer(dst_data, dst_strides_it[0], src_data, src_strides_it[0], - (npy_bool *)wheremask_data, wheremask_strides_it[0], - shape_it[0], src_itemsize, transferdata); + char *args[2] = {src_data, dst_data}; + if (stransfer(&cast_info.context, + args, &shape_it[0], strides, + (npy_bool *)wheremask_data, wheremask_strides_it[0], + cast_info.auxdata) < 0) { + break; + } } NPY_RAW_ITER_THREE_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it, src_data, src_strides_it, wheremask_data, wheremask_strides_it); NPY_END_THREADS; - - NPY_AUXDATA_FREE(transferdata); - + NPY_cast_info_xfree(&cast_info); return (needs_api && PyErr_Occurred()) ? -1 : 0; } diff --git a/numpy/core/src/multiarray/array_assign_scalar.c b/numpy/core/src/multiarray/array_assign_scalar.c index 023772776429..6cd5f4ad9be5 100644 --- a/numpy/core/src/multiarray/array_assign_scalar.c +++ b/numpy/core/src/multiarray/array_assign_scalar.c @@ -23,6 +23,7 @@ #include "lowlevel_strided_loops.h" #include "array_assign.h" +#include "dtype_transfer.h" /* * Assigns the scalar value to every element of the destination raw array. @@ -38,10 +39,7 @@ raw_array_assign_scalar(int ndim, npy_intp const *shape, npy_intp shape_it[NPY_MAXDIMS], dst_strides_it[NPY_MAXDIMS]; npy_intp coord[NPY_MAXDIMS]; - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; int aligned, needs_api = 0; - npy_intp src_itemsize = src_dtype->elsize; NPY_BEGIN_THREADS_DEF; @@ -63,12 +61,12 @@ raw_array_assign_scalar(int ndim, npy_intp const *shape, } /* Get the function to do the casting */ + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction(aligned, 0, dst_strides_it[0], src_dtype, dst_dtype, 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { + &cast_info, &needs_api) != NPY_SUCCEED) { return -1; } @@ -80,22 +78,24 @@ raw_array_assign_scalar(int ndim, npy_intp const *shape, NPY_BEGIN_THREADS_THRESHOLDED(nitems); } + npy_intp strides[2] = {0, dst_strides_it[0]}; + NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ - if (stransfer( - dst_data, dst_strides_it[0], src_data, 0, - shape_it[0], src_itemsize, transferdata) < 0) { + char *args[2] = {src_data, dst_data}; + if (cast_info.func(&cast_info.context, + args, &shape_it[0], strides, cast_info.auxdata) < 0) { goto fail; } } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it); NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return 0; fail: NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } @@ -117,10 +117,7 @@ raw_array_wheremasked_assign_scalar(int ndim, npy_intp const *shape, npy_intp wheremask_strides_it[NPY_MAXDIMS]; npy_intp coord[NPY_MAXDIMS]; - PyArray_MaskedStridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; int aligned, needs_api = 0; - npy_intp src_itemsize = src_dtype->elsize; NPY_BEGIN_THREADS_DEF; @@ -144,12 +141,12 @@ raw_array_wheremasked_assign_scalar(int ndim, npy_intp const *shape, } /* Get the function to do the casting */ + NPY_cast_info cast_info; if (PyArray_GetMaskedDTypeTransferFunction(aligned, 0, dst_strides_it[0], wheremask_strides_it[0], src_dtype, dst_dtype, wheremask_dtype, 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { + &cast_info, &needs_api) != NPY_SUCCEED) { return -1; } @@ -161,19 +158,26 @@ raw_array_wheremasked_assign_scalar(int ndim, npy_intp const *shape, NPY_BEGIN_THREADS_THRESHOLDED(nitems); } + npy_intp strides[2] = {0, dst_strides_it[0]}; + NPY_RAW_ITER_START(idim, ndim, coord, shape_it) { /* Process the innermost dimension */ - stransfer(dst_data, dst_strides_it[0], src_data, 0, - (npy_bool *)wheremask_data, wheremask_strides_it[0], - shape_it[0], src_itemsize, transferdata); + PyArray_MaskedStridedUnaryOp *stransfer; + stransfer = (PyArray_MaskedStridedUnaryOp *)cast_info.func; + + char *args[2] = {src_data, dst_data}; + if (stransfer(&cast_info.context, + args, &shape_it[0], strides, + (npy_bool *)wheremask_data, wheremask_strides_it[0], + cast_info.auxdata) < 0) { + break; + } } NPY_RAW_ITER_TWO_NEXT(idim, ndim, coord, shape_it, dst_data, dst_strides_it, wheremask_data, wheremask_strides_it); NPY_END_THREADS; - - NPY_AUXDATA_FREE(transferdata); - + NPY_cast_info_xfree(&cast_info); return (needs_api && PyErr_Occurred()) ? -1 : 0; } diff --git a/numpy/core/src/multiarray/array_coercion.c b/numpy/core/src/multiarray/array_coercion.c index 53d891049e23..22050a56ff6b 100644 --- a/numpy/core/src/multiarray/array_coercion.c +++ b/numpy/core/src/multiarray/array_coercion.c @@ -11,6 +11,7 @@ #include "descriptor.h" #include "convert_datatype.h" +#include "common_dtype.h" #include "dtypemeta.h" #include "array_coercion.h" @@ -205,7 +206,7 @@ _PyArray_MapPyTypeToDType( * @return DType, None if it a known non-scalar, or NULL if an unknown object. */ static NPY_INLINE PyArray_DTypeMeta * -discover_dtype_from_pytype(PyTypeObject *pytype) +npy_discover_dtype_from_pytype(PyTypeObject *pytype) { PyObject *DType; @@ -263,7 +264,7 @@ discover_dtype_from_pyobject( } } - PyArray_DTypeMeta *DType = discover_dtype_from_pytype(Py_TYPE(obj)); + PyArray_DTypeMeta *DType = npy_discover_dtype_from_pytype(Py_TYPE(obj)); if (DType != NULL) { return DType; } @@ -452,18 +453,21 @@ PyArray_Pack(PyArray_Descr *descr, char *item, PyObject *value) int res = 0; int needs_api = 0; - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *transferdata; + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction( - 0, 0, 0, tmp_descr, descr, 0, &stransfer, &transferdata, + 0, 0, 0, tmp_descr, descr, 0, &cast_info, &needs_api) == NPY_FAIL) { res = -1; goto finish; } - if (stransfer(item, 0, data, 0, 1, tmp_descr->elsize, transferdata) < 0) { + char *args[2] = {data, item}; + const npy_intp strides[2] = {0, 0}; + const npy_intp length = 1; + if (cast_info.func(&cast_info.context, + args, &length, strides, cast_info.auxdata) < 0) { res = -1; } - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); finish: if (PyDataType_REFCHK(tmp_descr)) { @@ -570,7 +574,7 @@ npy_new_coercion_cache( * @param current * @return next coercion cache object (or NULL) */ -NPY_NO_EXPORT NPY_INLINE coercion_cache_obj * +NPY_NO_EXPORT coercion_cache_obj * npy_unlink_coercion_cache(coercion_cache_obj *current) { coercion_cache_obj *next = current->next; @@ -585,7 +589,7 @@ npy_unlink_coercion_cache(coercion_cache_obj *current) return next; } -NPY_NO_EXPORT NPY_INLINE void +NPY_NO_EXPORT void npy_free_coercion_cache(coercion_cache_obj *next) { /* We only need to check from the last used cache pos */ while (next != NULL) { @@ -619,8 +623,12 @@ handle_promotion(PyArray_Descr **out_descr, PyArray_Descr *descr, } PyArray_Descr *new_descr = PyArray_PromoteTypes(descr, *out_descr); if (NPY_UNLIKELY(new_descr == NULL)) { - if (fixed_DType != NULL) { - /* If a DType is fixed, promotion must not fail. */ + if (fixed_DType != NULL || PyErr_ExceptionMatches(PyExc_FutureWarning)) { + /* + * If a DType is fixed, promotion must not fail. Do not catch + * FutureWarning (raised for string+numeric promotions). We could + * only catch TypeError here or even always raise the error. + */ return -1; } PyErr_Clear(); @@ -745,6 +753,7 @@ find_descriptor_from_array( NULL, DType, &flags, item_DType) < 0) { Py_DECREF(iter); Py_DECREF(elem); + Py_XDECREF(*out_descr); Py_XDECREF(item_DType); return -1; } @@ -922,6 +931,53 @@ PyArray_DiscoverDTypeAndShape_Recursive( Py_DECREF(arr); arr = NULL; } + else if (curr_dims > 0 && curr_dims != max_dims) { + /* + * Deprecated 2020-12-09, NumPy 1.20 + * + * See https://github.com/numpy/numpy/issues/17965 + * Shapely had objects which are not sequences but did export + * the array-interface (and so are arguably array-like). + * Previously numpy would not use array-like information during + * shape discovery, so that it ended up acting as if this was + * an (unknown) scalar but with the specified dtype. + * Thus we ignore "scalars" here, as the value stored in the + * array should be acceptable. + */ + if (PyArray_NDIM(arr) > 0 && NPY_UNLIKELY(!PySequence_Check(obj))) { + if (PyErr_WarnFormat(PyExc_FutureWarning, 1, + "The input object of type '%s' is an array-like " + "implementing one of the corresponding protocols " + "(`__array__`, `__array_interface__` or " + "`__array_struct__`); but not a sequence (or 0-D). " + "In the future, this object will be coerced as if it " + "was first converted using `np.array(obj)`. " + "To retain the old behaviour, you have to either " + "modify the type '%s', or assign to an empty array " + "created with `np.empty(correct_shape, dtype=object)`.", + Py_TYPE(obj)->tp_name, Py_TYPE(obj)->tp_name) < 0) { + Py_DECREF(arr); + return -1; + } + /* + * Strangely enough, even though we threw away the result here, + * we did use it during descriptor discovery, so promote it: + */ + if (update_shape(curr_dims, &max_dims, out_shape, + 0, NULL, NPY_FALSE, flags) < 0) { + *flags |= FOUND_RAGGED_ARRAY; + Py_DECREF(arr); + return max_dims; + } + if (!(*flags & DESCRIPTOR_WAS_SET) && handle_promotion( + out_descr, PyArray_DESCR(arr), fixed_DType, flags) < 0) { + Py_DECREF(arr); + return -1; + } + Py_DECREF(arr); + return max_dims; + } + } } if (arr != NULL) { /* @@ -979,14 +1035,28 @@ PyArray_DiscoverDTypeAndShape_Recursive( * and to handle it recursively. That is, unless we have hit the * dimension limit. */ - npy_bool is_sequence = (PySequence_Check(obj) && PySequence_Size(obj) >= 0); + npy_bool is_sequence = PySequence_Check(obj); + if (is_sequence) { + is_sequence = PySequence_Size(obj) >= 0; + if (NPY_UNLIKELY(!is_sequence)) { + /* NOTE: This should likely just raise all errors */ + if (PyErr_ExceptionMatches(PyExc_RecursionError) || + PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* + * Consider these unrecoverable errors, continuing execution + * might crash the interpreter. + */ + return -1; + } + PyErr_Clear(); + } + } if (NPY_UNLIKELY(*flags & DISCOVER_TUPLES_AS_ELEMENTS) && PyTuple_Check(obj)) { is_sequence = NPY_FALSE; } if (curr_dims == max_dims || !is_sequence) { /* Clear any PySequence_Size error which would corrupts further calls */ - PyErr_Clear(); max_dims = handle_scalar( obj, curr_dims, &max_dims, out_descr, out_shape, fixed_DType, flags, NULL); @@ -1037,6 +1107,11 @@ PyArray_DiscoverDTypeAndShape_Recursive( return curr_dims + 1; } + /* Allow keyboard interrupts. See gh issue 18117. */ + if (PyErr_CheckSignals() < 0) { + return -1; + } + /* Recursive call for each sequence item */ for (Py_ssize_t i = 0; i < size; i++) { max_dims = PyArray_DiscoverDTypeAndShape_Recursive( diff --git a/numpy/core/src/multiarray/array_coercion.h b/numpy/core/src/multiarray/array_coercion.h index 90ce0355a11c..c5ccad225549 100644 --- a/numpy/core/src/multiarray/array_coercion.h +++ b/numpy/core/src/multiarray/array_coercion.h @@ -15,7 +15,6 @@ typedef struct coercion_cache_obj { int depth; /* the dimension at which this object was found. */ } coercion_cache_obj; - NPY_NO_EXPORT int _PyArray_MapPyTypeToDType( PyArray_DTypeMeta *DType, PyTypeObject *pytype, npy_bool userdef); diff --git a/numpy/core/src/multiarray/array_method.c b/numpy/core/src/multiarray/array_method.c index cae452454811..3ecc20d1d60c 100644 --- a/numpy/core/src/multiarray/array_method.c +++ b/numpy/core/src/multiarray/array_method.c @@ -34,6 +34,7 @@ #include "arrayobject.h" #include "array_method.h" #include "dtypemeta.h" +#include "common_dtype.h" #include "convert_datatype.h" @@ -121,6 +122,19 @@ default_resolve_descriptors( } +NPY_INLINE static int +is_contiguous( + npy_intp const *strides, PyArray_Descr *const *descriptors, int nargs) +{ + for (int i = 0; i < nargs; i++) { + if (strides[i] != descriptors[i]->elsize) { + return 0; + } + } + return 1; +} + + /** * The default method to fetch the correct loop for a cast or ufunc * (at the time of writing only casts). @@ -138,18 +152,36 @@ default_resolve_descriptors( * @param flags * @return 0 on success -1 on failure. */ -static int -default_get_strided_loop( - PyArrayMethod_Context *NPY_UNUSED(context), - int NPY_UNUSED(aligned), int NPY_UNUSED(move_references), - npy_intp *NPY_UNUSED(strides), - PyArray_StridedUnaryOp **NPY_UNUSED(out_loop), - NpyAuxData **NPY_UNUSED(out_transferdata), - NPY_ARRAYMETHOD_FLAGS *NPY_UNUSED(flags)) +NPY_NO_EXPORT int +npy_default_get_strided_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) { - PyErr_SetString(PyExc_NotImplementedError, - "default loop getter is not implemented"); - return -1; + PyArray_Descr **descrs = context->descriptors; + PyArrayMethodObject *meth = context->method; + *flags = meth->flags & NPY_METH_RUNTIME_FLAGS; + *out_transferdata = NULL; + + int nargs = meth->nin + meth->nout; + if (aligned) { + if (meth->contiguous_loop == NULL || + !is_contiguous(strides, descrs, nargs)) { + *out_loop = meth->strided_loop; + return 0; + } + *out_loop = meth->contiguous_loop; + } + else { + if (meth->unaligned_contiguous_loop == NULL || + !is_contiguous(strides, descrs, nargs)) { + *out_loop = meth->unaligned_strided_loop; + return 0; + } + *out_loop = meth->unaligned_contiguous_loop; + } + return 0; } @@ -178,10 +210,12 @@ validate_spec(PyArrayMethod_Spec *spec) case NPY_UNSAFE_CASTING: break; default: - PyErr_Format(PyExc_TypeError, - "ArrayMethod has invalid casting `%d`. (method: %s)", - spec->casting, spec->name); - return -1; + if (spec->casting != -1) { + PyErr_Format(PyExc_TypeError, + "ArrayMethod has invalid casting `%d`. (method: %s)", + spec->casting, spec->name); + return -1; + } } for (int i = 0; i < nargs; i++) { @@ -225,7 +259,7 @@ fill_arraymethod_from_slots( PyArrayMethodObject *meth = res->method; /* Set the defaults */ - meth->get_strided_loop = &default_get_strided_loop; + meth->get_strided_loop = &npy_default_get_strided_loop; meth->resolve_descriptors = &default_resolve_descriptors; /* Fill in the slots passed by the user */ @@ -269,6 +303,13 @@ fill_arraymethod_from_slots( /* Check whether the slots are valid: */ if (meth->resolve_descriptors == &default_resolve_descriptors) { + if (spec->casting == -1) { + PyErr_Format(PyExc_TypeError, + "Cannot set casting to -1 (invalid) when not providing " + "the default `resolve_descriptors` function. " + "(method: %s)", spec->name); + return -1; + } for (int i = 0; i < meth->nin + meth->nout; i++) { if (res->dtypes[i] == NULL) { if (i < meth->nin) { @@ -295,7 +336,7 @@ fill_arraymethod_from_slots( } } } - if (meth->get_strided_loop != &default_get_strided_loop) { + if (meth->get_strided_loop != &npy_default_get_strided_loop) { /* Do not check the actual loop fields. */ return 0; } @@ -391,7 +432,7 @@ PyArrayMethod_FromSpec_int(PyArrayMethod_Spec *spec, int private) return NULL; } - ssize_t length = strlen(spec->name); + Py_ssize_t length = strlen(spec->name); res->method->name = PyMem_Malloc(length + 1); if (res->method->name == NULL) { Py_DECREF(res); @@ -468,6 +509,9 @@ boundarraymethod_dealloc(PyObject *self) * May raise an error, but usually should not. * The function validates the casting attribute compared to the returned * casting level. + * + * TODO: This function is not public API, and certain code paths will need + * changes and especially testing if they were to be made public. */ static PyObject * boundarraymethod__resolve_descripors( @@ -481,7 +525,7 @@ boundarraymethod__resolve_descripors( if (!PyTuple_CheckExact(descr_tuple) || PyTuple_Size(descr_tuple) != nin + nout) { - PyErr_Format(PyExc_ValueError, + PyErr_Format(PyExc_TypeError, "_resolve_descriptors() takes exactly one tuple with as many " "elements as the method takes arguments (%d+%d).", nin, nout); return NULL; @@ -494,7 +538,7 @@ boundarraymethod__resolve_descripors( } else if (tmp == Py_None) { if (i < nin) { - PyErr_SetString(PyExc_ValueError, + PyErr_SetString(PyExc_TypeError, "only output dtypes may be omitted (set to None)."); return NULL; } @@ -502,7 +546,7 @@ boundarraymethod__resolve_descripors( } else if (PyArray_DescrCheck(tmp)) { if (Py_TYPE(tmp) != (PyTypeObject *)self->dtypes[i]) { - PyErr_Format(PyExc_ValueError, + PyErr_Format(PyExc_TypeError, "input dtype %S was not an exact instance of the bound " "DType class %S.", tmp, self->dtypes[i]); return NULL; @@ -538,6 +582,8 @@ boundarraymethod__resolve_descripors( /* * The casting flags should be the most generic casting level (except the * cast-is-view flag. If no input is parametric, it must match exactly. + * + * (Note that these checks are only debugging checks.) */ int parametric = 0; for (int i = 0; i < nin + nout; i++) { @@ -546,43 +592,177 @@ boundarraymethod__resolve_descripors( break; } } - if (!parametric) { - /* - * Non-parametric can only mismatch if it switches from no to equiv - * (e.g. due to byteorder changes). - */ - if (self->method->casting != (casting & ~_NPY_CAST_IS_VIEW) && - !(self->method->casting == NPY_NO_CASTING && - casting == NPY_EQUIV_CASTING)) { + if (self->method->casting != -1) { + NPY_CASTING cast = casting & ~_NPY_CAST_IS_VIEW; + if (self->method->casting != + PyArray_MinCastSafety(cast, self->method->casting)) { PyErr_Format(PyExc_RuntimeError, - "resolve_descriptors cast level did not match stored one " - "(expected %d, got %d) for method %s", - self->method->casting, (casting & ~_NPY_CAST_IS_VIEW), - self->method->name); + "resolve_descriptors cast level did not match stored one. " + "(set level is %d, got %d for method %s)", + self->method->casting, cast, self->method->name); Py_DECREF(result_tuple); return NULL; } + if (!parametric) { + /* + * Non-parametric can only mismatch if it switches from equiv to no + * (e.g. due to byteorder changes). + */ + if (cast != self->method->casting && + self->method->casting != NPY_EQUIV_CASTING) { + PyErr_Format(PyExc_RuntimeError, + "resolve_descriptors cast level changed even though " + "the cast is non-parametric where the only possible " + "change should be from equivalent to no casting. " + "(set level is %d, got %d for method %s)", + self->method->casting, cast, self->method->name); + Py_DECREF(result_tuple); + return NULL; + } + } } - else { - NPY_CASTING cast = casting & ~_NPY_CAST_IS_VIEW; - if (cast != PyArray_MinCastSafety(cast, self->method->casting)) { - PyErr_Format(PyExc_RuntimeError, - "resolve_descriptors cast level did not match stored one " - "(expected %d, got %d) for method %s", - self->method->casting, (casting & ~_NPY_CAST_IS_VIEW), - self->method->name); - Py_DECREF(result_tuple); + + return Py_BuildValue("iN", casting, result_tuple); +} + + +/* + * TODO: This function is not public API, and certain code paths will need + * changes and especially testing if they were to be made public. + */ +static PyObject * +boundarraymethod__simple_strided_call( + PyBoundArrayMethodObject *self, PyObject *arr_tuple) +{ + PyArrayObject *arrays[NPY_MAXARGS]; + PyArray_Descr *descrs[NPY_MAXARGS]; + PyArray_Descr *out_descrs[NPY_MAXARGS]; + Py_ssize_t length = -1; + int aligned = 1; + char *args[NPY_MAXARGS]; + npy_intp strides[NPY_MAXARGS]; + int nin = self->method->nin; + int nout = self->method->nout; + + if (!PyTuple_CheckExact(arr_tuple) || + PyTuple_Size(arr_tuple) != nin + nout) { + PyErr_Format(PyExc_TypeError, + "_simple_strided_call() takes exactly one tuple with as many " + "arrays as the method takes arguments (%d+%d).", nin, nout); + return NULL; + } + + for (int i = 0; i < nin + nout; i++) { + PyObject *tmp = PyTuple_GetItem(arr_tuple, i); + if (tmp == NULL) { + return NULL; + } + else if (!PyArray_CheckExact(tmp)) { + PyErr_SetString(PyExc_TypeError, + "All inputs must be NumPy arrays."); return NULL; } + arrays[i] = (PyArrayObject *)tmp; + descrs[i] = PyArray_DESCR(arrays[i]); + + /* Check that the input is compatible with a simple method call. */ + if (Py_TYPE(descrs[i]) != (PyTypeObject *)self->dtypes[i]) { + PyErr_Format(PyExc_TypeError, + "input dtype %S was not an exact instance of the bound " + "DType class %S.", descrs[i], self->dtypes[i]); + return NULL; + } + if (PyArray_NDIM(arrays[i]) != 1) { + PyErr_SetString(PyExc_ValueError, + "All arrays must be one dimensional."); + return NULL; + } + if (i == 0) { + length = PyArray_SIZE(arrays[i]); + } + else if (PyArray_SIZE(arrays[i]) != length) { + PyErr_SetString(PyExc_ValueError, + "All arrays must have the same length."); + return NULL; + } + if (i >= nout) { + if (PyArray_FailUnlessWriteable( + arrays[i], "_simple_strided_call() output") < 0) { + return NULL; + } + } + + args[i] = PyArray_BYTES(arrays[i]); + strides[i] = PyArray_STRIDES(arrays[i])[0]; + /* TODO: We may need to distinguish aligned and itemsize-aligned */ + aligned &= PyArray_ISALIGNED(arrays[i]); + } + if (!aligned && !(self->method->flags & NPY_METH_SUPPORTS_UNALIGNED)) { + PyErr_SetString(PyExc_ValueError, + "method does not support unaligned input."); + return NULL; } - return Py_BuildValue("iN", casting, result_tuple); + NPY_CASTING casting = self->method->resolve_descriptors( + self->method, self->dtypes, descrs, out_descrs); + + if (casting < 0) { + PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; + PyErr_Fetch(&err_type, &err_value, &err_traceback); + PyErr_SetString(PyExc_TypeError, + "cannot perform method call with the given dtypes."); + npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); + return NULL; + } + + int dtypes_were_adapted = 0; + for (int i = 0; i < nin + nout; i++) { + /* NOTE: This check is probably much stricter than necessary... */ + dtypes_were_adapted |= descrs[i] != out_descrs[i]; + Py_DECREF(out_descrs[i]); + } + if (dtypes_were_adapted) { + PyErr_SetString(PyExc_TypeError, + "_simple_strided_call(): requires dtypes to not require a cast " + "(must match exactly with `_resolve_descriptors()`)."); + return NULL; + } + + PyArrayMethod_Context context = { + .caller = NULL, + .method = self->method, + .descriptors = descrs, + }; + PyArrayMethod_StridedLoop *strided_loop = NULL; + NpyAuxData *loop_data = NULL; + NPY_ARRAYMETHOD_FLAGS flags = 0; + + if (self->method->get_strided_loop( + &context, aligned, 0, strides, + &strided_loop, &loop_data, &flags) < 0) { + return NULL; + } + + /* + * TODO: Add floating point error checks if requested and + * possibly release GIL if allowed by the flags. + */ + int res = strided_loop(&context, args, &length, strides, loop_data); + if (loop_data != NULL) { + loop_data->free(loop_data); + } + if (res < 0) { + return NULL; + } + Py_RETURN_NONE; } PyMethodDef boundarraymethod_methods[] = { {"_resolve_descriptors", (PyCFunction)boundarraymethod__resolve_descripors, METH_O, "Resolve the given dtypes."}, + {"_simple_strided_call", (PyCFunction)boundarraymethod__simple_strided_call, + METH_O, "call on 1-d inputs and pre-allocated outputs (single call)."}, {NULL, 0, 0, NULL}, }; diff --git a/numpy/core/src/multiarray/array_method.h b/numpy/core/src/multiarray/array_method.h index 15ea948ce5fa..88167a6bb231 100644 --- a/numpy/core/src/multiarray/array_method.h +++ b/numpy/core/src/multiarray/array_method.h @@ -6,7 +6,6 @@ #include #include -#include typedef enum { @@ -50,6 +49,11 @@ typedef struct { } PyArrayMethod_Context; +typedef int (PyArrayMethod_StridedLoop)(PyArrayMethod_Context *context, + char *const *data, const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *transferdata); + + typedef NPY_CASTING (resolve_descriptors_function)( struct PyArrayMethodObject_tag *method, PyArray_DTypeMeta **dtypes, @@ -61,7 +65,7 @@ typedef int (get_loop_function)( PyArrayMethod_Context *context, int aligned, int move_references, npy_intp *strides, - PyArray_StridedUnaryOp **out_loop, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, NPY_ARRAYMETHOD_FLAGS *flags); @@ -104,10 +108,10 @@ typedef struct PyArrayMethodObject_tag { resolve_descriptors_function *resolve_descriptors; get_loop_function *get_strided_loop; /* Typical loop functions (contiguous ones are used in current casts) */ - PyArray_StridedUnaryOp *strided_loop; - PyArray_StridedUnaryOp *contiguous_loop; - PyArray_StridedUnaryOp *unaligned_strided_loop; - PyArray_StridedUnaryOp *unaligned_contiguous_loop; + PyArrayMethod_StridedLoop *strided_loop; + PyArrayMethod_StridedLoop *contiguous_loop; + PyArrayMethod_StridedLoop *unaligned_strided_loop; + PyArrayMethod_StridedLoop *unaligned_contiguous_loop; } PyArrayMethodObject; @@ -144,6 +148,21 @@ extern NPY_NO_EXPORT PyTypeObject PyBoundArrayMethod_Type; #define NPY_METH_unaligned_contiguous_loop 6 +/* + * Used internally (initially) for real to complex loops only + */ +NPY_NO_EXPORT int +npy_default_get_strided_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags); + + +/* + * TODO: This function is the internal version, and its error paths may + * need better tests when a public version is exposed. + */ NPY_NO_EXPORT PyBoundArrayMethodObject * PyArrayMethod_FromSpec_int(PyArrayMethod_Spec *spec, int private); diff --git a/numpy/core/src/multiarray/arrayfunction_override.c b/numpy/core/src/multiarray/arrayfunction_override.c index 2c07cdebc625..463a2d4d8724 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.c +++ b/numpy/core/src/multiarray/arrayfunction_override.c @@ -213,7 +213,7 @@ call_array_function(PyObject* argument, PyObject* method, * to NotImplemented to indicate the default implementation should * be used. */ -NPY_NO_EXPORT PyObject * +static PyObject * array_implement_array_function_internal( PyObject *public_api, PyObject *relevant_args, PyObject *args, PyObject *kwargs) @@ -341,18 +341,23 @@ array_implement_array_function( return NULL; } - /* Remove `like=` kwarg, which is NumPy-exclusive and thus not present + /* + * Remove `like=` kwarg, which is NumPy-exclusive and thus not present * in downstream libraries. If `like=` is specified but doesn't * implement `__array_function__`, raise a `TypeError`. */ if (kwargs != NULL && PyDict_Contains(kwargs, npy_ma_str_like)) { PyObject *like_arg = PyDict_GetItem(kwargs, npy_ma_str_like); - if (like_arg && !get_array_function(like_arg)) { - return PyErr_Format(PyExc_TypeError, - "The `like` argument must be an array-like that implements " - "the `__array_function__` protocol."); + if (like_arg != NULL) { + PyObject *tmp_has_override = get_array_function(like_arg); + if (tmp_has_override == NULL) { + return PyErr_Format(PyExc_TypeError, + "The `like` argument must be an array-like that " + "implements the `__array_function__` protocol."); + } + Py_DECREF(tmp_has_override); + PyDict_DelItem(kwargs, npy_ma_str_like); } - PyDict_DelItem(kwargs, npy_ma_str_like); } PyObject *res = array_implement_array_function_internal( @@ -364,66 +369,104 @@ array_implement_array_function( return res; } - /* * Implements the __array_function__ protocol for C array creation functions * only. Added as an extension to NEP-18 in an effort to bring NEP-35 to * life with minimal dispatch overhead. + * + * The caller must ensure that `like != NULL`. */ NPY_NO_EXPORT PyObject * array_implement_c_array_function_creation( - const char *function_name, PyObject *args, PyObject *kwargs) + const char *function_name, PyObject *like, + PyObject *args, PyObject *kwargs, + PyObject *const *fast_args, Py_ssize_t len_args, PyObject *kwnames) { - if (kwargs == NULL) { - return Py_NotImplemented; + PyObject *relevant_args = NULL; + PyObject *numpy_module = NULL; + PyObject *public_api = NULL; + PyObject *result = NULL; + + /* If `like` doesn't implement `__array_function__`, raise a `TypeError` */ + PyObject *tmp_has_override = get_array_function(like); + if (tmp_has_override == NULL) { + return PyErr_Format(PyExc_TypeError, + "The `like` argument must be an array-like that " + "implements the `__array_function__` protocol."); } + Py_DECREF(tmp_has_override); - /* Remove `like=` kwarg, which is NumPy-exclusive and thus not present - * in downstream libraries. If that key isn't present, return NULL and - * let originating call to continue. If the key is present but doesn't - * implement `__array_function__`, raise a `TypeError`. - */ - if (!PyDict_Contains(kwargs, npy_ma_str_like)) { - return Py_NotImplemented; + if (fast_args != NULL) { + /* + * Convert from vectorcall convention, since the protocol requires + * the normal convention. We have to do this late to ensure the + * normal path where NotImplemented is returned is fast. + */ + assert(args == NULL); + assert(kwargs == NULL); + args = PyTuple_New(len_args); + if (args == NULL) { + return NULL; + } + for (Py_ssize_t i = 0; i < len_args; i++) { + Py_INCREF(fast_args[i]); + PyTuple_SET_ITEM(args, i, fast_args[i]); + } + if (kwnames != NULL) { + kwargs = PyDict_New(); + if (kwargs == NULL) { + Py_DECREF(args); + return NULL; + } + Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames); + for (Py_ssize_t i = 0; i < nkwargs; i++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, i); + PyObject *value = fast_args[i+len_args]; + if (PyDict_SetItem(kwargs, key, value) < 0) { + Py_DECREF(args); + Py_DECREF(kwargs); + return NULL; + } + } + } } - PyObject *like_arg = PyDict_GetItem(kwargs, npy_ma_str_like); - if (like_arg == NULL) { - return NULL; + relevant_args = PyTuple_Pack(1, like); + if (relevant_args == NULL) { + goto finish; } - else if (!get_array_function(like_arg)) { - return PyErr_Format(PyExc_TypeError, - "The `like` argument must be an array-like that implements " - "the `__array_function__` protocol."); + /* The like argument must be present in the keyword arguments, remove it */ + if (PyDict_DelItem(kwargs, npy_ma_str_like) < 0) { + goto finish; } - PyObject *relevant_args = PyTuple_Pack(1, like_arg); - PyDict_DelItem(kwargs, npy_ma_str_like); - PyObject *numpy_module = PyImport_Import(npy_ma_str_numpy); + numpy_module = PyImport_Import(npy_ma_str_numpy); if (numpy_module == NULL) { - Py_DECREF(relevant_args); - return NULL; + goto finish; } - PyObject *public_api = PyObject_GetAttrString(numpy_module, function_name); + public_api = PyObject_GetAttrString(numpy_module, function_name); Py_DECREF(numpy_module); if (public_api == NULL) { - Py_DECREF(relevant_args); - return NULL; + goto finish; } if (!PyCallable_Check(public_api)) { - Py_DECREF(relevant_args); - Py_DECREF(public_api); - return PyErr_Format(PyExc_RuntimeError, - "numpy.%s is not callable.", - function_name); + PyErr_Format(PyExc_RuntimeError, + "numpy.%s is not callable.", function_name); + goto finish; } - PyObject* result = array_implement_array_function_internal( + result = array_implement_array_function_internal( public_api, relevant_args, args, kwargs); - Py_DECREF(relevant_args); - Py_DECREF(public_api); + finish: + if (kwnames != NULL) { + /* args and kwargs were converted from vectorcall convention */ + Py_XDECREF(args); + Py_XDECREF(kwargs); + } + Py_XDECREF(relevant_args); + Py_XDECREF(public_api); return result; } diff --git a/numpy/core/src/multiarray/arrayfunction_override.h b/numpy/core/src/multiarray/arrayfunction_override.h index fdcf1746d62d..fdf0dfcaf3f5 100644 --- a/numpy/core/src/multiarray/arrayfunction_override.h +++ b/numpy/core/src/multiarray/arrayfunction_override.h @@ -11,7 +11,9 @@ array__get_implementing_args( NPY_NO_EXPORT PyObject * array_implement_c_array_function_creation( - const char *function_name, PyObject *args, PyObject *kwargs); + const char *function_name, PyObject *like, + PyObject *args, PyObject *kwargs, + PyObject *const *fast_args, Py_ssize_t len_args, PyObject *kwnames); NPY_NO_EXPORT PyObject * array_function_method_impl(PyObject *func, PyObject *types, PyObject *args, diff --git a/numpy/core/src/multiarray/arrayobject.c b/numpy/core/src/multiarray/arrayobject.c index a2474d79fcf5..e7fbb88cd282 100644 --- a/numpy/core/src/multiarray/arrayobject.c +++ b/numpy/core/src/multiarray/arrayobject.c @@ -1356,11 +1356,13 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) switch (cmp_op) { case Py_LT: RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); - result = PyArray_GenericBinaryFunction(self, other, n_ops.less); + result = PyArray_GenericBinaryFunction( + (PyObject *)self, other, n_ops.less); break; case Py_LE: RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); - result = PyArray_GenericBinaryFunction(self, other, n_ops.less_equal); + result = PyArray_GenericBinaryFunction( + (PyObject *)self, other, n_ops.less_equal); break; case Py_EQ: RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); @@ -1410,9 +1412,8 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - result = PyArray_GenericBinaryFunction(self, - (PyObject *)other, - n_ops.equal); + result = PyArray_GenericBinaryFunction( + (PyObject *)self, (PyObject *)other, n_ops.equal); break; case Py_NE: RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); @@ -1462,18 +1463,18 @@ array_richcompare(PyArrayObject *self, PyObject *other, int cmp_op) return result; } - result = PyArray_GenericBinaryFunction(self, (PyObject *)other, - n_ops.not_equal); + result = PyArray_GenericBinaryFunction( + (PyObject *)self, (PyObject *)other, n_ops.not_equal); break; case Py_GT: RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); - result = PyArray_GenericBinaryFunction(self, other, - n_ops.greater); + result = PyArray_GenericBinaryFunction( + (PyObject *)self, other, n_ops.greater); break; case Py_GE: RICHCMP_GIVE_UP_IF_NEEDED(obj_self, other); - result = PyArray_GenericBinaryFunction(self, other, - n_ops.greater_equal); + result = PyArray_GenericBinaryFunction( + (PyObject *)self, other, n_ops.greater_equal); break; default: Py_INCREF(Py_NotImplemented); @@ -1754,10 +1755,6 @@ NPY_NO_EXPORT PyTypeObject PyArray_Type = { .tp_as_number = &array_as_number, .tp_as_sequence = &array_as_sequence, .tp_as_mapping = &array_as_mapping, - /* - * The tp_hash slot will be set PyObject_HashNotImplemented when the - * module is loaded. - */ .tp_str = (reprfunc)array_str, .tp_as_buffer = &array_as_buffer, .tp_flags =(Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE), diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index ecaca72a1848..ad74612272b2 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -42,6 +42,32 @@ #include "npy_cblas.h" #include "npy_buffer.h" + +/* + * Define a stack allocated dummy array with only the minimum information set: + * 1. The descr, the main field interesting here. + * 2. The flags, which are needed for alignment;. + * 3. The type is set to NULL and the base is the original array, if this + * is used within a subarray getitem to create a new view, the base + * must be walked until the type is not NULL. + * + * The following should create errors in debug mode (if deallocated + * incorrectly), since base would be incorrectly decref'd as well. + * This is especially important for nonzero and copyswap, which may run with + * the GIL released. + */ +static NPY_INLINE PyArrayObject_fields +get_dummy_stack_array(PyArrayObject *orig) +{ + PyArrayObject_fields new_fields; + new_fields.flags = PyArray_FLAGS(orig); + /* Set to NULL so the dummy object can be distinguished from the real one */ + Py_TYPE(&new_fields) = NULL; + new_fields.base = (PyObject *)orig; + return new_fields; +} + + /* check for sequences, but ignore the types numpy considers scalars */ static NPY_INLINE npy_bool PySequence_NoString_Check(PyObject *op) { @@ -674,6 +700,7 @@ OBJECT_setitem(PyObject *op, void *ov, void *NPY_UNUSED(ap)) return PyErr_Occurred() ? -1 : 0; } + /* VOID */ static PyObject * @@ -681,22 +708,21 @@ VOID_getitem(void *input, void *vap) { PyArrayObject *ap = vap; char *ip = input; - PyArray_Descr* descr; + PyArray_Descr* descr = PyArray_DESCR(vap); - descr = PyArray_DESCR(ap); if (PyDataType_HASFIELDS(descr)) { PyObject *key; PyObject *names; int i, n; PyObject *ret; PyObject *tup; - int savedflags; + PyArrayObject_fields dummy_fields = get_dummy_stack_array(ap); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; /* get the names from the fields dictionary*/ names = descr->names; n = PyTuple_GET_SIZE(names); ret = PyTuple_New(n); - savedflags = PyArray_FLAGS(ap); for (i = 0; i < n; i++) { npy_intp offset; PyArray_Descr *new; @@ -704,26 +730,19 @@ VOID_getitem(void *input, void *vap) tup = PyDict_GetItem(descr->fields, key); if (_unpack_field(tup, &new, &offset) < 0) { Py_DECREF(ret); - ((PyArrayObject_fields *)ap)->descr = descr; return NULL; } - /* - * TODO: temporarily modifying the array like this - * is bad coding style, should be changed. - */ - ((PyArrayObject_fields *)ap)->descr = new; + dummy_fields.descr = new; /* update alignment based on offset */ if ((new->alignment > 1) && ((((npy_intp)(ip+offset)) % new->alignment) != 0)) { - PyArray_CLEARFLAGS(ap, NPY_ARRAY_ALIGNED); + PyArray_CLEARFLAGS(dummy_arr, NPY_ARRAY_ALIGNED); } else { - PyArray_ENABLEFLAGS(ap, NPY_ARRAY_ALIGNED); + PyArray_ENABLEFLAGS(dummy_arr, NPY_ARRAY_ALIGNED); } - PyTuple_SET_ITEM(ret, i, PyArray_GETITEM(ap, ip+offset)); - ((PyArrayObject_fields *)ap)->flags = savedflags; + PyTuple_SET_ITEM(ret, i, PyArray_GETITEM(dummy_arr, ip+offset)); } - ((PyArrayObject_fields *)ap)->descr = descr; return ret; } @@ -739,11 +758,28 @@ VOID_getitem(void *input, void *vap) return NULL; } Py_INCREF(descr->subarray->base); + + /* + * NOTE: There is the possibility of recursive calls from the above + * field branch. These calls use a dummy arr for thread + * (and general) safety. However, we must set the base array, + * so if such a dummy array was passed (its type is NULL), + * we have walk its base until the initial array is found. + * + * TODO: This should be fixed, the next "generation" of GETITEM will + * probably need to pass in the original array (in addition + * to the dtype as a method). Alternatively, VOID dtypes + * could have special handling. + */ + PyObject *base = (PyObject *)ap; + while (Py_TYPE(base) == NULL) { + base = PyArray_BASE((PyArrayObject *)base); + } ret = (PyArrayObject *)PyArray_NewFromDescrAndBase( &PyArray_Type, descr->subarray->base, shape.len, shape.ptr, NULL, ip, PyArray_FLAGS(ap) & ~NPY_ARRAY_F_CONTIGUOUS, - NULL, (PyObject *)ap); + NULL, base); npy_free_cache_dim_obj(shape); return (PyObject *)ret; } @@ -761,7 +797,8 @@ NPY_NO_EXPORT int PyArray_CopyObject(PyArrayObject *, PyObject *); * individual fields of a numpy structure, in VOID_setitem. Compare to inner * loops in VOID_getitem and VOID_nonzero. * - * WARNING: Clobbers arr's dtype and alignment flag. + * WARNING: Clobbers arr's dtype and alignment flag, should not be used + * on the original array! */ NPY_NO_EXPORT int _setup_field(int i, PyArray_Descr *descr, PyArrayObject *arr, @@ -798,7 +835,7 @@ static int _copy_and_return_void_setitem(PyArray_Descr *dstdescr, char *dstdata, PyArray_Descr *srcdescr, char *srcdata){ PyArrayObject_fields dummy_struct; - PyArrayObject *dummy = (PyArrayObject *)&dummy_struct; + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_struct; npy_int names_size = PyTuple_GET_SIZE(dstdescr->names); npy_intp offset; npy_int i; @@ -808,11 +845,11 @@ _copy_and_return_void_setitem(PyArray_Descr *dstdescr, char *dstdata, if (PyArray_EquivTypes(srcdescr, dstdescr)) { for (i = 0; i < names_size; i++) { /* neither line can ever fail, in principle */ - if (_setup_field(i, dstdescr, dummy, &offset, dstdata)) { + if (_setup_field(i, dstdescr, dummy_arr, &offset, dstdata)) { return -1; } - PyArray_DESCR(dummy)->f->copyswap(dstdata + offset, - srcdata + offset, 0, dummy); + PyArray_DESCR(dummy_arr)->f->copyswap(dstdata + offset, + srcdata + offset, 0, dummy_arr); } return 0; } @@ -831,13 +868,10 @@ VOID_setitem(PyObject *op, void *input, void *vap) { char *ip = input; PyArrayObject *ap = vap; - PyArray_Descr *descr; - int flags; - int itemsize=PyArray_DESCR(ap)->elsize; + int itemsize = PyArray_DESCR(ap)->elsize; int res; + PyArray_Descr *descr = PyArray_DESCR(ap); - descr = PyArray_DESCR(ap); - flags = PyArray_FLAGS(ap); if (PyDataType_HASFIELDS(descr)) { PyObject *errmsg; npy_int i; @@ -874,11 +908,13 @@ VOID_setitem(PyObject *op, void *input, void *vap) return -1; } + PyArrayObject_fields dummy_fields = get_dummy_stack_array(ap); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; + for (i = 0; i < names_size; i++) { PyObject *item; - /* temporarily make ap have only this field */ - if (_setup_field(i, descr, ap, &offset, ip) == -1) { + if (_setup_field(i, descr, dummy_arr, &offset, ip) == -1) { failed = 1; break; } @@ -888,7 +924,7 @@ VOID_setitem(PyObject *op, void *input, void *vap) break; } /* use setitem to set this field */ - if (PyArray_SETITEM(ap, ip + offset, item) < 0) { + if (PyArray_SETITEM(dummy_arr, ip + offset, item) < 0) { failed = 1; break; } @@ -898,24 +934,23 @@ VOID_setitem(PyObject *op, void *input, void *vap) /* Otherwise must be non-void scalar. Try to assign to each field */ npy_intp names_size = PyTuple_GET_SIZE(descr->names); + PyArrayObject_fields dummy_fields = get_dummy_stack_array(ap); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; + for (i = 0; i < names_size; i++) { /* temporarily make ap have only this field */ - if (_setup_field(i, descr, ap, &offset, ip) == -1) { + if (_setup_field(i, descr, dummy_arr, &offset, ip) == -1) { failed = 1; break; } /* use setitem to set this field */ - if (PyArray_SETITEM(ap, ip + offset, op) < 0) { + if (PyArray_SETITEM(dummy_arr, ip + offset, op) < 0) { failed = 1; break; } } } - /* reset clobbered attributes */ - ((PyArrayObject_fields *)(ap))->descr = descr; - ((PyArrayObject_fields *)(ap))->flags = flags; - if (failed) { return -1; } @@ -924,7 +959,6 @@ VOID_setitem(PyObject *op, void *input, void *vap) else if (PyDataType_HASSUBARRAY(descr)) { /* copy into an array of the same basic type */ PyArray_Dims shape = {NULL, -1}; - PyArrayObject *ret; if (!(PyArray_IntpConverter(descr->subarray->shape, &shape))) { npy_free_cache_dim_obj(shape); PyErr_SetString(PyExc_ValueError, @@ -932,10 +966,15 @@ VOID_setitem(PyObject *op, void *input, void *vap) return -1; } Py_INCREF(descr->subarray->base); - ret = (PyArrayObject *)PyArray_NewFromDescrAndBase( + /* + * Note we set no base object here, as to not rely on the input + * being a valid object for base setting. `ret` nevertheless does + * does not own its data, this is generally not good, but localized. + */ + PyArrayObject *ret = (PyArrayObject *)PyArray_NewFromDescrAndBase( &PyArray_Type, descr->subarray->base, shape.len, shape.ptr, NULL, ip, - PyArray_FLAGS(ap), NULL, (PyObject *)ap); + PyArray_FLAGS(ap), NULL, NULL); npy_free_cache_dim_obj(shape); if (!ret) { return -1; @@ -2287,6 +2326,7 @@ STRING_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, return; } + /* */ static void VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, @@ -2303,29 +2343,26 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, if (PyArray_HASFIELDS(arr)) { PyObject *key, *value; - Py_ssize_t pos = 0; + PyArrayObject_fields dummy_fields = get_dummy_stack_array(arr); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; + while (PyDict_Next(descr->fields, &pos, &key, &value)) { npy_intp offset; - PyArray_Descr * new; + PyArray_Descr *new; if (NPY_TITLE_KEY(key, value)) { continue; } if (_unpack_field(value, &new, &offset) < 0) { - ((PyArrayObject_fields *)arr)->descr = descr; return; } - /* - * TODO: temporarily modifying the array like this - * is bad coding style, should be changed. - */ - ((PyArrayObject_fields *)arr)->descr = new; + + dummy_fields.descr = new; new->f->copyswapn(dst+offset, dstride, (src != NULL ? src+offset : NULL), - sstride, n, swap, arr); + sstride, n, swap, dummy_arr); } - ((PyArrayObject_fields *)arr)->descr = descr; return; } if (PyDataType_HASSUBARRAY(descr)) { @@ -2351,11 +2388,6 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, } new = descr->subarray->base; - /* - * TODO: temporarily modifying the array like this - * is bad coding style, should be changed. - */ - ((PyArrayObject_fields *)arr)->descr = new; dstptr = dst; srcptr = src; subitemsize = new->elsize; @@ -2363,16 +2395,20 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride, /* There cannot be any elements, so return */ return; } + + PyArrayObject_fields dummy_fields = get_dummy_stack_array(arr); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; + ((PyArrayObject_fields *)dummy_arr)->descr = new; + num = descr->elsize / subitemsize; for (i = 0; i < n; i++) { new->f->copyswapn(dstptr, subitemsize, srcptr, - subitemsize, num, swap, arr); + subitemsize, num, swap, dummy_arr); dstptr += dstride; if (srcptr) { srcptr += sstride; } } - ((PyArrayObject_fields *)arr)->descr = descr; return; } /* Must be a naive Void type (e.g. a "V8") so simple copy is sufficient. */ @@ -2396,26 +2432,24 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr) PyObject *key, *value; Py_ssize_t pos = 0; + PyArrayObject_fields dummy_fields = get_dummy_stack_array(arr); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; + while (PyDict_Next(descr->fields, &pos, &key, &value)) { npy_intp offset; + PyArray_Descr * new; if (NPY_TITLE_KEY(key, value)) { continue; } if (_unpack_field(value, &new, &offset) < 0) { - ((PyArrayObject_fields *)arr)->descr = descr; return; } - /* - * TODO: temporarily modifying the array like this - * is bad coding style, should be changed. - */ - ((PyArrayObject_fields *)arr)->descr = new; + dummy_fields.descr = new; new->f->copyswap(dst+offset, (src != NULL ? src+offset : NULL), - swap, arr); + swap, dummy_arr); } - ((PyArrayObject_fields *)arr)->descr = descr; return; } if (PyDataType_HASSUBARRAY(descr)) { @@ -2439,20 +2473,19 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr) } new = descr->subarray->base; - /* - * TODO: temporarily modifying the array like this - * is bad coding style, should be changed. - */ - ((PyArrayObject_fields *)arr)->descr = new; subitemsize = new->elsize; if (subitemsize == 0) { /* There cannot be any elements, so return */ return; } + + PyArrayObject_fields dummy_fields = get_dummy_stack_array(arr); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; + dummy_fields.descr = new; + num = descr->elsize / subitemsize; new->f->copyswapn(dst, subitemsize, src, - subitemsize, num, swap, arr); - ((PyArrayObject_fields *)arr)->descr = descr; + subitemsize, num, swap, dummy_arr); return; } /* Must be a naive Void type (e.g. a "V8") so simple copy is sufficient. */ @@ -2707,11 +2740,11 @@ VOID_nonzero (char *ip, PyArrayObject *ap) if (PyArray_HASFIELDS(ap)) { PyArray_Descr *descr; PyObject *key, *value; - int savedflags; Py_ssize_t pos = 0; + PyArrayObject_fields dummy_fields = get_dummy_stack_array(ap); + PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields; descr = PyArray_DESCR(ap); - savedflags = PyArray_FLAGS(ap); while (PyDict_Next(descr->fields, &pos, &key, &value)) { PyArray_Descr * new; npy_intp offset; @@ -2722,12 +2755,8 @@ VOID_nonzero (char *ip, PyArrayObject *ap) PyErr_Clear(); continue; } - /* - * TODO: temporarily modifying the array like this - * is bad coding style, should be changed. - */ - ((PyArrayObject_fields *)ap)->descr = new; - ((PyArrayObject_fields *)ap)->flags = savedflags; + + dummy_fields.descr = new; if ((new->alignment > 1) && !__ALIGNED(ip + offset, new->alignment)) { PyArray_CLEARFLAGS(ap, NPY_ARRAY_ALIGNED); @@ -2735,13 +2764,11 @@ VOID_nonzero (char *ip, PyArrayObject *ap) else { PyArray_ENABLEFLAGS(ap, NPY_ARRAY_ALIGNED); } - if (new->f->nonzero(ip+offset, ap)) { + if (new->f->nonzero(ip+offset, dummy_arr)) { nonz = NPY_TRUE; break; } } - ((PyArrayObject_fields *)ap)->descr = descr; - ((PyArrayObject_fields *)ap)->flags = savedflags; return nonz; } len = PyArray_DESCR(ap)->elsize; diff --git a/numpy/core/src/multiarray/buffer.c b/numpy/core/src/multiarray/buffer.c index 813850224714..5458c81cccec 100644 --- a/numpy/core/src/multiarray/buffer.c +++ b/numpy/core/src/multiarray/buffer.c @@ -878,6 +878,7 @@ void_getbuffer(PyObject *self, Py_buffer *view, int flags) */ _buffer_info_t *info = _buffer_get_info(&scalar->_buffer_info, self, flags); if (info == NULL) { + Py_DECREF(self); return -1; } view->format = info->format; diff --git a/numpy/core/src/multiarray/calculation.c b/numpy/core/src/multiarray/calculation.c index 43d88271b10d..de67b35b53d6 100644 --- a/numpy/core/src/multiarray/calculation.c +++ b/numpy/core/src/multiarray/calculation.c @@ -392,7 +392,7 @@ __New_PyArray_Std(PyArrayObject *self, int axis, int rtype, PyArrayObject *out, else { val = PyArray_DIM(arrnew,i); } - PyTuple_SET_ITEM(newshape, i, PyLong_FromLong((long)val)); + PyTuple_SET_ITEM(newshape, i, PyLong_FromSsize_t(val)); } arr2 = (PyArrayObject *)PyArray_Reshape(arr1, newshape); Py_DECREF(arr1); @@ -423,7 +423,8 @@ __New_PyArray_Std(PyArrayObject *self, int axis, int rtype, PyArrayObject *out, return NULL; } arr2 = (PyArrayObject *)PyArray_EnsureAnyArray( - PyArray_GenericBinaryFunction(arr1, obj3, n_ops.multiply)); + PyArray_GenericBinaryFunction((PyObject *)arr1, obj3, + n_ops.multiply)); Py_DECREF(arr1); Py_DECREF(obj3); if (arr2 == NULL) { @@ -1211,7 +1212,7 @@ PyArray_Conjugate(PyArrayObject *self, PyArrayObject *out) n_ops.conjugate); } else { - return PyArray_GenericBinaryFunction(self, + return PyArray_GenericBinaryFunction((PyObject *)self, (PyObject *)out, n_ops.conjugate); } diff --git a/numpy/core/src/multiarray/common.h b/numpy/core/src/multiarray/common.h index ef9bc79da325..83209cd38b94 100644 --- a/numpy/core/src/multiarray/common.h +++ b/numpy/core/src/multiarray/common.h @@ -209,7 +209,7 @@ npy_is_aligned(const void * p, const npy_uintp alignment) } /* Get equivalent "uint" alignment given an itemsize, for use in copy code */ -static NPY_INLINE int +static NPY_INLINE npy_uintp npy_uint_alignment(int itemsize) { npy_uintp alignment = 0; /* return value of 0 means unaligned */ @@ -267,7 +267,7 @@ npy_memchr(char * haystack, char needle, } else { /* usually find elements to skip path */ - if (NPY_CPU_HAVE_UNALIGNED_ACCESS && needle == 0 && stride == 1) { + if (!NPY_ALIGNMENT_REQUIRED && needle == 0 && stride == 1) { /* iterate until last multiple of 4 */ char * block_end = haystack + size - (size % sizeof(unsigned int)); while (p < block_end) { diff --git a/numpy/core/src/multiarray/common_dtype.c b/numpy/core/src/multiarray/common_dtype.c new file mode 100644 index 000000000000..a88085f6f084 --- /dev/null +++ b/numpy/core/src/multiarray/common_dtype.c @@ -0,0 +1,318 @@ +#define PY_SSIZE_T_CLEAN +#include + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE +#include +#include "numpy/arrayobject.h" + +#include "common_dtype.h" +#include "dtypemeta.h" +#include "abstractdtypes.h" + + +/* + * This file defines all logic necessary for generic "common dtype" + * operations. This is unfortunately surprisingly complicated to get right + * due to the value based logic NumPy uses and the fact that NumPy has + * no clear (non-transitive) type promotion hierarchy. + * Unlike most languages `int32 + float32 -> float64` instead of `float32`. + * The other complicated thing is value-based-promotion, which means that + * in many cases a Python 1, may end up as an `int8` or `uint8`. + * + * This file implements the necessary logic so that `np.result_type(...)` + * can give the correct result for any order of inputs and can further + * generalize to user DTypes. + */ + + +/** + * This function defines the common DType operator. + * + * Note that the common DType will not be "object" (unless one of the dtypes + * is object), even though object can technically represent all values + * correctly. + * + * TODO: Before exposure, we should review the return value (e.g. no error + * when no common DType is found). + * + * @param dtype1 DType class to find the common type for. + * @param dtype2 Second DType class. + * @return The common DType or NULL with an error set + */ +NPY_NO_EXPORT NPY_INLINE PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) +{ + if (dtype1 == dtype2) { + Py_INCREF(dtype1); + return dtype1; + } + + PyArray_DTypeMeta *common_dtype; + + common_dtype = dtype1->common_dtype(dtype1, dtype2); + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(common_dtype); + common_dtype = dtype2->common_dtype(dtype2, dtype1); + } + if (common_dtype == NULL) { + return NULL; + } + if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_DECREF(Py_NotImplemented); + PyErr_Format(PyExc_TypeError, + "The DTypes %S and %S do not have a common DType. " + "For example they cannot be stored in a single array unless " + "the dtype is `object`.", dtype1, dtype2); + return NULL; + } + return common_dtype; +} + + +/** + * This function takes a list of dtypes and "reduces" them (in a sense, + * it finds the maximal dtype). Note that "maximum" here is defined by + * knowledge (or category or domain). A user DType must always "know" + * about all NumPy dtypes, floats "know" about integers, integers "know" + * about unsigned integers. + * + * c + * / \ + * a \ <-- The actual promote(a, b) may be c or unknown. + * / \ \ + * a b c + * + * The reduction is done "pairwise". In the above `a.__common_dtype__(b)` + * has a result (so `a` knows more) and `a.__common_dtype__(c)` returns + * NotImplemented (so `c` knows more). You may notice that the result + * `res = a.__common_dtype__(b)` is not important. We could try to use it + * to remove the whole branch if `res is c` or by checking if + * `c.__common_dtype(res) is c`. + * Right now, we only clear initial elements in the most simple case where + * `a.__common_dtype(b) is a` (and thus `b` cannot alter the end-result). + * Clearing means, we do not have to worry about them later. + * + * There is one further subtlety. If we have an abstract DType and a + * non-abstract one, we "prioritize" the non-abstract DType here. + * In this sense "prioritizing" means that we use: + * abstract.__common_dtype__(other) + * If both return NotImplemented (which is acceptable and even expected in + * this case, see later) then `other` will be considered to know more. + * + * The reason why this may be acceptable for abstract DTypes, is that + * the value-dependent abstract DTypes may provide default fall-backs. + * The priority inversion effectively means that abstract DTypes are ordered + * just below their concrete counterparts. + * (This fall-back is convenient but not perfect, it can lead to + * non-minimal promotions: e.g. `np.uint24 + 2**20 -> int32`. And such + * cases may also be possible in some mixed type scenarios; they can be + * avoided by defining the promotion explicitly in the user DType.) + * + * @param length Number of DTypes + * @param dtypes + */ +static PyArray_DTypeMeta * +reduce_dtypes_to_most_knowledgeable( + npy_intp length, PyArray_DTypeMeta **dtypes) +{ + assert(length >= 2); + npy_intp half = length / 2; + + PyArray_DTypeMeta *res = NULL; + + for (npy_intp low = 0; low < half; low++) { + npy_intp high = length - 1 - low; + if (dtypes[high] == dtypes[low]) { + Py_INCREF(dtypes[low]); + Py_XSETREF(res, dtypes[low]); + } + else { + if (dtypes[high]->abstract) { + /* + * Priority inversion, start with abstract, because if it + * returns `other`, we can let other pass instead. + */ + PyArray_DTypeMeta *tmp = dtypes[low]; + dtypes[low] = dtypes[high]; + dtypes[high] = tmp; + } + + Py_XSETREF(res, dtypes[low]->common_dtype(dtypes[low], dtypes[high])); + if (res == NULL) { + return NULL; + } + } + + if (res == (PyArray_DTypeMeta *)Py_NotImplemented) { + PyArray_DTypeMeta *tmp = dtypes[low]; + dtypes[low] = dtypes[high]; + dtypes[high] = tmp; + } + if (res == dtypes[low]) { + /* `dtypes[high]` cannot influence the final result, so clear: */ + dtypes[high] = NULL; + } + } + + if (length == 2) { + return res; + } + Py_DECREF(res); + return reduce_dtypes_to_most_knowledgeable(length - half, dtypes); +} + + +/** + * Promotes a list of DTypes with each other in a way that should guarantee + * stable results even when changing the order. + * + * In general this approach always works as long as the most generic dtype + * is either strictly larger, or compatible with all other dtypes. + * For example promoting float16 with any other float, integer, or unsigned + * integer again gives a floating point number. And any floating point number + * promotes in the "same way" as `float16`. + * If a user inserts more than one type into the NumPy type hierarchy, this + * can break. Given: + * uint24 + int32 -> int48 # Promotes to a *new* dtype! + * + * The following becomes problematic (order does not matter): + * uint24 + int16 + uint32 -> int64 + * <== (uint24 + int16) + (uint24 + uint32) -> int64 + * <== int32 + uint32 -> int64 + * + * It is impossible to achieve an `int48` result in the above. + * + * This is probably only resolvable by asking `uint24` to take over the + * whole reduction step; which we currently do not do. + * (It may be possible to notice the last up-cast and implement use something + * like: `uint24.nextafter(int32).__common_dtype__(uint32)`, but that seems + * even harder to grasp.) + * + * Note that a case where two dtypes are mixed (and know nothing about each + * other) will always generate an error: + * uint24 + int48 + int64 -> Error + * + * Even though `int64` is a safe solution, since `uint24 + int64 -> int64` and + * `int48 + int64 -> int64` and `int64` and there cannot be a smaller solution. + * + * //TODO: Maybe this function should allow not setting an error? + * + * @param length Number of dtypes (and values) must be at least 1 + * @param dtypes The concrete or abstract DTypes to promote + * @return NULL or the promoted DType. + */ +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_PromoteDTypeSequence( + npy_intp length, PyArray_DTypeMeta **dtypes_in) +{ + if (length == 1) { + Py_INCREF(dtypes_in[0]); + return dtypes_in[0]; + } + PyArray_DTypeMeta *result = NULL; + + /* Copy dtypes so that we can reorder them (only allocate when many) */ + PyObject *_scratch_stack[NPY_MAXARGS]; + PyObject **_scratch_heap = NULL; + PyArray_DTypeMeta **dtypes = (PyArray_DTypeMeta **)_scratch_stack; + + if (length > NPY_MAXARGS) { + _scratch_heap = PyMem_Malloc(length * sizeof(PyObject *)); + if (_scratch_heap == NULL) { + PyErr_NoMemory(); + return NULL; + } + dtypes = (PyArray_DTypeMeta **)_scratch_heap; + } + + memcpy(dtypes, dtypes_in, length * sizeof(PyObject *)); + + /* + * `result` is the last promotion result, which can usually be reused if + * it is not NotImplemneted. + * The passed in dtypes are partially sorted (and cleared, when clearly + * not relevant anymore). + * `dtypes[0]` will be the most knowledgeable (highest category) which + * we consider the "main_dtype" here. + */ + result = reduce_dtypes_to_most_knowledgeable(length, dtypes); + if (result == NULL) { + goto finish; + } + PyArray_DTypeMeta *main_dtype = dtypes[0]; + + npy_intp reduce_start = 1; + if (result == (PyArray_DTypeMeta *)Py_NotImplemented) { + Py_SETREF(result, NULL); + } + else { + /* (new) first value is already taken care of in `result` */ + reduce_start = 2; + } + /* + * At this point, we have only looked at every DType at most once. + * The `main_dtype` must know all others (or it will be a failure) and + * all dtypes returned by its `common_dtype` must be guaranteed to succeed + * promotion with one another. + * It is the job of the "main DType" to ensure that at this point order + * is irrelevant. + * If this turns out to be a limitation, this "reduction" will have to + * become a default version and we have to allow DTypes to override it. + */ + PyArray_DTypeMeta *prev = NULL; + for (npy_intp i = reduce_start; i < length; i++) { + if (dtypes[i] == NULL || dtypes[i] == prev) { + continue; + } + /* + * "Promote" the current dtype with the main one (which should be + * a higher category). We assume that the result is not in a lower + * category. + */ + PyArray_DTypeMeta *promotion = main_dtype->common_dtype( + main_dtype, dtypes[i]); + if (promotion == NULL) { + Py_XSETREF(result, NULL); + goto finish; + } + else if ((PyObject *)promotion == Py_NotImplemented) { + Py_DECREF(Py_NotImplemented); + Py_XSETREF(result, NULL); + PyObject *dtypes_in_tuple = PyTuple_New(length); + if (dtypes_in_tuple == NULL) { + goto finish; + } + for (npy_intp l=0; l < length; l++) { + Py_INCREF(dtypes_in[l]); + PyTuple_SET_ITEM(dtypes_in_tuple, l, (PyObject *)dtypes_in[l]); + } + PyErr_Format(PyExc_TypeError, + "The DType %S could not be promoted by %S. This means that " + "no common DType exists for the given inputs. " + "For example they cannot be stored in a single array unless " + "the dtype is `object`. The full list of DTypes is: %S", + dtypes[i], main_dtype, dtypes_in_tuple); + Py_DECREF(dtypes_in_tuple); + goto finish; + } + if (result == NULL) { + result = promotion; + continue; + } + + /* + * The above promoted, now "reduce" with the current result; note that + * in the typical cases we expect this step to be a no-op. + */ + Py_SETREF(result, PyArray_CommonDType(result, promotion)); + Py_DECREF(promotion); + if (result == NULL) { + goto finish; + } + } + + finish: + PyMem_Free(_scratch_heap); + return result; +} diff --git a/numpy/core/src/multiarray/common_dtype.h b/numpy/core/src/multiarray/common_dtype.h new file mode 100644 index 000000000000..b3666531a532 --- /dev/null +++ b/numpy/core/src/multiarray/common_dtype.h @@ -0,0 +1,17 @@ +#ifndef _NPY_COMMON_DTYPE_H_ +#define _NPY_COMMON_DTYPE_H_ + +#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define _MULTIARRAYMODULE + +#include +#include "dtypemeta.h" + +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2); + +NPY_NO_EXPORT PyArray_DTypeMeta * +PyArray_PromoteDTypeSequence( + npy_intp length, PyArray_DTypeMeta **dtypes_in); + +#endif /* _NPY_COMMON_DTYPE_H_ */ diff --git a/numpy/core/src/multiarray/compiled_base.c b/numpy/core/src/multiarray/compiled_base.c index 061db2250d1c..de793f87c156 100644 --- a/numpy/core/src/multiarray/compiled_base.c +++ b/numpy/core/src/multiarray/compiled_base.c @@ -13,7 +13,12 @@ #include "alloc.h" #include "ctors.h" #include "common.h" +#include "simd/simd.h" +typedef enum { + PACK_ORDER_LITTLE = 0, + PACK_ORDER_BIG +} PACK_ORDER; /* * Returns -1 if the array is monotonic decreasing, @@ -1032,7 +1037,7 @@ arr_ravel_multi_index(PyObject *self, PyObject *args, PyObject *kwds) NpyIter *iter = NULL; - char *kwlist[] = {"multi_index", "dims", "mode", "order", NULL}; + static char *kwlist[] = {"multi_index", "dims", "mode", "order", NULL}; memset(op, 0, sizeof(op)); dtype[0] = NULL; @@ -1227,42 +1232,7 @@ arr_unravel_index(PyObject *self, PyObject *args, PyObject *kwds) int i, ret_ndim; npy_intp ret_dims[NPY_MAXDIMS], ret_strides[NPY_MAXDIMS]; - char *kwlist[] = {"indices", "shape", "order", NULL}; - - /* - * TODO: remove this in favor of warning raised in the dispatcher when - * __array_function__ is enabled by default. - */ - - /* - * Continue to support the older "dims" argument in place - * of the "shape" argument. Issue an appropriate warning - * if "dims" is detected in keywords, then replace it with - * the new "shape" argument and continue processing as usual. - */ - if (kwds) { - PyObject *dims_item, *shape_item; - dims_item = _PyDict_GetItemStringWithError(kwds, "dims"); - if (dims_item == NULL && PyErr_Occurred()){ - return NULL; - } - shape_item = _PyDict_GetItemStringWithError(kwds, "shape"); - if (shape_item == NULL && PyErr_Occurred()){ - return NULL; - } - if (dims_item != NULL && shape_item == NULL) { - if (DEPRECATE("'shape' argument should be" - " used instead of 'dims'") < 0) { - return NULL; - } - if (PyDict_SetItemString(kwds, "shape", dims_item) < 0) { - return NULL; - } - if (PyDict_DelItemString(kwds, "dims") < 0) { - return NULL; - } - } - } + static char *kwlist[] = {"indices", "shape", "order", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&|O&:unravel_index", kwlist, @@ -1420,7 +1390,7 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) { PyObject *obj; PyObject *str; - #if (PY_VERSION_HEX >= 0x030700A2) + #if PY_VERSION_HEX >= 0x030700A2 && (!defined(PYPY_VERSION_NUM) || PYPY_VERSION_NUM > 0x07030300) const char *docstr; #else char *docstr; @@ -1500,27 +1470,12 @@ arr_add_docstring(PyObject *NPY_UNUSED(dummy), PyObject *args) Py_RETURN_NONE; } -#if defined NPY_HAVE_SSE2_INTRINSICS -#include -#endif - -#ifdef NPY_HAVE_NEON - typedef npy_uint64 uint64_unaligned __attribute__((aligned(16))); - static NPY_INLINE int32_t - sign_mask(uint8x16_t input) - { - int8x8_t m0 = vcreate_s8(0x0706050403020100ULL); - uint8x16_t v0 = vshlq_u8(vshrq_n_u8(input, 7), vcombine_s8(m0, m0)); - uint64x2_t v1 = vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(v0))); - return (int)vgetq_lane_u64(v1, 0) + ((int)vgetq_lane_u64(v1, 1) << 8); - } -#endif /* * This function packs boolean values in the input array into the bits of a * byte array. Truth values are determined as usual: 0 is false, everything * else is true. */ -static NPY_INLINE void +static NPY_GCC_OPT_3 NPY_INLINE void pack_inner(const char *inptr, npy_intp element_size, /* in bytes */ npy_intp n_in, @@ -1528,7 +1483,7 @@ pack_inner(const char *inptr, char *outptr, npy_intp n_out, npy_intp out_stride, - char order) + PACK_ORDER order) { /* * Loop through the elements of inptr. @@ -1540,62 +1495,64 @@ pack_inner(const char *inptr, npy_intp index = 0; int remain = n_in % 8; /* uneven bits */ -#if defined NPY_HAVE_SSE2_INTRINSICS && defined HAVE__M_FROM_INT64 +#if NPY_SIMD if (in_stride == 1 && element_size == 1 && n_out > 2) { - __m128i zero = _mm_setzero_si128(); + npyv_u8 v_zero = npyv_zero_u8(); /* don't handle non-full 8-byte remainder */ npy_intp vn_out = n_out - (remain ? 1 : 0); - vn_out -= (vn_out & 1); - for (index = 0; index < vn_out; index += 2) { - unsigned int r; - npy_uint64 a = *(npy_uint64*)inptr; - npy_uint64 b = *(npy_uint64*)(inptr + 8); - if (order == 'b') { - a = npy_bswap8(a); - b = npy_bswap8(b); + const int vstep = npyv_nlanes_u64; + const int vstepx4 = vstep * 4; + const int isAligned = npy_is_aligned(outptr, sizeof(npy_uint64)); + vn_out -= (vn_out & (vstep - 1)); + for (; index <= vn_out - vstepx4; index += vstepx4, inptr += npyv_nlanes_u8 * 4) { + npyv_u8 v0 = npyv_load_u8((const npy_uint8*)inptr); + npyv_u8 v1 = npyv_load_u8((const npy_uint8*)inptr + npyv_nlanes_u8 * 1); + npyv_u8 v2 = npyv_load_u8((const npy_uint8*)inptr + npyv_nlanes_u8 * 2); + npyv_u8 v3 = npyv_load_u8((const npy_uint8*)inptr + npyv_nlanes_u8 * 3); + if (order == PACK_ORDER_BIG) { + v0 = npyv_rev64_u8(v0); + v1 = npyv_rev64_u8(v1); + v2 = npyv_rev64_u8(v2); + v3 = npyv_rev64_u8(v3); } - - /* note x86 can load unaligned */ - __m128i v = _mm_set_epi64(_m_from_int64(b), _m_from_int64(a)); - /* false -> 0x00 and true -> 0xFF (there is no cmpneq) */ - v = _mm_cmpeq_epi8(v, zero); - v = _mm_cmpeq_epi8(v, zero); - /* extract msb of 16 bytes and pack it into 16 bit */ - r = _mm_movemask_epi8(v); - /* store result */ - memcpy(outptr, &r, 1); - outptr += out_stride; - memcpy(outptr, (char*)&r + 1, 1); - outptr += out_stride; - inptr += 16; - } - } -#elif defined NPY_HAVE_NEON - if (in_stride == 1 && element_size == 1 && n_out > 2) { - /* don't handle non-full 8-byte remainder */ - npy_intp vn_out = n_out - (remain ? 1 : 0); - vn_out -= (vn_out & 1); - for (index = 0; index < vn_out; index += 2) { - unsigned int r; - npy_uint64 a = *((uint64_unaligned*)inptr); - npy_uint64 b = *((uint64_unaligned*)(inptr + 8)); - if (order == 'b') { - a = npy_bswap8(a); - b = npy_bswap8(b); + npy_uint64 bb[4]; + bb[0] = npyv_tobits_b8(npyv_cmpneq_u8(v0, v_zero)); + bb[1] = npyv_tobits_b8(npyv_cmpneq_u8(v1, v_zero)); + bb[2] = npyv_tobits_b8(npyv_cmpneq_u8(v2, v_zero)); + bb[3] = npyv_tobits_b8(npyv_cmpneq_u8(v3, v_zero)); + if(out_stride == 1 && + (!NPY_ALIGNMENT_REQUIRED || isAligned)) { + npy_uint64 *ptr64 = (npy_uint64*)outptr; + #if NPY_SIMD_WIDTH == 16 + npy_uint64 bcomp = bb[0] | (bb[1] << 16) | (bb[2] << 32) | (bb[3] << 48); + ptr64[0] = bcomp; + #elif NPY_SIMD_WIDTH == 32 + ptr64[0] = bb[0] | (bb[1] << 32); + ptr64[1] = bb[2] | (bb[3] << 32); + #else + ptr64[0] = bb[0]; ptr64[1] = bb[1]; + ptr64[2] = bb[2]; ptr64[3] = bb[3]; + #endif + outptr += vstepx4; + } else { + for(int i = 0; i < 4; i++) { + for (int j = 0; j < vstep; j++) { + memcpy(outptr, (char*)&bb[i] + j, 1); + outptr += out_stride; + } + } + } + } + for (; index < vn_out; index += vstep, inptr += npyv_nlanes_u8) { + npyv_u8 va = npyv_load_u8((const npy_uint8*)inptr); + if (order == PACK_ORDER_BIG) { + va = npyv_rev64_u8(va); + } + npy_uint64 bb = npyv_tobits_b8(npyv_cmpneq_u8(va, v_zero)); + for (int i = 0; i < vstep; ++i) { + memcpy(outptr, (char*)&bb + i, 1); + outptr += out_stride; } - uint64x2_t v = vcombine_u64(vcreate_u64(a), vcreate_u64(b)); - uint64x2_t zero = vdupq_n_u64(0); - /* false -> 0x00 and true -> 0xFF */ - v = vreinterpretq_u64_u8(vmvnq_u8(vceqq_u8(vreinterpretq_u8_u64(v), vreinterpretq_u8_u64(zero)))); - /* extract msb of 16 bytes and pack it into 16 bit */ - uint8x16_t input = vreinterpretq_u8_u64(v); - r = sign_mask(input); - /* store result */ - memcpy(outptr, &r, 1); - outptr += out_stride; - memcpy(outptr, (char*)&r + 1, 1); - outptr += out_stride; - inptr += 16; } } #endif @@ -1606,14 +1563,11 @@ pack_inner(const char *inptr, /* Don't reset index. Just handle remainder of above block */ for (; index < n_out; index++) { unsigned char build = 0; - int i, maxi; - npy_intp j; - - maxi = (index == n_out - 1) ? remain : 8; - if (order == 'b') { - for (i = 0; i < maxi; i++) { + int maxi = (index == n_out - 1) ? remain : 8; + if (order == PACK_ORDER_BIG) { + for (int i = 0; i < maxi; i++) { build <<= 1; - for (j = 0; j < element_size; j++) { + for (npy_intp j = 0; j < element_size; j++) { build |= (inptr[j] != 0); } inptr += in_stride; @@ -1624,9 +1578,9 @@ pack_inner(const char *inptr, } else { - for (i = 0; i < maxi; i++) { + for (int i = 0; i < maxi; i++) { build >>= 1; - for (j = 0; j < element_size; j++) { + for (npy_intp j = 0; j < element_size; j++) { build |= (inptr[j] != 0) ? 128 : 0; } inptr += in_stride; @@ -1720,13 +1674,13 @@ pack_bits(PyObject *input, int axis, char order) Py_XDECREF(ot); goto fail; } - + const PACK_ORDER ordere = order == 'b' ? PACK_ORDER_BIG : PACK_ORDER_LITTLE; NPY_BEGIN_THREADS_THRESHOLDED(PyArray_DIM(out, axis)); while (PyArray_ITER_NOTDONE(it)) { pack_inner(PyArray_ITER_DATA(it), PyArray_ITEMSIZE(new), PyArray_DIM(new, axis), PyArray_STRIDE(new, axis), PyArray_ITER_DATA(ot), PyArray_DIM(out, axis), - PyArray_STRIDE(out, axis), order); + PyArray_STRIDE(out, axis), ordere); PyArray_ITER_NEXT(it); PyArray_ITER_NEXT(ot); } diff --git a/numpy/core/src/multiarray/conversion_utils.c b/numpy/core/src/multiarray/conversion_utils.c index dd18f71fd03e..3c4c21dedd23 100644 --- a/numpy/core/src/multiarray/conversion_utils.c +++ b/numpy/core/src/multiarray/conversion_utils.c @@ -715,6 +715,78 @@ PyArray_ConvertClipmodeSequence(PyObject *object, NPY_CLIPMODE *modes, int n) return NPY_SUCCEED; } +static int correlatemode_parser(char const *str, Py_ssize_t length, void *data) +{ + NPY_CORRELATEMODE *val = (NPY_CORRELATEMODE *)data; + int is_exact = 0; + + if (length < 1) { + return -1; + } + if (str[0] == 'V' || str[0] == 'v') { + *val = NPY_VALID; + is_exact = (length == 5 && strcmp(str, "valid") == 0); + } + else if (str[0] == 'S' || str[0] == 's') { + *val = NPY_SAME; + is_exact = (length == 4 && strcmp(str, "same") == 0); + } + else if (str[0] == 'F' || str[0] == 'f') { + *val = NPY_FULL; + is_exact = (length == 4 && strcmp(str, "full") == 0); + } + else { + return -1; + } + + /* Filters out the case sensitive/non-exact + * match inputs and other inputs and outputs DeprecationWarning + */ + if (!is_exact) { + /* Numpy 1.21, 2021-01-19 */ + if (DEPRECATE("inexact matches and case insensitive matches for " + "convolve/correlate mode are deprecated, please " + "use one of 'valid', 'same', or 'full' instead.") < 0) { + return -1; + } + } + + return 0; +} + +/* + * Convert an object to NPY_VALID / NPY_SAME / NPY_FULL + */ +NPY_NO_EXPORT int +PyArray_CorrelatemodeConverter(PyObject *object, NPY_CORRELATEMODE *val) +{ + if (PyUnicode_Check(object)) { + return string_converter_helper( + object, (void *)val, correlatemode_parser, "mode", + "must be one of 'valid', 'same', or 'full'"); + } + + else { + /* For users passing integers */ + int number = PyArray_PyIntAsInt(object); + if (error_converting(number)) { + PyErr_SetString(PyExc_TypeError, + "convolve/correlate mode not understood"); + return NPY_FAIL; + } + if (number <= (int) NPY_FULL + && number >= (int) NPY_VALID) { + *val = (NPY_CORRELATEMODE) number; + return NPY_SUCCEED; + } + else { + PyErr_Format(PyExc_ValueError, + "integer convolve/correlate mode must be 0, 1, or 2"); + return NPY_FAIL; + } + } +} + static int casting_parser(char const *str, Py_ssize_t length, void *data) { NPY_CASTING *casting = (NPY_CASTING *)data; diff --git a/numpy/core/src/multiarray/conversion_utils.h b/numpy/core/src/multiarray/conversion_utils.h index bee0c6064d05..7d1871c43ddb 100644 --- a/numpy/core/src/multiarray/conversion_utils.h +++ b/numpy/core/src/multiarray/conversion_utils.h @@ -42,6 +42,9 @@ PyArray_TypestrConvert(int itemsize, int gentype); NPY_NO_EXPORT PyObject * PyArray_IntTupleFromIntp(int len, npy_intp const *vals); +NPY_NO_EXPORT int +PyArray_CorrelatemodeConverter(PyObject *object, NPY_CORRELATEMODE *val); + NPY_NO_EXPORT int PyArray_SelectkindConverter(PyObject *obj, NPY_SELECTKIND *selectkind); diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index f9dd35a73e18..d197a4bea31e 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -8,6 +8,7 @@ #include "numpy/arrayscalars.h" #include "npy_config.h" +#include "lowlevel_strided_loops.h" #include "npy_pycompat.h" #include "numpy/npy_math.h" @@ -16,15 +17,18 @@ #include "common.h" #include "ctors.h" #include "dtypemeta.h" +#include "common_dtype.h" #include "scalartypes.h" #include "mapping.h" #include "legacy_dtype_implementation.h" +#include "abstractdtypes.h" #include "convert_datatype.h" #include "_datetime.h" #include "datetime_strings.h" #include "array_method.h" #include "usertypes.h" +#include "dtype_transfer.h" /* @@ -61,10 +65,16 @@ PyArray_GetObjectToGenericCastingImpl(void); * @returns A castingimpl (PyArrayDTypeMethod *), None or NULL with an * error set. */ -static PyObject * +NPY_NO_EXPORT PyObject * PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) { - PyObject *res = PyDict_GetItem(from->castingimpls, (PyObject *)to); + PyObject *res; + if (from == to) { + res = from->within_dtype_castingimpl; + } + else { + res = PyDict_GetItemWithError(from->castingimpls, (PyObject *)to); + } if (res != NULL || PyErr_Occurred()) { Py_XINCREF(res); return res; @@ -90,7 +100,7 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) else if (from->type_num < NPY_NTYPES && to->type_num < NPY_NTYPES) { /* All builtin dtypes have their casts explicitly defined. */ PyErr_Format(PyExc_RuntimeError, - "builtin cast from %S to %s not found, this should not " + "builtin cast from %S to %S not found, this should not " "be possible.", from, to); return NULL; } @@ -131,6 +141,12 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) if (res == NULL) { return NULL; } + if (from == to) { + PyErr_Format(PyExc_RuntimeError, + "Internal NumPy error, within-DType cast missing for %S!", from); + Py_DECREF(res); + return NULL; + } if (PyDict_SetItem(from->castingimpls, (PyObject *)to, res) < 0) { Py_DECREF(res); return NULL; @@ -342,6 +358,45 @@ PyArray_CastAnyTo(PyArrayObject *out, PyArrayObject *mp) } +static NPY_CASTING +_get_cast_safety_from_castingimpl(PyArrayMethodObject *castingimpl, + PyArray_DTypeMeta *dtypes[2], PyArray_Descr *from, PyArray_Descr *to) +{ + PyArray_Descr *descrs[2] = {from, to}; + PyArray_Descr *out_descrs[2]; + + NPY_CASTING casting = castingimpl->resolve_descriptors( + castingimpl, dtypes, descrs, out_descrs); + if (casting < 0) { + return -1; + } + /* The returned descriptors may not match, requiring a second check */ + if (out_descrs[0] != descrs[0]) { + NPY_CASTING from_casting = PyArray_GetCastSafety( + descrs[0], out_descrs[0], NULL); + casting = PyArray_MinCastSafety(casting, from_casting); + if (casting < 0) { + goto finish; + } + } + if (descrs[1] != NULL && out_descrs[1] != descrs[1]) { + NPY_CASTING from_casting = PyArray_GetCastSafety( + descrs[1], out_descrs[1], NULL); + casting = PyArray_MinCastSafety(casting, from_casting); + if (casting < 0) { + goto finish; + } + } + + finish: + Py_DECREF(out_descrs[0]); + Py_DECREF(out_descrs[1]); + /* NPY_NO_CASTING has to be used for (NPY_EQUIV_CASTING|_NPY_CAST_IS_VIEW) */ + assert(casting != (NPY_EQUIV_CASTING|_NPY_CAST_IS_VIEW)); + return casting; +} + + /** * Given two dtype instances, find the correct casting safety. * @@ -359,7 +414,6 @@ NPY_NO_EXPORT NPY_CASTING PyArray_GetCastSafety( PyArray_Descr *from, PyArray_Descr *to, PyArray_DTypeMeta *to_dtype) { - NPY_CASTING casting; if (to != NULL) { to_dtype = NPY_DTYPE(to); } @@ -373,41 +427,68 @@ PyArray_GetCastSafety( } PyArrayMethodObject *castingimpl = (PyArrayMethodObject *)meth; - PyArray_DTypeMeta *dtypes[2] = {NPY_DTYPE(from), to_dtype}; - PyArray_Descr *descrs[2] = {from, to}; - PyArray_Descr *out_descrs[2]; - - casting = castingimpl->resolve_descriptors( - castingimpl, dtypes, descrs, out_descrs); + NPY_CASTING casting = _get_cast_safety_from_castingimpl(castingimpl, + dtypes, from, to); Py_DECREF(meth); - if (casting < 0) { + + return casting; +} + + +/** + * Check whether a cast is safe, see also `PyArray_GetCastSafety` for + * a similiar function. Unlike GetCastSafety, this function checks the + * `castingimpl->casting` when available. This allows for two things: + * + * 1. It avoids calling `resolve_descriptors` in some cases. + * 2. Strings need to discover the length, but in some cases we know that the + * cast is valid (assuming the string length is discovered first). + * + * The latter means that a `can_cast` could return True, but the cast fail + * because the parametric type cannot guess the correct output descriptor. + * (I.e. if `object_arr.astype("S")` did _not_ inspect the objects, and the + * user would have to guess the string length.) + * + * @param casting the requested casting safety. + * @param from + * @param to The descriptor to cast to (may be NULL) + * @param to_dtype If `to` is NULL, must pass the to_dtype (otherwise this + * is ignored). + * @return 0 for an invalid cast, 1 for a valid and -1 for an error. + */ +static int +PyArray_CheckCastSafety(NPY_CASTING casting, + PyArray_Descr *from, PyArray_Descr *to, PyArray_DTypeMeta *to_dtype) +{ + if (to != NULL) { + to_dtype = NPY_DTYPE(to); + } + PyObject *meth = PyArray_GetCastingImpl(NPY_DTYPE(from), to_dtype); + if (meth == NULL) { return -1; } - /* The returned descriptors may not match, requiring a second check */ - if (out_descrs[0] != descrs[0]) { - NPY_CASTING from_casting = PyArray_GetCastSafety( - descrs[0], out_descrs[0], NULL); - casting = PyArray_MinCastSafety(casting, from_casting); - if (casting < 0) { - goto finish; - } + if (meth == Py_None) { + Py_DECREF(Py_None); + return -1; } - if (descrs[1] != NULL && out_descrs[1] != descrs[1]) { - NPY_CASTING from_casting = PyArray_GetCastSafety( - descrs[1], out_descrs[1], NULL); - casting = PyArray_MinCastSafety(casting, from_casting); - if (casting < 0) { - goto finish; - } + PyArrayMethodObject *castingimpl = (PyArrayMethodObject *)meth; + + if (PyArray_MinCastSafety(castingimpl->casting, casting) == casting) { + /* No need to check using `castingimpl.resolve_descriptors()` */ + Py_DECREF(meth); + return 1; } - finish: - Py_DECREF(out_descrs[0]); - Py_DECREF(out_descrs[1]); - /* NPY_NO_CASTING has to be used for (NPY_EQUIV_CASTING|_NPY_CAST_IS_VIEW) */ - assert(casting != (NPY_EQUIV_CASTING|_NPY_CAST_IS_VIEW)); - return casting; + PyArray_DTypeMeta *dtypes[2] = {NPY_DTYPE(from), to_dtype}; + NPY_CASTING safety = _get_cast_safety_from_castingimpl(castingimpl, + dtypes, from, to); + Py_DECREF(meth); + /* If casting is the smaller (or equal) safety we match */ + if (safety < 0) { + return -1; + } + return PyArray_MinCastSafety(safety, casting) == casting; } @@ -417,7 +498,24 @@ PyArray_GetCastSafety( NPY_NO_EXPORT int PyArray_CanCastSafely(int fromtype, int totype) { -#if NPY_USE_NEW_CASTINGIMPL + /* Identity */ + if (fromtype == totype) { + return 1; + } + /* + * As a micro-optimization, keep the cast table around. This can probably + * be removed as soon as the ufunc loop lookup is modified (presumably + * before the 1.21 release). It does no harm, but the main user of this + * function is the ufunc-loop lookup calling it until a loop matches! + * + * (The table extends further, but is not strictly correct for void). + * TODO: Check this! + */ + if ((unsigned int)fromtype <= NPY_CLONGDOUBLE && + (unsigned int)totype <= NPY_CLONGDOUBLE) { + return _npy_can_cast_safely_table[fromtype][totype]; + } + PyArray_DTypeMeta *from = PyArray_DTypeFromTypeNum(fromtype); if (from == NULL) { PyErr_WriteUnraisable(NULL); @@ -444,9 +542,6 @@ PyArray_CanCastSafely(int fromtype, int totype) int res = PyArray_MinCastSafety(safety, NPY_SAFE_CASTING) == NPY_SAFE_CASTING; Py_DECREF(castingimpl); return res; -#else - return PyArray_LegacyCanCastSafely(fromtype, totype); -#endif } @@ -460,11 +555,7 @@ PyArray_CanCastSafely(int fromtype, int totype) NPY_NO_EXPORT npy_bool PyArray_CanCastTo(PyArray_Descr *from, PyArray_Descr *to) { -#if NPY_USE_NEW_CASTINGIMPL return PyArray_CanCastTypeTo(from, to, NPY_SAFE_CASTING); -#else - return PyArray_LegacyCanCastTo(from, to); -#endif } @@ -539,7 +630,8 @@ NPY_NO_EXPORT npy_bool PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, NPY_CASTING casting) { -#if NPY_USE_NEW_CASTINGIMPL + PyArray_DTypeMeta *to_dtype = NPY_DTYPE(to); + /* * NOTE: This code supports U and S, this is identical to the code * in `ctors.c` which does not allow these dtypes to be attached @@ -551,24 +643,21 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, * TODO: We should grow support for `np.can_cast("d", "S")` being * different from `np.can_cast("d", "S0")` here, at least for * the python side API. + * The `to = NULL` branch, which considers "S0" to be "flexible" + * should probably be deprecated. + * (This logic is duplicated in `PyArray_CanCastArrayTo`) */ - NPY_CASTING safety; if (PyDataType_ISUNSIZED(to) && to->subarray == NULL) { - safety = PyArray_GetCastSafety(from, NULL, NPY_DTYPE(to)); - } - else { - safety = PyArray_GetCastSafety(from, to, NPY_DTYPE(to)); + to = NULL; /* consider mainly S0 and U0 as S and U */ } - if (safety < 0) { + int is_valid = PyArray_CheckCastSafety(casting, from, to, to_dtype); + /* Clear any errors and consider this unsafe (should likely be changed) */ + if (is_valid < 0) { PyErr_Clear(); return 0; } - /* If casting is the smaller (or equal) safety we match */ - return PyArray_MinCastSafety(safety, casting) == casting; -#else - return PyArray_LegacyCanCastTypeTo(from, to, casting); -#endif + return is_valid; } @@ -576,36 +665,53 @@ PyArray_CanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, static int min_scalar_type_num(char *valueptr, int type_num, int *is_small_unsigned); + +/* + * NOTE: This function uses value based casting logic for scalars. It will + * require updates when we phase out value-based-casting. + */ NPY_NO_EXPORT npy_bool can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyArray_Descr *to, NPY_CASTING casting) { - int swap; - int is_small_unsigned = 0, type_num; - npy_bool ret; - PyArray_Descr *dtype; - - /* An aligned memory buffer large enough to hold any type */ - npy_longlong value[4]; - /* * If the two dtypes are actually references to the same object * or if casting type is forced unsafe then always OK. + * + * TODO: Assuming that unsafe casting always works is not actually correct */ if (scal_type == to || casting == NPY_UNSAFE_CASTING ) { return 1; } + int valid = PyArray_CheckCastSafety(casting, scal_type, to, NPY_DTYPE(to)); + if (valid == 1) { + /* This is definitely a valid cast. */ + return 1; + } + if (valid < 0) { + /* Probably must return 0, but just keep trying for now. */ + PyErr_Clear(); + } + /* - * If the scalar isn't a number, or the rule is stricter than - * NPY_SAFE_CASTING, use the straight type-based rules + * If the scalar isn't a number, value-based casting cannot kick in and + * we must not attempt it. + * (Additional fast-checks would be possible, but probably unnecessary.) */ - if (!PyTypeNum_ISNUMBER(scal_type->type_num) || - casting < NPY_SAFE_CASTING) { - return PyArray_CanCastTypeTo(scal_type, to, casting); + if (!PyTypeNum_ISNUMBER(scal_type->type_num)) { + return 0; } - swap = !PyArray_ISNBO(scal_type->byteorder); + /* + * At this point we have to check value-based casting. + */ + PyArray_Descr *dtype; + int is_small_unsigned = 0, type_num; + /* An aligned memory buffer large enough to hold any builtin numeric type */ + npy_longlong value[4]; + + int swap = !PyArray_ISNBO(scal_type->byteorder); scal_type->f->copyswap(&value, scal_data, swap, NULL); type_num = min_scalar_type_num((char *)&value, scal_type->type_num, @@ -631,7 +737,7 @@ can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyObject_Print(to, stdout, 0); printf("\n"); #endif - ret = PyArray_CanCastTypeTo(dtype, to, casting); + npy_bool ret = PyArray_CanCastTypeTo(dtype, to, casting); Py_DECREF(dtype); return ret; } @@ -647,14 +753,29 @@ PyArray_CanCastArrayTo(PyArrayObject *arr, PyArray_Descr *to, NPY_CASTING casting) { PyArray_Descr *from = PyArray_DESCR(arr); + PyArray_DTypeMeta *to_dtype = NPY_DTYPE(to); + + /* NOTE, TODO: The same logic as `PyArray_CanCastTypeTo`: */ + if (PyDataType_ISUNSIZED(to) && to->subarray == NULL) { + to = NULL; + } - /* If it's a scalar, check the value */ - if (PyArray_NDIM(arr) == 0 && !PyArray_HASFIELDS(arr)) { + /* + * If it's a scalar, check the value. (This only currently matters for + * numeric types and for `to == NULL` it can't be numeric.) + */ + if (PyArray_NDIM(arr) == 0 && !PyArray_HASFIELDS(arr) && to != NULL) { return can_cast_scalar_to(from, PyArray_DATA(arr), to, casting); } - /* Otherwise, use the standard rules */ - return PyArray_CanCastTypeTo(from, to, casting); + /* Otherwise, use the standard rules (same as `PyArray_CanCastTypeTo`) */ + int is_valid = PyArray_CheckCastSafety(casting, from, to, to_dtype); + /* Clear any errors and consider this unsafe (should likely be changed) */ + if (is_valid < 0) { + PyErr_Clear(); + return 0; + } + return is_valid; } @@ -828,7 +949,6 @@ PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) return descr; } -#if NPY_USE_NEW_CASTINGIMPL PyObject *tmp = PyArray_GetCastingImpl(NPY_DTYPE(descr), given_DType); if (tmp == NULL || tmp == Py_None) { Py_XDECREF(tmp); @@ -851,67 +971,76 @@ PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType) error:; /* (; due to compiler limitations) */ PyObject *err_type = NULL, *err_value = NULL, *err_traceback = NULL; PyErr_Fetch(&err_type, &err_value, &err_traceback); - PyErr_Format(PyExc_ValueError, + PyErr_Format(PyExc_TypeError, "cannot cast dtype %S to %S.", descr, given_DType); - npy_PyErr_ChainExceptions(err_type, err_value, err_traceback); + npy_PyErr_ChainExceptionsCause(err_type, err_value, err_traceback); return NULL; - -#else /* NPY_USE_NEW_CASTS */ - if (!given_DType->legacy) { - PyErr_SetString(PyExc_NotImplementedError, - "Must use casting to find the correct DType for a parametric " - "user DType. This is not yet implemented (this error should be " - "unreachable)."); - return NULL; - } - - PyArray_Descr *flex_dtype = PyArray_DescrNew(given_DType->singleton); - return PyArray_AdaptFlexibleDType(descr, flex_dtype); -#endif /* NPY_USE_NEW_CASTS */ } -/** - * This function defines the common DType operator. - * - * Note that the common DType will not be "object" (unless one of the dtypes - * is object), even though object can technically represent all values - * correctly. - * - * TODO: Before exposure, we should review the return value (e.g. no error - * when no common DType is found). +/* + * Helper to find the target descriptor for multiple arrays given an input + * one that may be a DType class (e.g. "U" or "S"). + * Works with arrays, since that is what `concatenate` works with. However, + * unlike `np.array(...)` or `arr.astype()` we will never inspect the array's + * content, which means that object arrays can only be cast to strings if a + * fixed width is provided (same for string -> generic datetime). * - * @param dtype1 DType class to find the common type for. - * @param dtype2 Second DType class. - * @return The common DType or NULL with an error set + * As this function uses `PyArray_ExtractDTypeAndDescriptor`, it should + * eventually be refactored to move the step to an earlier point. */ -NPY_NO_EXPORT PyArray_DTypeMeta * -PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2) +NPY_NO_EXPORT PyArray_Descr * +PyArray_FindConcatenationDescriptor( + npy_intp n, PyArrayObject **arrays, PyObject *requested_dtype) { - if (dtype1 == dtype2) { - Py_INCREF(dtype1); - return dtype1; + if (requested_dtype == NULL) { + return PyArray_LegacyResultType(n, arrays, 0, NULL); } PyArray_DTypeMeta *common_dtype; - - common_dtype = dtype1->common_dtype(dtype1, dtype2); - if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { - Py_DECREF(common_dtype); - common_dtype = dtype2->common_dtype(dtype2, dtype1); - } - if (common_dtype == NULL) { + PyArray_Descr *result = NULL; + if (PyArray_ExtractDTypeAndDescriptor( + requested_dtype, &result, &common_dtype) < 0) { return NULL; } - if (common_dtype == (PyArray_DTypeMeta *)Py_NotImplemented) { - Py_DECREF(Py_NotImplemented); - PyErr_Format(PyExc_TypeError, - "The DTypes %S and %S do not have a common DType. " - "For example they cannot be stored in a single array unless " - "the dtype is `object`.", dtype1, dtype2); - return NULL; + if (result != NULL) { + if (result->subarray != NULL) { + PyErr_Format(PyExc_TypeError, + "The dtype `%R` is not a valid dtype for concatenation " + "since it is a subarray dtype (the subarray dimensions " + "would be added as array dimensions).", result); + Py_SETREF(result, NULL); + } + goto finish; } - return common_dtype; + assert(n > 0); /* concatenate requires at least one array input. */ + + /* + * NOTE: This code duplicates `PyArray_CastToDTypeAndPromoteDescriptors` + * to use arrays, copying the descriptors seems not better. + */ + PyArray_Descr *descr = PyArray_DESCR(arrays[0]); + result = PyArray_CastDescrToDType(descr, common_dtype); + if (result == NULL || n == 1) { + goto finish; + } + for (npy_intp i = 1; i < n; i++) { + descr = PyArray_DESCR(arrays[i]); + PyArray_Descr *curr = PyArray_CastDescrToDType(descr, common_dtype); + if (curr == NULL) { + Py_SETREF(result, NULL); + goto finish; + } + Py_SETREF(result, common_dtype->common_instance(result, curr)); + Py_DECREF(curr); + if (result == NULL) { + goto finish; + } + } + + finish: + Py_DECREF(common_dtype); + return result; } @@ -937,6 +1066,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) } if (!common_dtype->parametric) { + /* Note that this path loses all metadata */ res = common_dtype->default_descr(common_dtype); Py_DECREF(common_dtype); return res; @@ -970,28 +1100,17 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) * Produces the smallest size and lowest kind type to which all * input types can be cast. * - * Equivalent to functools.reduce(PyArray_PromoteTypes, types) + * Roughly equivalent to functools.reduce(PyArray_PromoteTypes, types) + * but uses a more complex pairwise approach. */ NPY_NO_EXPORT PyArray_Descr * PyArray_PromoteTypeSequence(PyArray_Descr **types, npy_intp ntypes) { - npy_intp i; - PyArray_Descr *ret = NULL; if (ntypes == 0) { PyErr_SetString(PyExc_TypeError, "at least one type needed to promote"); return NULL; } - ret = types[0]; - Py_INCREF(ret); - for (i = 1; i < ntypes; ++i) { - PyArray_Descr *tmp = PyArray_PromoteTypes(types[i], ret); - Py_DECREF(ret); - ret = tmp; - if (ret == NULL) { - return NULL; - } - } - return ret; + return PyArray_ResultType(0, NULL, ntypes, types); } /* @@ -1357,6 +1476,8 @@ dtype_kind_to_simplified_ordering(char kind) * If the scalars are of a lower or same category as the arrays, they may be * demoted to a lower type within their category (the lowest type they can * be cast to safely according to scalar casting rules). + * + * If any new style dtype is involved (non-legacy), always returns 0. */ NPY_NO_EXPORT int should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, @@ -1373,6 +1494,9 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, /* Compute the maximum "kinds" and whether everything is scalar */ for (npy_intp i = 0; i < narrs; ++i) { + if (!NPY_DTYPE(PyArray_DESCR(arr[i]))->legacy) { + return 0; + } if (PyArray_NDIM(arr[i]) == 0) { int kind = dtype_kind_to_simplified_ordering( PyArray_DESCR(arr[i])->kind); @@ -1394,6 +1518,9 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, * finish computing the max array kind */ for (npy_intp i = 0; i < ndtypes; ++i) { + if (!NPY_DTYPE(dtypes[i])->legacy) { + return 0; + } int kind = dtype_kind_to_simplified_ordering(dtypes[i]->kind); if (kind > max_array_kind) { max_array_kind = kind; @@ -1410,6 +1537,209 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, /*NUMPY_API + * + * Produces the result type of a bunch of inputs, using the same rules + * as `np.result_type`. + * + * NOTE: This function is expected to through a transitional period or + * change behaviour. DTypes should always be strictly enforced for + * 0-D arrays, while "weak DTypes" will be used to represent Python + * integers, floats, and complex in all cases. + * (Within this function, these are currently flagged on the array + * object to work through `np.result_type`, this may change.) + * + * Until a time where this transition is complete, we probably cannot + * add new "weak DTypes" or allow users to create their own. + */ +NPY_NO_EXPORT PyArray_Descr * +PyArray_ResultType( + npy_intp narrs, PyArrayObject *arrs[], + npy_intp ndtypes, PyArray_Descr *descrs[]) +{ + PyArray_Descr *result = NULL; + + if (narrs + ndtypes <= 1) { + /* If the input is a single value, skip promotion. */ + if (narrs == 1) { + result = PyArray_DTYPE(arrs[0]); + } + else if (ndtypes == 1) { + result = descrs[0]; + } + else { + PyErr_SetString(PyExc_TypeError, + "no arrays or types available to calculate result type"); + return NULL; + } + return ensure_dtype_nbo(result); + } + + void **info_on_heap = NULL; + void *_info_on_stack[NPY_MAXARGS * 2]; + PyArray_DTypeMeta **all_DTypes; + PyArray_Descr **all_descriptors; + + if (narrs + ndtypes > NPY_MAXARGS) { + info_on_heap = PyMem_Malloc(2 * (narrs+ndtypes) * sizeof(PyObject *)); + if (info_on_heap == NULL) { + PyErr_NoMemory(); + return NULL; + } + all_DTypes = (PyArray_DTypeMeta **)info_on_heap; + all_descriptors = (PyArray_Descr **)(info_on_heap + narrs + ndtypes); + } + else { + all_DTypes = (PyArray_DTypeMeta **)_info_on_stack; + all_descriptors = (PyArray_Descr **)(_info_on_stack + narrs + ndtypes); + } + + /* Copy all dtypes into a single array defining non-value-based behaviour */ + for (npy_intp i=0; i < ndtypes; i++) { + all_DTypes[i] = NPY_DTYPE(descrs[i]); + Py_INCREF(all_DTypes[i]); + all_descriptors[i] = descrs[i]; + } + + int at_least_one_scalar = 0; + int all_pyscalar = ndtypes == 0; + for (npy_intp i=0, i_all=ndtypes; i < narrs; i++, i_all++) { + /* Array descr is also the correct "default" for scalars: */ + if (PyArray_NDIM(arrs[i]) == 0) { + at_least_one_scalar = 1; + } + + if (!(PyArray_FLAGS(arrs[i]) & _NPY_ARRAY_WAS_PYSCALAR)) { + /* This was not a scalar with an abstract DType */ + all_descriptors[i_all] = PyArray_DTYPE(arrs[i]); + all_DTypes[i_all] = NPY_DTYPE(all_descriptors[i_all]); + Py_INCREF(all_DTypes[i_all]); + all_pyscalar = 0; + continue; + } + + /* + * The original was a Python scalar with an abstract DType. + * In a future world, this type of code may need to work on the + * DType level first and discover those from the original value. + * But, right now we limit the logic to int, float, and complex + * and do it here to allow for a transition without losing all of + * our remaining sanity. + */ + if (PyArray_ISFLOAT(arrs[i])) { + all_DTypes[i_all] = &PyArray_PyFloatAbstractDType; + } + else if (PyArray_ISCOMPLEX(arrs[i])) { + all_DTypes[i_all] = &PyArray_PyComplexAbstractDType; + } + else { + /* N.B.: Could even be an object dtype here for large ints */ + all_DTypes[i_all] = &PyArray_PyIntAbstractDType; + } + Py_INCREF(all_DTypes[i_all]); + /* + * Leave the decriptor empty, if we need it, we will have to go + * to more extreme lengths unfortunately. + */ + all_descriptors[i_all] = NULL; + } + + PyArray_DTypeMeta *common_dtype = PyArray_PromoteDTypeSequence( + narrs+ndtypes, all_DTypes); + for (npy_intp i=0; i < narrs+ndtypes; i++) { + Py_DECREF(all_DTypes[i]); + } + if (common_dtype == NULL) { + goto error; + } + + if (common_dtype->abstract) { + /* (ab)use default descriptor to define a default */ + PyArray_Descr *tmp_descr = common_dtype->default_descr(common_dtype); + if (tmp_descr == NULL) { + goto error; + } + Py_INCREF(NPY_DTYPE(tmp_descr)); + Py_SETREF(common_dtype, NPY_DTYPE(tmp_descr)); + Py_DECREF(tmp_descr); + } + + /* + * NOTE: Code duplicates `PyArray_CastToDTypeAndPromoteDescriptors`, but + * supports special handling of the abstract values. + */ + if (!common_dtype->parametric) { + /* Note that this "fast" path loses all metadata */ + result = common_dtype->default_descr(common_dtype); + } + else { + result = PyArray_CastDescrToDType(all_descriptors[0], common_dtype); + + for (npy_intp i = 1; i < ndtypes+narrs; i++) { + PyArray_Descr *curr; + if (NPY_LIKELY(i < ndtypes || + !(PyArray_FLAGS(arrs[i-ndtypes]) & _NPY_ARRAY_WAS_PYSCALAR))) { + curr = PyArray_CastDescrToDType(all_descriptors[i], common_dtype); + } + else { + /* + * Unlike `PyArray_CastToDTypeAndPromoteDescriptors` deal with + * plain Python values "graciously". This recovers the original + * value the long route, but it should almost never happen... + */ + PyObject *tmp = PyArray_GETITEM( + arrs[i-ndtypes], PyArray_BYTES(arrs[i-ndtypes])); + if (tmp == NULL) { + goto error; + } + curr = common_dtype->discover_descr_from_pyobject(common_dtype, tmp); + Py_DECREF(tmp); + } + if (curr == NULL) { + goto error; + } + Py_SETREF(result, common_dtype->common_instance(result, curr)); + Py_DECREF(curr); + if (result == NULL) { + goto error; + } + } + } + + /* + * Unfortunately, when 0-D "scalar" arrays are involved and mixed, we + * have to use the value-based logic. The intention is to move away from + * the complex logic arising from it. We thus fall back to the legacy + * version here. + * It may be possible to micro-optimize this to skip some of the above + * logic when this path is necessary. + */ + if (at_least_one_scalar && !all_pyscalar && result->type_num < NPY_NTYPES) { + PyArray_Descr *legacy_result = PyArray_LegacyResultType( + narrs, arrs, ndtypes, descrs); + if (legacy_result == NULL) { + /* + * Going from error to success should not really happen, but is + * probably OK if it does. + */ + goto error; + } + /* Return the old "legacy" result (could warn here if different) */ + Py_SETREF(result, legacy_result); + } + + Py_DECREF(common_dtype); + PyMem_Free(info_on_heap); + return result; + + error: + Py_XDECREF(result); + Py_XDECREF(common_dtype); + PyMem_Free(info_on_heap); + return NULL; +} + + +/* * Produces the result type of a bunch of inputs, using the UFunc * type promotion rules. Use this function when you have a set of * input arrays, and need to determine an output array dtype. @@ -1421,11 +1751,11 @@ should_use_min_scalar(npy_intp narrs, PyArrayObject **arr, * Otherwise, does a type promotion on the MinScalarType * of all the inputs. Data types passed directly are treated as array * types. - * */ NPY_NO_EXPORT PyArray_Descr * -PyArray_ResultType(npy_intp narrs, PyArrayObject **arr, - npy_intp ndtypes, PyArray_Descr **dtypes) +PyArray_LegacyResultType( + npy_intp narrs, PyArrayObject **arr, + npy_intp ndtypes, PyArray_Descr **dtypes) { npy_intp i; @@ -1524,6 +1854,49 @@ PyArray_ResultType(npy_intp narrs, PyArrayObject **arr, } } +/** + * Promotion of descriptors (of arbitrary DType) to their correctly + * promoted instances of the given DType. + * I.e. the given DType could be a string, which then finds the correct + * string length, given all `descrs`. + * + * @param ndescrs number of descriptors to cast and find the common instance. + * At least one must be passed in. + * @param descrs The descriptors to work with. + * @param DType The DType of the desired output descriptor. + */ +NPY_NO_EXPORT PyArray_Descr * +PyArray_CastToDTypeAndPromoteDescriptors( + npy_intp ndescr, PyArray_Descr *descrs[], PyArray_DTypeMeta *DType) +{ + assert(ndescr > 0); + + PyArray_Descr *result = PyArray_CastDescrToDType(descrs[0], DType); + if (result == NULL || ndescr == 1) { + return result; + } + if (!DType->parametric) { + /* Note that this "fast" path loses all metadata */ + Py_DECREF(result); + return DType->default_descr(DType); + } + + for (npy_intp i = 1; i < ndescr; i++) { + PyArray_Descr *curr = PyArray_CastDescrToDType(descrs[i], DType); + if (curr == NULL) { + Py_DECREF(result); + return NULL; + } + Py_SETREF(result, DType->common_instance(result, curr)); + Py_DECREF(curr); + if (result == NULL) { + return NULL; + } + } + return result; +} + + /*NUMPY_API * Is the typenum valid? */ @@ -1808,7 +2181,7 @@ PyArray_ConvertToCommonType(PyObject *op, int *retn) * @return 0 on success -1 on failure. */ NPY_NO_EXPORT int -PyArray_AddCastingImplmentation(PyBoundArrayMethodObject *meth) +PyArray_AddCastingImplementation(PyBoundArrayMethodObject *meth) { if (meth->method->nin != 1 || meth->method->nout != 1) { PyErr_SetString(PyExc_TypeError, @@ -1816,6 +2189,11 @@ PyArray_AddCastingImplmentation(PyBoundArrayMethodObject *meth) return -1; } if (meth->dtypes[0] == meth->dtypes[1]) { + /* + * The method casting between instances of the same dtype is special, + * since it is common, it is stored explicitly (currently) and must + * obey additional constraints to ensure convenient casting. + */ if (!(meth->method->flags & NPY_METH_SUPPORTS_UNALIGNED)) { PyErr_Format(PyExc_TypeError, "A cast where input and output DType (class) are identical " @@ -1823,13 +2201,16 @@ PyArray_AddCastingImplmentation(PyBoundArrayMethodObject *meth) meth->method->name); return -1; } - if ((meth->method->casting & ~_NPY_CAST_IS_VIEW) != NPY_NO_CASTING) { - PyErr_Format(PyExc_TypeError, - "A cast where input and output DType (class) are identical " - "must signal `no-casting`. (method: %s)", - meth->method->name); + if (meth->dtypes[0]->within_dtype_castingimpl != NULL) { + PyErr_Format(PyExc_RuntimeError, + "A cast was already added for %S -> %S. (method: %s)", + meth->dtypes[0], meth->dtypes[1], meth->method->name); return -1; } + Py_INCREF(meth->method); + meth->dtypes[0]->within_dtype_castingimpl = (PyObject *)meth->method; + + return 0; } if (PyDict_Contains(meth->dtypes[0]->castingimpls, (PyObject *)meth->dtypes[1])) { @@ -1860,7 +2241,7 @@ PyArray_AddCastingImplementation_FromSpec(PyArrayMethod_Spec *spec, int private) if (meth == NULL) { return -1; } - int res = PyArray_AddCastingImplmentation(meth); + int res = PyArray_AddCastingImplementation(meth); Py_DECREF(meth); if (res < 0) { return -1; @@ -1906,6 +2287,30 @@ legacy_same_dtype_resolve_descriptors( } +NPY_NO_EXPORT int +legacy_cast_get_strided_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + PyArray_Descr **descrs = context->descriptors; + int out_needs_api = 0; + + *flags = context->method->flags & NPY_METH_RUNTIME_FLAGS; + + if (get_wrapped_legacy_cast_function( + aligned, strides[0], strides[1], descrs[0], descrs[1], + move_references, out_loop, out_transferdata, &out_needs_api, 0) < 0) { + return -1; + } + if (!out_needs_api) { + *flags &= ~NPY_METH_REQUIRES_PYAPI; + } + return 0; +} + + /* * Simple dtype resolver for casting between two different (non-parametric) * (legacy) dtypes. @@ -1945,10 +2350,77 @@ simple_cast_resolve_descriptors( } +NPY_NO_EXPORT int +get_byteswap_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + PyArray_Descr **descrs = context->descriptors; + assert(descrs[0]->kind == descrs[1]->kind); + assert(descrs[0]->elsize == descrs[1]->elsize); + int itemsize = descrs[0]->elsize; + *flags = NPY_METH_NO_FLOATINGPOINT_ERRORS; + *out_transferdata = NULL; + if (descrs[0]->kind == 'c') { + /* + * TODO: we have an issue with complex, since the below loops + * use the itemsize, the complex alignment would be too small. + * Using aligned = 0, might cause slow downs in some cases. + */ + aligned = 0; + } + + if (PyDataType_ISNOTSWAPPED(descrs[0]) == + PyDataType_ISNOTSWAPPED(descrs[1])) { + *out_loop = PyArray_GetStridedCopyFn( + aligned, strides[0], strides[1], itemsize); + } + else if (!PyTypeNum_ISCOMPLEX(descrs[0]->type_num)) { + *out_loop = PyArray_GetStridedCopySwapFn( + aligned, strides[0], strides[1], itemsize); + } + else { + *out_loop = PyArray_GetStridedCopySwapPairFn( + aligned, strides[0], strides[1], itemsize); + } + if (*out_loop == NULL) { + return -1; + } + return 0; +} + + +NPY_NO_EXPORT int +complex_to_noncomplex_get_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + static PyObject *cls = NULL; + int ret; + npy_cache_import("numpy.core", "ComplexWarning", &cls); + if (cls == NULL) { + return -1; + } + ret = PyErr_WarnEx(cls, + "Casting complex values to real discards " + "the imaginary part", 1); + if (ret < 0) { + return -1; + } + return npy_default_get_strided_loop( + context, aligned, move_references, strides, + out_loop, out_transferdata, flags); +} + + static int add_numeric_cast(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) { - PyType_Slot slots[6]; + PyType_Slot slots[7]; PyArray_DTypeMeta *dtypes[2] = {from, to}; PyArrayMethod_Spec spec = { .name = "numeric_cast", @@ -1959,8 +2431,8 @@ add_numeric_cast(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) .dtypes = dtypes, }; - npy_intp from_itemsize = dtypes[0]->singleton->elsize; - npy_intp to_itemsize = dtypes[1]->singleton->elsize; + npy_intp from_itemsize = from->singleton->elsize; + npy_intp to_itemsize = to->singleton->elsize; slots[0].slot = NPY_METH_resolve_descriptors; slots[0].pfunc = &simple_cast_resolve_descriptors; @@ -1977,20 +2449,36 @@ add_numeric_cast(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to) slots[4].slot = NPY_METH_unaligned_contiguous_loop; slots[4].pfunc = PyArray_GetStridedNumericCastFn( 0, from_itemsize, to_itemsize, from->type_num, to->type_num); - slots[5].slot = 0; - slots[5].pfunc = NULL; + if (PyTypeNum_ISCOMPLEX(from->type_num) && + !PyTypeNum_ISCOMPLEX(to->type_num) && + !PyTypeNum_ISBOOL(to->type_num)) { + /* + * The get_loop function must also give a ComplexWarning. We could + * consider moving this warning into the inner-loop at some point + * for simplicity (this requires ensuring it is only emitted once). + */ + slots[5].slot = NPY_METH_get_loop; + slots[5].pfunc = &complex_to_noncomplex_get_loop; + slots[6].slot = 0; + slots[6].pfunc = NULL; + } + else { + /* Use the default get loop function. */ + slots[5].slot = 0; + slots[5].pfunc = NULL; + } assert(slots[1].pfunc && slots[2].pfunc && slots[3].pfunc && slots[4].pfunc); /* Find the correct casting level, and special case no-cast */ if (dtypes[0]->kind == dtypes[1]->kind && from_itemsize == to_itemsize) { - spec.casting = NPY_NO_CASTING; + spec.casting = NPY_EQUIV_CASTING; /* When there is no casting (equivalent C-types) use byteswap loops */ slots[0].slot = NPY_METH_resolve_descriptors; slots[0].pfunc = &legacy_same_dtype_resolve_descriptors; slots[1].slot = NPY_METH_get_loop; - slots[1].pfunc = NULL; + slots[1].pfunc = &get_byteswap_loop; slots[2].slot = 0; slots[2].pfunc = NULL; @@ -2142,7 +2630,6 @@ cast_to_string_resolve_descriptors( dtypes[1]->type_num == NPY_STRING); return NPY_UNSAFE_CASTING; } - assert(self->casting == NPY_SAFE_CASTING); if (loop_descrs[1]->elsize >= size) { return NPY_SAFE_CASTING; @@ -2173,9 +2660,9 @@ add_other_to_and_from_string_cast( */ PyArray_DTypeMeta *dtypes[2] = {other, string}; PyType_Slot slots[] = { - {NPY_METH_get_loop, NULL}, + {NPY_METH_get_loop, &legacy_cast_get_strided_loop}, {NPY_METH_resolve_descriptors, &cast_to_string_resolve_descriptors}, - {0, NULL}}; + {0, NULL}}; PyArrayMethod_Spec spec = { .name = "legacy_cast_to_string", .nin = 1, @@ -2184,9 +2671,9 @@ add_other_to_and_from_string_cast( .dtypes = dtypes, .slots = slots, }; - /* Almost everything can be safely cast to string (except unicode) */ + /* Almost everything can be same-kind cast to string (except unicode) */ if (other->type_num != NPY_UNICODE) { - spec.casting = NPY_SAFE_CASTING; + spec.casting = NPY_SAME_KIND_CASTING; /* same-kind if too short */ } else { spec.casting = NPY_UNSAFE_CASTING; @@ -2233,6 +2720,35 @@ string_to_string_resolve_descriptors( } +NPY_NO_EXPORT int +string_to_string_get_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + int unicode_swap = 0; + PyArray_Descr **descrs = context->descriptors; + + assert(NPY_DTYPE(descrs[0]) == NPY_DTYPE(descrs[1])); + *flags = context->method->flags & NPY_METH_RUNTIME_FLAGS; + if (descrs[0]->type_num == NPY_UNICODE) { + if (PyDataType_ISNOTSWAPPED(descrs[0]) != + PyDataType_ISNOTSWAPPED(descrs[1])) { + unicode_swap = 1; + } + } + + if (PyArray_GetStridedZeroPadCopyFn( + aligned, unicode_swap, strides[0], strides[1], + descrs[0]->elsize, descrs[1]->elsize, + out_loop, out_transferdata) == NPY_FAIL) { + return -1; + } + return 0; +} + + /* * Add string casts. Right now all string casts are just legacy-wrapped ones * (except string<->string and unicode<->unicode), but they do require @@ -2272,12 +2788,12 @@ PyArray_InitializeStringCasts(void) /* string<->string and unicode<->unicode have their own specialized casts */ PyArray_DTypeMeta *dtypes[2]; PyType_Slot slots[] = { - {NPY_METH_get_loop, NULL}, + {NPY_METH_get_loop, &string_to_string_get_loop}, {NPY_METH_resolve_descriptors, &string_to_string_resolve_descriptors}, - {0, NULL}}; + {0, NULL}}; PyArrayMethod_Spec spec = { .name = "string_to_string_cast", - .casting = NPY_NO_CASTING, + .casting = NPY_UNSAFE_CASTING, .nin = 1, .nout = 1, .flags = (NPY_METH_REQUIRES_PYAPI | @@ -2422,6 +2938,57 @@ int give_bad_field_error(PyObject *key) } +static int +nonstructured_to_structured_get_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, + npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + if (context->descriptors[1]->names != NULL) { + int needs_api = 0; + if (get_fields_transfer_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api) == NPY_FAIL) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + else if (context->descriptors[1]->subarray != NULL) { + int needs_api = 0; + if (get_subarray_transfer_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api) == NPY_FAIL) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + else { + /* + * TODO: This could be a simple zero padded cast, adding a decref + * in case of `move_references`. But for now use legacy casts + * (which is the behaviour at least up to 1.20). + */ + int needs_api = 0; + if (get_wrapped_legacy_cast_function( + 1, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api, 1) < 0) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + return 0; +} + + static PyObject * PyArray_GetGenericToVoidCastingImpl(void) { @@ -2439,9 +3006,9 @@ PyArray_GetGenericToVoidCastingImpl(void) method->name = "any_to_void_cast"; method->flags = NPY_METH_SUPPORTS_UNALIGNED | NPY_METH_REQUIRES_PYAPI; - method->casting = NPY_SAFE_CASTING; + method->casting = -1; method->resolve_descriptors = &nonstructured_to_structured_resolve_descriptors; - method->get_strided_loop = NULL; + method->get_strided_loop = &nonstructured_to_structured_get_loop; return (PyObject *)method; } @@ -2511,6 +3078,56 @@ structured_to_nonstructured_resolve_descriptors( } +static int +structured_to_nonstructured_get_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, + npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + if (context->descriptors[0]->names != NULL) { + int needs_api = 0; + if (get_fields_transfer_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api) == NPY_FAIL) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + else if (context->descriptors[0]->subarray != NULL) { + int needs_api = 0; + if (get_subarray_transfer_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api) == NPY_FAIL) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + else { + /* + * In general this is currently defined through legacy behaviour via + * scalars, and should likely just not be allowed. + */ + int needs_api = 0; + if (get_wrapped_legacy_cast_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api, 1) < 0) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + return 0; +} + + static PyObject * PyArray_GetVoidToGenericCastingImpl(void) { @@ -2528,9 +3145,9 @@ PyArray_GetVoidToGenericCastingImpl(void) method->name = "void_to_any_cast"; method->flags = NPY_METH_SUPPORTS_UNALIGNED | NPY_METH_REQUIRES_PYAPI; - method->casting = NPY_UNSAFE_CASTING; + method->casting = -1; method->resolve_descriptors = &structured_to_nonstructured_resolve_descriptors; - method->get_strided_loop = NULL; + method->get_strided_loop = &structured_to_nonstructured_get_loop; return (PyObject *)method; } @@ -2694,6 +3311,56 @@ void_to_void_resolve_descriptors( } +NPY_NO_EXPORT int +void_to_void_get_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, + npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + if (context->descriptors[0]->names != NULL || + context->descriptors[1]->names != NULL) { + int needs_api = 0; + if (get_fields_transfer_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api) == NPY_FAIL) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + else if (context->descriptors[0]->subarray != NULL || + context->descriptors[1]->subarray != NULL) { + int needs_api = 0; + if (get_subarray_transfer_function( + aligned, strides[0], strides[1], + context->descriptors[0], context->descriptors[1], + move_references, out_loop, out_transferdata, + &needs_api) == NPY_FAIL) { + return -1; + } + *flags = needs_api ? NPY_METH_REQUIRES_PYAPI : 0; + } + else { + /* + * This is a string-like copy of the two bytes (zero padding if + * necessary) + */ + if (PyArray_GetStridedZeroPadCopyFn( + 0, 0, strides[0], strides[1], + context->descriptors[0]->elsize, context->descriptors[1]->elsize, + out_loop, out_transferdata) == NPY_FAIL) { + return -1; + } + *flags = 0; + } + return 0; +} + + /* * This initializes the void to void cast. Voids include structured dtypes, * which means that they can cast from and to any other dtype and, in that @@ -2705,12 +3372,12 @@ PyArray_InitializeVoidToVoidCast(void) PyArray_DTypeMeta *Void = PyArray_DTypeFromTypeNum(NPY_VOID); PyArray_DTypeMeta *dtypes[2] = {Void, Void}; PyType_Slot slots[] = { - {NPY_METH_get_loop, NULL}, + {NPY_METH_get_loop, &void_to_void_get_loop}, {NPY_METH_resolve_descriptors, &void_to_void_resolve_descriptors}, - {0, NULL}}; + {0, NULL}}; PyArrayMethod_Spec spec = { .name = "void_to_void_cast", - .casting = NPY_NO_CASTING, + .casting = -1, /* may not cast at all */ .nin = 1, .nout = 1, .flags = NPY_METH_REQUIRES_PYAPI | NPY_METH_SUPPORTS_UNALIGNED, @@ -2792,7 +3459,7 @@ PyArray_GetObjectToGenericCastingImpl(void) method->flags = NPY_METH_SUPPORTS_UNALIGNED | NPY_METH_REQUIRES_PYAPI; method->casting = NPY_UNSAFE_CASTING; method->resolve_descriptors = &object_to_any_resolve_descriptors; - method->get_strided_loop = NULL; + method->get_strided_loop = &object_to_any_get_loop; return (PyObject *)method; } @@ -2848,12 +3515,38 @@ PyArray_GetGenericToObjectCastingImpl(void) method->flags = NPY_METH_SUPPORTS_UNALIGNED | NPY_METH_REQUIRES_PYAPI; method->casting = NPY_SAFE_CASTING; method->resolve_descriptors = &any_to_object_resolve_descriptors; - method->get_strided_loop = NULL; + method->get_strided_loop = &any_to_object_get_loop; return (PyObject *)method; } +/* + * Casts within the object dtype is always just a plain copy/view. + * For that reason, this function might remain unimplemented. + */ +static int +object_to_object_get_loop( + PyArrayMethod_Context *NPY_UNUSED(context), + int NPY_UNUSED(aligned), int move_references, + npy_intp *NPY_UNUSED(strides), + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + *flags = NPY_METH_REQUIRES_PYAPI | NPY_METH_NO_FLOATINGPOINT_ERRORS; + if (move_references) { + *out_loop = &_strided_to_strided_move_references; + *out_transferdata = NULL; + } + else { + *out_loop = &_strided_to_strided_copy_references; + *out_transferdata = NULL; + } + return 0; +} + + static int PyArray_InitializeObjectToObjectCast(void) { @@ -2864,8 +3557,8 @@ PyArray_InitializeObjectToObjectCast(void) PyArray_DTypeMeta *Object = PyArray_DTypeFromTypeNum(NPY_OBJECT); PyArray_DTypeMeta *dtypes[2] = {Object, Object}; PyType_Slot slots[] = { - {NPY_METH_get_loop, NULL}, - {0, NULL}}; + {NPY_METH_get_loop, &object_to_object_get_loop}, + {0, NULL}}; PyArrayMethod_Spec spec = { .name = "object_to_object_cast", .casting = NPY_NO_CASTING | _NPY_CAST_IS_VIEW, diff --git a/numpy/core/src/multiarray/convert_datatype.h b/numpy/core/src/multiarray/convert_datatype.h index cc1930f77db2..ba16d4d1bd5a 100644 --- a/numpy/core/src/multiarray/convert_datatype.h +++ b/numpy/core/src/multiarray/convert_datatype.h @@ -5,6 +5,9 @@ extern NPY_NO_EXPORT npy_intp REQUIRED_STR_LEN[]; +NPY_NO_EXPORT PyObject * +PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to); + NPY_NO_EXPORT PyObject * _get_castingimpl(PyObject *NPY_UNUSED(module), PyObject *args); @@ -17,8 +20,10 @@ PyArray_ObjectType(PyObject *op, int minimum_type); NPY_NO_EXPORT PyArrayObject ** PyArray_ConvertToCommonType(PyObject *op, int *retn); -NPY_NO_EXPORT PyArray_DTypeMeta * -PyArray_CommonDType(PyArray_DTypeMeta *dtype1, PyArray_DTypeMeta *dtype2); +NPY_NO_EXPORT PyArray_Descr * +PyArray_LegacyResultType( + npy_intp narrs, PyArrayObject **arr, + npy_intp ndtypes, PyArray_Descr **dtypes); NPY_NO_EXPORT int PyArray_ValidType(int type); @@ -26,7 +31,7 @@ PyArray_ValidType(int type); NPY_NO_EXPORT int dtype_kind_to_ordering(char kind); -/* Like PyArray_CanCastArrayTo */ +/* Used by PyArray_CanCastArrayTo and in the legacy ufunc type resolution */ NPY_NO_EXPORT npy_bool can_cast_scalar_to(PyArray_Descr *scal_type, char *scal_data, PyArray_Descr *to, NPY_CASTING casting); @@ -49,8 +54,12 @@ npy_set_invalid_cast_error( NPY_NO_EXPORT PyArray_Descr * PyArray_CastDescrToDType(PyArray_Descr *descr, PyArray_DTypeMeta *given_DType); +NPY_NO_EXPORT PyArray_Descr * +PyArray_FindConcatenationDescriptor( + npy_intp n, PyArrayObject **arrays, PyObject *requested_dtype); + NPY_NO_EXPORT int -PyArray_AddCastingImplmentation(PyBoundArrayMethodObject *meth); +PyArray_AddCastingImplementation(PyBoundArrayMethodObject *meth); NPY_NO_EXPORT int PyArray_AddCastingImplementation_FromSpec(PyArrayMethod_Spec *spec, int private); @@ -69,6 +78,13 @@ legacy_same_dtype_resolve_descriptors( PyArray_Descr **given_descrs, PyArray_Descr **loop_descrs); +NPY_NO_EXPORT int +legacy_cast_get_strided_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags); + NPY_NO_EXPORT NPY_CASTING simple_cast_resolve_descriptors( PyArrayMethodObject *self, diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index f6031e370fe7..ef28d7797926 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -552,7 +552,7 @@ PyArray_AssignFromCache_Recursive( else { PyArrayObject *view; view = (PyArrayObject *)array_item_asarray(self, i); - if (view < 0) { + if (view == NULL) { goto fail; } if (PyArray_AssignFromCache_Recursive(view, ndim, cache) < 0) { @@ -665,9 +665,15 @@ PyArray_NewFromDescr_int( int allow_emptystring) { PyArrayObject_fields *fa; - int i; npy_intp nbytes; + if (nd > NPY_MAXDIMS || nd < 0) { + PyErr_Format(PyExc_ValueError, + "number of dimensions must be within [0, %d]", NPY_MAXDIMS); + Py_DECREF(descr); + return NULL; + } + if (descr->subarray) { PyObject *ret; npy_intp newdims[2*NPY_MAXDIMS]; @@ -687,14 +693,6 @@ PyArray_NewFromDescr_int( return ret; } - if ((unsigned int)nd > (unsigned int)NPY_MAXDIMS) { - PyErr_Format(PyExc_ValueError, - "number of dimensions must be within [0, %d]", - NPY_MAXDIMS); - Py_DECREF(descr); - return NULL; - } - /* Check datatype element size */ nbytes = descr->elsize; if (PyDataType_ISUNSIZED(descr)) { @@ -718,39 +716,6 @@ PyArray_NewFromDescr_int( } } - /* Check dimensions and multiply them to nbytes */ - for (i = 0; i < nd; i++) { - npy_intp dim = dims[i]; - - if (dim == 0) { - /* - * Compare to PyArray_OverflowMultiplyList that - * returns 0 in this case. - */ - continue; - } - - if (dim < 0) { - PyErr_SetString(PyExc_ValueError, - "negative dimensions are not allowed"); - Py_DECREF(descr); - return NULL; - } - - /* - * Care needs to be taken to avoid integer overflow when - * multiplying the dimensions together to get the total size of the - * array. - */ - if (npy_mul_with_overflow_intp(&nbytes, nbytes, dim)) { - PyErr_SetString(PyExc_ValueError, - "array is too big; `arr.size * arr.dtype.itemsize` " - "is larger than the maximum possible size."); - Py_DECREF(descr); - return NULL; - } - } - fa = (PyArrayObject_fields *) subtype->tp_alloc(subtype, 0); if (fa == NULL) { Py_DECREF(descr); @@ -786,26 +751,57 @@ PyArray_NewFromDescr_int( goto fail; } fa->strides = fa->dimensions + nd; - if (nd) { - memcpy(fa->dimensions, dims, sizeof(npy_intp)*nd); + + /* Copy dimensions, check them, and find total array size `nbytes` */ + for (int i = 0; i < nd; i++) { + fa->dimensions[i] = dims[i]; + + if (fa->dimensions[i] == 0) { + /* + * Compare to PyArray_OverflowMultiplyList that + * returns 0 in this case. + */ + continue; + } + + if (fa->dimensions[i] < 0) { + PyErr_SetString(PyExc_ValueError, + "negative dimensions are not allowed"); + goto fail; + } + + /* + * Care needs to be taken to avoid integer overflow when multiplying + * the dimensions together to get the total size of the array. + */ + if (npy_mul_with_overflow_intp(&nbytes, nbytes, fa->dimensions[i])) { + PyErr_SetString(PyExc_ValueError, + "array is too big; `arr.size * arr.dtype.itemsize` " + "is larger than the maximum possible size."); + goto fail; + } } - if (strides == NULL) { /* fill it in */ + + /* Fill the strides (or copy them if they were passed in) */ + if (strides == NULL) { + /* fill the strides and set the contiguity flags */ _array_fill_strides(fa->strides, dims, nd, descr->elsize, flags, &(fa->flags)); } else { - /* - * we allow strides even when we create - * the memory, but be careful with this... - */ - if (nd) { - memcpy(fa->strides, strides, sizeof(npy_intp)*nd); + /* User to provided strides (user is responsible for correctness) */ + for (int i = 0; i < nd; i++) { + fa->strides[i] = strides[i]; } + /* Since the strides were passed in must update contiguity */ + PyArray_UpdateFlags((PyArrayObject *)fa, + NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS); } } else { - fa->dimensions = fa->strides = NULL; - fa->flags |= NPY_ARRAY_F_CONTIGUOUS; + fa->dimensions = NULL; + fa->strides = NULL; + fa->flags |= NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS; } if (data == NULL) { @@ -844,12 +840,11 @@ PyArray_NewFromDescr_int( fa->data = data; /* - * always update the flags to get the right CONTIGUOUS, ALIGN properties - * not owned data and input strides may not be aligned and on some - * platforms (debian sparc) malloc does not provide enough alignment for - * long double types + * Always update the aligned flag. Not owned data or input strides may + * not be aligned. Also on some platforms (debian sparc) malloc does not + * provide enough alignment for long double types. */ - PyArray_UpdateFlags((PyArrayObject *)fa, NPY_ARRAY_UPDATE_ALL); + PyArray_UpdateFlags((PyArrayObject *)fa, NPY_ARRAY_ALIGNED); /* Set the base object. It's important to do it here so that * __array_finalize__ below receives it @@ -862,15 +857,20 @@ PyArray_NewFromDescr_int( } /* - * call the __array_finalize__ - * method if a subtype. - * If obj is NULL, then call method with Py_None + * call the __array_finalize__ method if a subtype was requested. + * If obj is NULL use Py_None for the Python callback. */ - if ((subtype != &PyArray_Type)) { - PyObject *res, *func, *args; + if (subtype != &PyArray_Type) { + PyObject *res, *func; func = PyObject_GetAttr((PyObject *)fa, npy_ma_str_array_finalize); - if (func && func != Py_None) { + if (func == NULL) { + goto fail; + } + else if (func == Py_None) { + Py_DECREF(func); + } + else { if (PyCapsule_CheckExact(func)) { /* A C-function is stored here */ PyArray_FinalizeFunc *cfunc; @@ -884,14 +884,10 @@ PyArray_NewFromDescr_int( } } else { - args = PyTuple_New(1); if (obj == NULL) { - obj=Py_None; + obj = Py_None; } - Py_INCREF(obj); - PyTuple_SET_ITEM(args, 0, obj); - res = PyObject_Call(func, args, NULL); - Py_DECREF(args); + res = PyObject_CallFunctionObjArgs(func, obj, NULL); Py_DECREF(func); if (res == NULL) { goto fail; @@ -901,7 +897,6 @@ PyArray_NewFromDescr_int( } } } - else Py_XDECREF(func); } return (PyObject *)fa; @@ -1150,7 +1145,7 @@ _array_from_buffer_3118(PyObject *memoryview) * bpo-32780 * bpo-32782 * - * Note that even if the above are fixed in master, we have to drop the + * Note that even if the above are fixed in main, we have to drop the * early patch versions of python to actually make use of the fixes. */ if (!npy_ctypes_check(Py_TYPE(view->obj))) { @@ -2105,6 +2100,39 @@ _is_default_descr(PyObject *descr, PyObject *typestr) { return PyObject_RichCompareBool(typestr, typestr2, Py_EQ); } + +/* + * A helper function to transition away from ignoring errors during + * special attribute lookups during array coercion. + */ +static NPY_INLINE int +deprecated_lookup_error_clearing(PyTypeObject *type, char *attribute) +{ + PyObject *exc_type, *exc_value, *traceback; + PyErr_Fetch(&exc_type, &exc_value, &traceback); + + /* DEPRECATED 2021-05-12, NumPy 1.21. */ + int res = PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "An exception was ignored while fetching the attribute `%s` from " + "an object of type '%s'. With the exception of `AttributeError` " + "NumPy will always raise this exception in the future. Raise this " + "deprecation warning to see the original exception. " + "(Warning added NumPy 1.21)", attribute, type->tp_name); + + if (res < 0) { + npy_PyErr_ChainExceptionsCause(exc_type, exc_value, traceback); + return -1; + } + else { + /* `PyErr_Fetch` cleared the original error, delete the references */ + Py_DECREF(exc_type); + Py_XDECREF(exc_value); + Py_XDECREF(traceback); + return 0; + } +} + + /*NUMPY_API*/ NPY_NO_EXPORT PyObject * PyArray_FromInterface(PyObject *origin) @@ -2124,7 +2152,15 @@ PyArray_FromInterface(PyObject *origin) if (iface == NULL) { if (PyErr_Occurred()) { - PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + if (PyErr_ExceptionMatches(PyExc_RecursionError) || + PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* RecursionError and MemoryError are considered fatal */ + return NULL; + } + if (deprecated_lookup_error_clearing( + Py_TYPE(origin), "__array_interface__") < 0) { + return NULL; + } } return Py_NotImplemented; } @@ -2392,7 +2428,15 @@ PyArray_FromArrayAttr(PyObject *op, PyArray_Descr *typecode, PyObject *context) array_meth = PyArray_LookupSpecial_OnInstance(op, "__array__"); if (array_meth == NULL) { if (PyErr_Occurred()) { - PyErr_Clear(); /* TODO[gh-14801]: propagate crashes during attribute access? */ + if (PyErr_ExceptionMatches(PyExc_RecursionError) || + PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* RecursionError and MemoryError are considered fatal */ + return NULL; + } + if (deprecated_lookup_error_clearing( + Py_TYPE(op), "__array__") < 0) { + return NULL; + } } return Py_NotImplemented; } @@ -2530,8 +2574,6 @@ PyArray_EnsureAnyArray(PyObject *op) NPY_NO_EXPORT int PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) { - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; NpyIter *dst_iter, *src_iter; NpyIter_IterNextFunc *dst_iternext, *src_iternext; @@ -2540,9 +2582,7 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) npy_intp *dst_countptr, *src_countptr; npy_uint32 baseflags; - char *dst_data, *src_data; npy_intp dst_count, src_count, count; - npy_intp src_itemsize; npy_intp dst_size, src_size; int needs_api; @@ -2614,7 +2654,6 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) /* Since buffering is disabled, we can cache the stride */ src_stride = NpyIter_GetInnerStrideArray(src_iter)[0]; src_countptr = NpyIter_GetInnerLoopSizePtr(src_iter); - src_itemsize = PyArray_DESCR(src)->elsize; if (dst_iternext == NULL || src_iternext == NULL) { NpyIter_Deallocate(dst_iter); @@ -2631,14 +2670,14 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) * we can pass them to this function to take advantage of * contiguous strides, etc. */ + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction( IsUintAligned(src) && IsAligned(src) && IsUintAligned(dst) && IsAligned(dst), src_stride, dst_stride, PyArray_DESCR(src), PyArray_DESCR(dst), 0, - &stransfer, &transferdata, - &needs_api) != NPY_SUCCEED) { + &cast_info, &needs_api) != NPY_SUCCEED) { NpyIter_Deallocate(dst_iter); NpyIter_Deallocate(src_iter); return -1; @@ -2650,15 +2689,15 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) dst_count = *dst_countptr; src_count = *src_countptr; - dst_data = dst_dataptr[0]; - src_data = src_dataptr[0]; + char *args[2] = {src_dataptr[0], dst_dataptr[0]}; + npy_intp strides[2] = {src_stride, dst_stride}; + int res = 0; for(;;) { /* Transfer the biggest amount that fits both */ count = (src_count < dst_count) ? src_count : dst_count; - if (stransfer( - dst_data, dst_stride, src_data, src_stride, - count, src_itemsize, transferdata) < 0) { + if (cast_info.func(&cast_info.context, + args, &count, strides, cast_info.auxdata) < 0) { res = -1; break; } @@ -2670,11 +2709,11 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) break; } dst_count = *dst_countptr; - dst_data = dst_dataptr[0]; + args[1] = dst_dataptr[0]; } else { dst_count -= count; - dst_data += count*dst_stride; + args[1] += count*dst_stride; } /* If we exhausted the src block, refresh it */ @@ -2684,17 +2723,17 @@ PyArray_CopyAsFlat(PyArrayObject *dst, PyArrayObject *src, NPY_ORDER order) break; } src_count = *src_countptr; - src_data = src_dataptr[0]; + args[0] = src_dataptr[0]; } else { src_count -= count; - src_data += count*src_stride; + args[0] += count*src_stride; } } NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); NpyIter_Deallocate(dst_iter); NpyIter_Deallocate(src_iter); if (res > 0) { @@ -2892,7 +2931,7 @@ _arange_safe_ceil_to_intp(double value) "arange: cannot compute length"); return -1; } - if (!(NPY_MIN_INTP <= ivalue && ivalue <= NPY_MAX_INTP)) { + if (!((double)NPY_MIN_INTP <= ivalue && ivalue <= (double)NPY_MAX_INTP)) { PyErr_SetString(PyExc_OverflowError, "arange: overflow while computing length"); return -1; diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 9c1b606bb564..b9d81e8368c5 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -29,6 +29,9 @@ #include "dtypemeta.h" #include "usertypes.h" +#include "dtype_transfer.h" +#include + /* * Computes the python `ret, d = divmod(d, unit)`. * @@ -947,10 +950,6 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, return -1; } - ind = ((int)meta->base - (int)NPY_FR_Y)*2; - totry = _multiples_table[ind]; - baseunit = _multiples_table[ind + 1]; - num = 3; if (meta->base == NPY_FR_W) { num = 4; @@ -959,6 +958,7 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, num = 2; } if (meta->base >= NPY_FR_s) { + /* _multiplies_table only has entries up to NPY_FR_s */ ind = ((int)NPY_FR_s - (int)NPY_FR_Y)*2; totry = _multiples_table[ind]; baseunit = _multiples_table[ind + 1]; @@ -971,6 +971,11 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, num = 0; } } + else { + ind = ((int)meta->base - (int)NPY_FR_Y)*2; + totry = _multiples_table[ind]; + baseunit = _multiples_table[ind + 1]; + } for (i = 0; i < num; i++) { q = totry[i] / den; @@ -3803,6 +3808,72 @@ time_to_time_resolve_descriptors( } +static int +time_to_time_get_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + int requires_wrap = 0; + int inner_aligned = aligned; + PyArray_Descr **descrs = context->descriptors; + *flags = NPY_METH_NO_FLOATINGPOINT_ERRORS; + + PyArray_DatetimeMetaData *meta1 = get_datetime_metadata_from_dtype(descrs[0]); + PyArray_DatetimeMetaData *meta2 = get_datetime_metadata_from_dtype(descrs[1]); + + if (meta1->base == meta2->base && meta1->num == meta2->num) { + /* + * If the metadata matches, use the low-level copy or copy-swap + * functions. (If they do not match, but swapping is necessary this + * path is hit recursively.) + */ + if (PyDataType_ISNOTSWAPPED(descrs[0]) == + PyDataType_ISNOTSWAPPED(descrs[1])) { + *out_loop = PyArray_GetStridedCopyFn( + aligned, strides[0], strides[1], NPY_SIZEOF_DATETIME); + } + else { + *out_loop = PyArray_GetStridedCopySwapFn( + aligned, strides[0], strides[1], NPY_SIZEOF_DATETIME); + } + return 0; + } + + if (!PyDataType_ISNOTSWAPPED(descrs[0]) || + !PyDataType_ISNOTSWAPPED(descrs[1])) { + inner_aligned = 1; + requires_wrap = 1; + } + if (get_nbo_cast_datetime_transfer_function( + inner_aligned, descrs[0], descrs[1], + out_loop, out_transferdata) == NPY_FAIL) { + return -1; + } + + if (!requires_wrap) { + return 0; + } + + PyArray_Descr *src_wrapped_dtype = ensure_dtype_nbo(descrs[0]); + PyArray_Descr *dst_wrapped_dtype = ensure_dtype_nbo(descrs[1]); + + int needs_api = 0; + int res = wrap_aligned_transferfunction( + aligned, 0, + strides[0], strides[1], + descrs[0], descrs[1], + src_wrapped_dtype, dst_wrapped_dtype, + out_loop, out_transferdata, &needs_api); + Py_DECREF(src_wrapped_dtype); + Py_DECREF(dst_wrapped_dtype); + + assert(needs_api == 0); + return res; +} + + /* Handles datetime<->timedelta type resolution (both directions) */ static NPY_CASTING datetime_to_timedelta_resolve_descriptors( @@ -3844,9 +3915,7 @@ time_to_string_resolve_descriptors( PyArray_Descr **given_descrs, PyArray_Descr **loop_descrs) { - Py_INCREF(given_descrs[0]); - loop_descrs[0] = given_descrs[0]; - if (given_descrs[1] != NULL) { + if (given_descrs[1] != NULL && dtypes[0]->type_num == NPY_DATETIME) { /* * At the time of writing, NumPy does not check the length here, * but will error if filling fails. @@ -3863,6 +3932,10 @@ time_to_string_resolve_descriptors( size = get_datetime_iso_8601_strlen(0, meta->base); } else { + /* + * This is arguably missing space for the unit, e.g. for: + * `np.timedelta64(1231234342124, 'ms')` + */ size = 21; } if (dtypes[1]->type_num == NPY_UNICODE) { @@ -3870,15 +3943,49 @@ time_to_string_resolve_descriptors( } loop_descrs[1] = PyArray_DescrNewFromType(dtypes[1]->type_num); if (loop_descrs[1] == NULL) { - Py_DECREF(loop_descrs[0]); return -1; } loop_descrs[1]->elsize = size; } - assert(self->casting == NPY_UNSAFE_CASTING); + + loop_descrs[0] = ensure_dtype_nbo(given_descrs[0]); + if (loop_descrs[0] == NULL) { + Py_DECREF(loop_descrs[1]); + return -1; + } + return NPY_UNSAFE_CASTING; } +static int +datetime_to_string_get_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + PyArray_Descr **descrs = context->descriptors; + *flags = context->method->flags & NPY_METH_RUNTIME_FLAGS; + + if (descrs[1]->type_num == NPY_STRING) { + if (get_nbo_datetime_to_string_transfer_function( + descrs[0], descrs[1], + out_loop, out_transferdata) == NPY_FAIL) { + return -1; + } + } + else { + assert(descrs[1]->type_num == NPY_UNICODE); + int out_needs_api; + if (get_datetime_to_unicode_transfer_function( + aligned, strides[0], strides[1], descrs[0], descrs[1], + out_loop, out_transferdata, &out_needs_api) == NPY_FAIL) { + return -1; + } + } + return 0; +} + static NPY_CASTING string_to_datetime_cast_resolve_descriptors( @@ -3887,27 +3994,58 @@ string_to_datetime_cast_resolve_descriptors( PyArray_Descr *given_descrs[2], PyArray_Descr *loop_descrs[2]) { - /* We currently support byte-swapping, so any (unicode) string is OK */ - Py_INCREF(given_descrs[0]); - loop_descrs[0] = given_descrs[0]; - if (given_descrs[1] == NULL) { /* NOTE: This doesn't actually work, and will error during the cast */ loop_descrs[1] = dtypes[1]->default_descr(dtypes[1]); if (loop_descrs[1] == NULL) { - Py_DECREF(loop_descrs[0]); return -1; } } else { - Py_INCREF(given_descrs[1]); - loop_descrs[1] = given_descrs[1]; + loop_descrs[1] = ensure_dtype_nbo(given_descrs[1]); + if (loop_descrs[1] == NULL) { + return -1; + } } + /* We currently support byte-swapping, so any (unicode) string is OK */ + Py_INCREF(given_descrs[0]); + loop_descrs[0] = given_descrs[0]; + return NPY_UNSAFE_CASTING; } +static int +string_to_datetime_cast_get_loop( + PyArrayMethod_Context *context, + int aligned, int NPY_UNUSED(move_references), npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + PyArray_Descr **descrs = context->descriptors; + *flags = context->method->flags & NPY_METH_RUNTIME_FLAGS; + + if (descrs[0]->type_num == NPY_STRING) { + if (get_nbo_string_to_datetime_transfer_function( + descrs[0], descrs[1], out_loop, out_transferdata) == NPY_FAIL) { + return -1; + } + } + else { + assert(descrs[0]->type_num == NPY_UNICODE); + int out_needs_api; + if (get_unicode_to_datetime_transfer_function( + aligned, strides[0], strides[1], descrs[0], descrs[1], + out_loop, out_transferdata, &out_needs_api) == NPY_FAIL) { + return -1; + } + } + return 0; +} + + + /* * This registers the castingimpl for all datetime related casts. */ @@ -3922,7 +4060,7 @@ PyArray_InitializeDatetimeCasts() .name = "datetime_casts", .nin = 1, .nout = 1, - .casting = NPY_NO_CASTING, + .casting = NPY_UNSAFE_CASTING, .flags = NPY_METH_SUPPORTS_UNALIGNED, .slots = slots, .dtypes = dtypes, @@ -3930,7 +4068,7 @@ PyArray_InitializeDatetimeCasts() slots[0].slot = NPY_METH_resolve_descriptors; slots[0].pfunc = &time_to_time_resolve_descriptors; slots[1].slot = NPY_METH_get_loop; - slots[1].pfunc = NULL; + slots[1].pfunc = &time_to_time_get_loop; slots[2].slot = 0; slots[2].pfunc = NULL; @@ -3955,10 +4093,12 @@ PyArray_InitializeDatetimeCasts() * Casting between timedelta and datetime uses legacy casting loops, but * custom dtype resolution (to handle copying of the time unit). */ + spec.flags = NPY_METH_REQUIRES_PYAPI; + slots[0].slot = NPY_METH_resolve_descriptors; slots[0].pfunc = &datetime_to_timedelta_resolve_descriptors; slots[1].slot = NPY_METH_get_loop; - slots[1].pfunc = NULL; + slots[1].pfunc = &legacy_cast_get_strided_loop; slots[2].slot = 0; slots[2].pfunc = NULL; @@ -4024,26 +4164,36 @@ PyArray_InitializeDatetimeCasts() /* * Casts can error and need API (unicodes needs it for string->unicode). * Unicode handling is currently implemented via a legacy cast. + * Datetime->string has its own fast cast while timedelta->string uses + * the legacy fallback. */ - spec.flags = NPY_METH_SUPPORTS_UNALIGNED | NPY_METH_REQUIRES_PYAPI; - slots[0].slot = NPY_METH_resolve_descriptors; slots[0].pfunc = &time_to_string_resolve_descriptors; + /* Strided loop differs for the two */ slots[1].slot = NPY_METH_get_loop; - slots[1].pfunc = NULL; slots[2].slot = 0; slots[2].pfunc = NULL; + dtypes[0] = datetime; for (int num = NPY_DATETIME; num <= NPY_TIMEDELTA; num++) { + if (num == NPY_DATETIME) { + dtypes[0] = datetime; + spec.flags = NPY_METH_SUPPORTS_UNALIGNED | NPY_METH_REQUIRES_PYAPI; + slots[1].pfunc = &datetime_to_string_get_loop; + } + else { + dtypes[0] = timedelta; + spec.flags = NPY_METH_REQUIRES_PYAPI; + slots[1].pfunc = &legacy_cast_get_strided_loop; + } + for (int str = NPY_STRING; str <= NPY_UNICODE; str++) { - dtypes[0] = PyArray_DTypeFromTypeNum(num); dtypes[1] = PyArray_DTypeFromTypeNum(str); int res = PyArray_AddCastingImplementation_FromSpec(&spec, 1); - Py_SETREF(dtypes[0], NULL); Py_SETREF(dtypes[1], NULL); if (res < 0) { - return -1; + goto fail; } } } @@ -4070,7 +4220,7 @@ PyArray_InitializeDatetimeCasts() slots[0].slot = NPY_METH_resolve_descriptors; slots[0].pfunc = &string_to_datetime_cast_resolve_descriptors; slots[1].slot = NPY_METH_get_loop; - slots[1].pfunc = NULL; + slots[1].pfunc = &string_to_datetime_cast_get_loop; slots[2].slot = 0; slots[2].pfunc = NULL; diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 2cf1575515bc..f0564146d9e6 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -934,8 +934,8 @@ NPY_NO_EXPORT PyObject * array_busday_offset(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { - char *kwlist[] = {"dates", "offsets", "roll", - "weekmask", "holidays", "busdaycal", "out", NULL}; + static char *kwlist[] = {"dates", "offsets", "roll", + "weekmask", "holidays", "busdaycal", "out", NULL}; PyObject *dates_in = NULL, *offsets_in = NULL, *out_in = NULL; @@ -1065,8 +1065,8 @@ NPY_NO_EXPORT PyObject * array_busday_count(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { - char *kwlist[] = {"begindates", "enddates", - "weekmask", "holidays", "busdaycal", "out", NULL}; + static char *kwlist[] = {"begindates", "enddates", + "weekmask", "holidays", "busdaycal", "out", NULL}; PyObject *dates_begin_in = NULL, *dates_end_in = NULL, *out_in = NULL; @@ -1210,8 +1210,8 @@ NPY_NO_EXPORT PyObject * array_is_busday(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { - char *kwlist[] = {"dates", - "weekmask", "holidays", "busdaycal", "out", NULL}; + static char *kwlist[] = {"dates", + "weekmask", "holidays", "busdaycal", "out", NULL}; PyObject *dates_in = NULL, *out_in = NULL; diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index a8d575248bb7..b8b477e5d70f 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -108,6 +108,11 @@ _try_convert_from_dtype_attr(PyObject *obj) goto fail; } + if (PyArray_DescrCheck(dtypedescr)) { + /* The dtype attribute is already a valid descriptor */ + return (PyArray_Descr *)dtypedescr; + } + if (Py_EnterRecursiveCall( " while trying to convert the given data type from its " "`.dtype` attribute.") != 0) { @@ -122,6 +127,15 @@ _try_convert_from_dtype_attr(PyObject *obj) goto fail; } + /* Deprecated 2021-01-05, NumPy 1.21 */ + if (DEPRECATE("in the future the `.dtype` attribute of a given data" + "type object must be a valid dtype instance. " + "`data_type.dtype` may need to be coerced using " + "`np.dtype(data_type.dtype)`. (Deprecated NumPy 1.20)") < 0) { + Py_DECREF(newdescr); + return NULL; + } + return newdescr; fail: @@ -3214,7 +3228,9 @@ arraydescr_richcompare(PyArray_Descr *self, PyObject *other, int cmp_op) { PyArray_Descr *new = _convert_from_any(other, 0); if (new == NULL) { - return NULL; + /* Cannot convert `other` to dtype */ + PyErr_Clear(); + Py_RETURN_NOTIMPLEMENTED; } npy_bool ret; diff --git a/numpy/core/src/multiarray/dragon4.c b/numpy/core/src/multiarray/dragon4.c index a7b252a77e7e..1d8c275700a2 100644 --- a/numpy/core/src/multiarray/dragon4.c +++ b/numpy/core/src/multiarray/dragon4.c @@ -1130,8 +1130,9 @@ BigInt_ShiftLeft(BigInt *result, npy_uint32 shift) * * exponent - value exponent in base 2 * * mantissaBit - index of the highest set mantissa bit * * hasUnequalMargins - is the high margin twice as large as the low margin - * * cutoffMode - how to interpret cutoffNumber: fractional or total digits? - * * cutoffNumber - cut off printing after this many digits. -1 for no cutoff + * * cutoffMode - how to interpret cutoff_*: fractional or total digits? + * * cutoff_max - cut off printing after this many digits. -1 for no cutoff + * * cutoff_min - print at least this many digits. -1 for no cutoff * * pOutBuffer - buffer to output into * * bufferSize - maximum characters that can be printed to pOutBuffer * * pOutExponent - the base 10 exponent of the first digit @@ -1142,7 +1143,7 @@ static npy_uint32 Dragon4(BigInt *bigints, const npy_int32 exponent, const npy_uint32 mantissaBit, const npy_bool hasUnequalMargins, const DigitMode digitMode, const CutoffMode cutoffMode, - npy_int32 cutoffNumber, char *pOutBuffer, + npy_int32 cutoff_max, npy_int32 cutoff_min, char *pOutBuffer, npy_uint32 bufferSize, npy_int32 *pOutExponent) { char *curDigit = pOutBuffer; @@ -1169,7 +1170,8 @@ Dragon4(BigInt *bigints, const npy_int32 exponent, BigInt *temp2 = &bigints[6]; const npy_float64 log10_2 = 0.30102999566398119521373889472449; - npy_int32 digitExponent, cutoffExponent, hiBlock; + npy_int32 digitExponent, hiBlock; + npy_int32 cutoff_max_Exponent, cutoff_min_Exponent; npy_uint32 outputDigit; /* current digit being output */ npy_uint32 outputLen; npy_bool isEven = BigInt_IsEven(mantissa); @@ -1294,9 +1296,9 @@ Dragon4(BigInt *bigints, const npy_int32 exponent, * increases the number. This will either correct digitExponent to an * accurate value or it will clamp it above the accurate value. */ - if (cutoffNumber >= 0 && cutoffMode == CutoffMode_FractionLength && - digitExponent <= -cutoffNumber) { - digitExponent = -cutoffNumber + 1; + if (cutoff_max >= 0 && cutoffMode == CutoffMode_FractionLength && + digitExponent <= -cutoff_max) { + digitExponent = -cutoff_max + 1; } @@ -1347,26 +1349,44 @@ Dragon4(BigInt *bigints, const npy_int32 exponent, } /* - * Compute the cutoff exponent (the exponent of the final digit to print). - * Default to the maximum size of the output buffer. + * Compute the cutoff_max exponent (the exponent of the final digit to + * print). Default to the maximum size of the output buffer. */ - cutoffExponent = digitExponent - bufferSize; - if (cutoffNumber >= 0) { + cutoff_max_Exponent = digitExponent - bufferSize; + if (cutoff_max >= 0) { npy_int32 desiredCutoffExponent; if (cutoffMode == CutoffMode_TotalLength) { - desiredCutoffExponent = digitExponent - cutoffNumber; - if (desiredCutoffExponent > cutoffExponent) { - cutoffExponent = desiredCutoffExponent; + desiredCutoffExponent = digitExponent - cutoff_max; + if (desiredCutoffExponent > cutoff_max_Exponent) { + cutoff_max_Exponent = desiredCutoffExponent; } } - /* Otherwise it's CutoffMode_FractionLength. Print cutoffNumber digits + /* Otherwise it's CutoffMode_FractionLength. Print cutoff_max digits * past the decimal point or until we reach the buffer size */ else { - desiredCutoffExponent = -cutoffNumber; - if (desiredCutoffExponent > cutoffExponent) { - cutoffExponent = desiredCutoffExponent; + desiredCutoffExponent = -cutoff_max; + if (desiredCutoffExponent > cutoff_max_Exponent) { + cutoff_max_Exponent = desiredCutoffExponent; + } + } + } + /* Also compute the cutoff_min exponent. */ + cutoff_min_Exponent = digitExponent; + if (cutoff_min >= 0) { + npy_int32 desiredCutoffExponent; + + if (cutoffMode == CutoffMode_TotalLength) { + desiredCutoffExponent = digitExponent - cutoff_min; + if (desiredCutoffExponent < cutoff_min_Exponent) { + cutoff_min_Exponent = desiredCutoffExponent; + } + } + else { + desiredCutoffExponent = -cutoff_min; + if (desiredCutoffExponent < cutoff_min_Exponent) { + cutoff_min_Exponent = desiredCutoffExponent; } } } @@ -1432,14 +1452,17 @@ Dragon4(BigInt *bigints, const npy_int32 exponent, /* * stop looping if we are far enough away from our neighboring - * values or if we have reached the cutoff digit + * values (and we have printed at least the requested minimum + * digits) or if we have reached the cutoff digit */ cmp = BigInt_Compare(scaledValue, scaledMarginLow); low = isEven ? (cmp <= 0) : (cmp < 0); cmp = BigInt_Compare(scaledValueHigh, scale); high = isEven ? (cmp >= 0) : (cmp > 0); - if (low | high | (digitExponent == cutoffExponent)) + if (((low | high) & (digitExponent <= cutoff_min_Exponent)) | + (digitExponent == cutoff_max_Exponent)) { break; + } /* store the output digit */ *curDigit = (char)('0' + outputDigit); @@ -1471,7 +1494,7 @@ Dragon4(BigInt *bigints, const npy_int32 exponent, DEBUG_ASSERT(outputDigit < 10); if ((scaledValue->length == 0) | - (digitExponent == cutoffExponent)) { + (digitExponent == cutoff_max_Exponent)) { break; } @@ -1589,6 +1612,7 @@ typedef struct Dragon4_Options { DigitMode digit_mode; CutoffMode cutoff_mode; npy_int32 precision; + npy_int32 min_digits; npy_bool sign; TrimMode trim_mode; npy_int32 digits_left; @@ -1617,11 +1641,12 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, BigInt *mantissa, npy_int32 exponent, char signbit, npy_uint32 mantissaBit, npy_bool hasUnequalMargins, DigitMode digit_mode, CutoffMode cutoff_mode, npy_int32 precision, - TrimMode trim_mode, npy_int32 digits_left, - npy_int32 digits_right) + npy_int32 min_digits, TrimMode trim_mode, + npy_int32 digits_left, npy_int32 digits_right) { npy_int32 printExponent; npy_int32 numDigits, numWholeDigits=0, has_sign=0; + npy_int32 add_digits; npy_int32 maxPrintLen = (npy_int32)bufferSize - 1, pos = 0; @@ -1644,8 +1669,9 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, BigInt *mantissa, } numDigits = Dragon4(mantissa, exponent, mantissaBit, hasUnequalMargins, - digit_mode, cutoff_mode, precision, buffer + has_sign, - maxPrintLen - has_sign, &printExponent); + digit_mode, cutoff_mode, precision, min_digits, + buffer + has_sign, maxPrintLen - has_sign, + &printExponent); DEBUG_ASSERT(numDigits > 0); DEBUG_ASSERT(numDigits <= bufferSize); @@ -1744,9 +1770,10 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, BigInt *mantissa, buffer[pos++] = '.'; } - desiredFractionalDigits = precision; - if (cutoff_mode == CutoffMode_TotalLength && precision >= 0) { - desiredFractionalDigits = precision - numWholeDigits; + add_digits = digit_mode == DigitMode_Unique ? min_digits : precision; + desiredFractionalDigits = add_digits < 0 ? 0 : add_digits; + if (cutoff_mode == CutoffMode_TotalLength) { + desiredFractionalDigits = add_digits - numWholeDigits; } if (trim_mode == TrimMode_LeaveOneZero) { @@ -1757,10 +1784,9 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, BigInt *mantissa, } } else if (trim_mode == TrimMode_None && - digit_mode != DigitMode_Unique && desiredFractionalDigits > numFractionDigits && pos < maxPrintLen) { - /* add trailing zeros up to precision length */ + /* add trailing zeros up to add_digits length */ /* compute the number of trailing zeros needed */ npy_int32 count = desiredFractionalDigits - numFractionDigits; if (pos + count > maxPrintLen) { @@ -1778,7 +1804,7 @@ FormatPositional(char *buffer, npy_uint32 bufferSize, BigInt *mantissa, * when rounding, we may still end up with trailing zeros. Remove them * depending on trim settings. */ - if (precision >= 0 && trim_mode != TrimMode_None && numFractionDigits > 0) { + if (trim_mode != TrimMode_None && numFractionDigits > 0) { while (buffer[pos-1] == '0') { pos--; numFractionDigits--; @@ -1852,7 +1878,7 @@ static npy_uint32 FormatScientific (char *buffer, npy_uint32 bufferSize, BigInt *mantissa, npy_int32 exponent, char signbit, npy_uint32 mantissaBit, npy_bool hasUnequalMargins, DigitMode digit_mode, - npy_int32 precision, TrimMode trim_mode, + npy_int32 precision, npy_int32 min_digits, TrimMode trim_mode, npy_int32 digits_left, npy_int32 exp_digits) { npy_int32 printExponent; @@ -1860,12 +1886,12 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, BigInt *mantissa, char *pCurOut; npy_int32 numFractionDigits; npy_int32 leftchars; + npy_int32 add_digits; if (digit_mode != DigitMode_Unique) { DEBUG_ASSERT(precision >= 0); } - DEBUG_ASSERT(bufferSize > 0); pCurOut = buffer; @@ -1893,7 +1919,9 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, BigInt *mantissa, } numDigits = Dragon4(mantissa, exponent, mantissaBit, hasUnequalMargins, - digit_mode, CutoffMode_TotalLength, precision + 1, + digit_mode, CutoffMode_TotalLength, + precision < 0 ? -1 : precision + 1, + min_digits < 0 ? -1 : min_digits + 1, pCurOut, bufferSize, &printExponent); DEBUG_ASSERT(numDigits > 0); @@ -1928,6 +1956,8 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, BigInt *mantissa, --bufferSize; } + add_digits = digit_mode == DigitMode_Unique ? min_digits : precision; + add_digits = add_digits < 0 ? 0 : add_digits; if (trim_mode == TrimMode_LeaveOneZero) { /* if we didn't print any fractional digits, add the 0 */ if (numFractionDigits == 0 && bufferSize > 1) { @@ -1937,13 +1967,12 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, BigInt *mantissa, ++numFractionDigits; } } - else if (trim_mode == TrimMode_None && - digit_mode != DigitMode_Unique) { - /* add trailing zeros up to precision length */ - if (precision > (npy_int32)numFractionDigits) { + else if (trim_mode == TrimMode_None) { + /* add trailing zeros up to add_digits length */ + if (add_digits > (npy_int32)numFractionDigits) { char *pEnd; /* compute the number of trailing zeros needed */ - npy_int32 numZeros = (precision - numFractionDigits); + npy_int32 numZeros = (add_digits - numFractionDigits); if (numZeros > (npy_int32)bufferSize - 1) { numZeros = (npy_int32)bufferSize - 1; @@ -1961,7 +1990,7 @@ FormatScientific (char *buffer, npy_uint32 bufferSize, BigInt *mantissa, * when rounding, we may still end up with trailing zeros. Remove them * depending on trim settings. */ - if (precision >= 0 && trim_mode != TrimMode_None && numFractionDigits > 0) { + if (trim_mode != TrimMode_None && numFractionDigits > 0) { --pCurOut; while (*pCurOut == '0') { --pCurOut; @@ -2153,14 +2182,14 @@ Format_floatbits(char *buffer, npy_uint32 bufferSize, BigInt *mantissa, return FormatScientific(buffer, bufferSize, mantissa, exponent, signbit, mantissaBit, hasUnequalMargins, opt->digit_mode, opt->precision, - opt->trim_mode, opt->digits_left, - opt->exp_digits); + opt->min_digits, opt->trim_mode, + opt->digits_left, opt->exp_digits); } else { return FormatPositional(buffer, bufferSize, mantissa, exponent, signbit, mantissaBit, hasUnequalMargins, opt->digit_mode, opt->cutoff_mode, - opt->precision, opt->trim_mode, + opt->precision, opt->min_digits, opt->trim_mode, opt->digits_left, opt->digits_right); } } @@ -3100,7 +3129,7 @@ Dragon4_Positional_##Type##_opt(npy_type *val, Dragon4_Options *opt)\ \ PyObject *\ Dragon4_Positional_##Type(npy_type *val, DigitMode digit_mode,\ - CutoffMode cutoff_mode, int precision,\ + CutoffMode cutoff_mode, int precision, int min_digits, \ int sign, TrimMode trim, int pad_left, int pad_right)\ {\ Dragon4_Options opt;\ @@ -3109,6 +3138,7 @@ Dragon4_Positional_##Type(npy_type *val, DigitMode digit_mode,\ opt.digit_mode = digit_mode;\ opt.cutoff_mode = cutoff_mode;\ opt.precision = precision;\ + opt.min_digits = min_digits;\ opt.sign = sign;\ opt.trim_mode = trim;\ opt.digits_left = pad_left;\ @@ -3136,7 +3166,8 @@ Dragon4_Scientific_##Type##_opt(npy_type *val, Dragon4_Options *opt)\ }\ PyObject *\ Dragon4_Scientific_##Type(npy_type *val, DigitMode digit_mode, int precision,\ - int sign, TrimMode trim, int pad_left, int exp_digits)\ + int min_digits, int sign, TrimMode trim, int pad_left, \ + int exp_digits)\ {\ Dragon4_Options opt;\ \ @@ -3144,6 +3175,7 @@ Dragon4_Scientific_##Type(npy_type *val, DigitMode digit_mode, int precision,\ opt.digit_mode = digit_mode;\ opt.cutoff_mode = CutoffMode_TotalLength;\ opt.precision = precision;\ + opt.min_digits = min_digits;\ opt.sign = sign;\ opt.trim_mode = trim;\ opt.digits_left = pad_left;\ @@ -3166,8 +3198,8 @@ make_dragon4_typefuncs(LongDouble, npy_longdouble, NPY_LONGDOUBLE_BINFMT_NAME) PyObject * Dragon4_Positional(PyObject *obj, DigitMode digit_mode, CutoffMode cutoff_mode, - int precision, int sign, TrimMode trim, int pad_left, - int pad_right) + int precision, int min_digits, int sign, TrimMode trim, + int pad_left, int pad_right) { npy_double val; Dragon4_Options opt; @@ -3176,6 +3208,7 @@ Dragon4_Positional(PyObject *obj, DigitMode digit_mode, CutoffMode cutoff_mode, opt.digit_mode = digit_mode; opt.cutoff_mode = cutoff_mode; opt.precision = precision; + opt.min_digits = min_digits; opt.sign = sign; opt.trim_mode = trim; opt.digits_left = pad_left; @@ -3208,7 +3241,8 @@ Dragon4_Positional(PyObject *obj, DigitMode digit_mode, CutoffMode cutoff_mode, PyObject * Dragon4_Scientific(PyObject *obj, DigitMode digit_mode, int precision, - int sign, TrimMode trim, int pad_left, int exp_digits) + int min_digits, int sign, TrimMode trim, int pad_left, + int exp_digits) { npy_double val; Dragon4_Options opt; @@ -3217,6 +3251,7 @@ Dragon4_Scientific(PyObject *obj, DigitMode digit_mode, int precision, opt.digit_mode = digit_mode; opt.cutoff_mode = CutoffMode_TotalLength; opt.precision = precision; + opt.min_digits = min_digits; opt.sign = sign; opt.trim_mode = trim; opt.digits_left = pad_left; diff --git a/numpy/core/src/multiarray/dragon4.h b/numpy/core/src/multiarray/dragon4.h index 3a99bde6c53b..4b76bf9e582c 100644 --- a/numpy/core/src/multiarray/dragon4.h +++ b/numpy/core/src/multiarray/dragon4.h @@ -112,12 +112,12 @@ typedef enum TrimMode PyObject *\ Dragon4_Positional_##Type(npy_type *val, DigitMode digit_mode,\ CutoffMode cutoff_mode, int precision,\ - int sign, TrimMode trim, int pad_left,\ - int pad_right);\ + int min_digits, int sign, TrimMode trim, \ + int pad_left, int pad_right);\ PyObject *\ Dragon4_Scientific_##Type(npy_type *val, DigitMode digit_mode,\ - int precision, int sign, TrimMode trim,\ - int pad_left, int exp_digits); + int precision, int min_digits, int sign, \ + TrimMode trim, int pad_left, int exp_digits); make_dragon4_typedecl(Half, npy_half) make_dragon4_typedecl(Float, npy_float) @@ -128,12 +128,13 @@ make_dragon4_typedecl(LongDouble, npy_longdouble) PyObject * Dragon4_Positional(PyObject *obj, DigitMode digit_mode, CutoffMode cutoff_mode, - int precision, int sign, TrimMode trim, int pad_left, - int pad_right); + int precision, int min_digits, int sign, TrimMode trim, + int pad_left, int pad_right); PyObject * Dragon4_Scientific(PyObject *obj, DigitMode digit_mode, int precision, - int sign, TrimMode trim, int pad_left, int exp_digits); + int min_digits, int sign, TrimMode trim, int pad_left, + int exp_digits); #endif diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 630bd76f3955..aa8cc84ffa6f 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -18,6 +18,7 @@ #define _MULTIARRAYMODULE #include +#include "lowlevel_strided_loops.h" #include "npy_pycompat.h" #include "convert_datatype.h" @@ -28,8 +29,11 @@ #include "array_assign.h" #include "shape.h" -#include "lowlevel_strided_loops.h" +#include "dtype_transfer.h" #include "alloc.h" +#include "dtypemeta.h" +#include "array_method.h" +#include "array_coercion.h" #define NPY_LOWLEVEL_BUFFER_BLOCKSIZE 128 @@ -71,46 +75,26 @@ _safe_print(PyObject *obj) * Returns NPY_SUCCEED or NPY_FAIL. */ static int -get_decsrcref_transfer_function(int aligned, +get_decref_transfer_function(int aligned, npy_intp src_stride, PyArray_Descr *src_dtype, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api); - -/* - * Returns a transfer function which zeros out the dest values. - * - * Returns NPY_SUCCEED or NPY_FAIL. - */ -static int -get_setdstzero_transfer_function(int aligned, - npy_intp dst_stride, - PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, + NPY_cast_info *cast_info, int *out_needs_api); -/* - * Returns a transfer function which sets a boolean type to ones. - * - * Returns NPY_SUCCEED or NPY_FAIL. - */ -NPY_NO_EXPORT int -get_bool_setdstone_transfer_function(npy_intp dst_stride, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *NPY_UNUSED(out_needs_api)); /*************************** COPY REFERENCES *******************************/ /* Moves references from src to dst */ -static int -_strided_to_strided_move_references(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +NPY_NO_EXPORT int +_strided_to_strided_move_references( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + PyObject *src_ref = NULL, *dst_ref = NULL; while (N > 0) { memcpy(&src_ref, src, sizeof(src_ref)); @@ -134,12 +118,16 @@ _strided_to_strided_move_references(char *dst, npy_intp dst_stride, } /* Copies references from src to dst */ -static int -_strided_to_strided_copy_references(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +NPY_NO_EXPORT int +_strided_to_strided_copy_references( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + PyObject *src_ref = NULL, *dst_ref = NULL; while (N > 0) { memcpy(&src_ref, src, sizeof(src_ref)); @@ -161,42 +149,242 @@ _strided_to_strided_copy_references(char *dst, npy_intp dst_stride, return 0; } +/************************** ANY TO OBJECT *********************************/ + +typedef struct { + NpyAuxData base; + PyArray_GetItemFunc *getitem; + PyArrayObject_fields arr_fields; + NPY_cast_info decref_src; +} _any_to_object_auxdata; + + +static void +_any_to_object_auxdata_free(NpyAuxData *auxdata) +{ + _any_to_object_auxdata *data = (_any_to_object_auxdata *)auxdata; + + Py_DECREF(data->arr_fields.descr); + NPY_cast_info_xfree(&data->decref_src); + PyMem_Free(data); +} + + +static NpyAuxData * +_any_to_object_auxdata_clone(NpyAuxData *auxdata) +{ + _any_to_object_auxdata *data = (_any_to_object_auxdata *)auxdata; + + _any_to_object_auxdata *res = PyMem_Malloc(sizeof(_any_to_object_auxdata)); + + res->base = data->base; + res->getitem = data->getitem; + res->arr_fields = data->arr_fields; + Py_INCREF(res->arr_fields.descr); + + if (data->decref_src.func != NULL) { + if (NPY_cast_info_copy(&res->decref_src, &data->decref_src) < 0) { + NPY_AUXDATA_FREE((NpyAuxData *)res); + return NULL; + } + } + else { + res->decref_src.func = NULL; + } + return (NpyAuxData *)res; +} + + +static int +_strided_to_strided_any_to_object( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) +{ + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _any_to_object_auxdata *data = (_any_to_object_auxdata *)auxdata; + + PyObject *dst_ref = NULL; + char *orig_src = src; + while (N > 0) { + memcpy(&dst_ref, dst, sizeof(dst_ref)); + Py_XDECREF(dst_ref); + dst_ref = data->getitem(src, &data->arr_fields); + memcpy(dst, &dst_ref, sizeof(PyObject *)); + + if (dst_ref == NULL) { + return -1; + } + src += src_stride; + dst += dst_stride; + --N; + } + if (data->decref_src.func != NULL) { + /* If necessary, clear the input buffer (`move_references`) */ + if (data->decref_src.func(&data->decref_src.context, + &orig_src, &N, &src_stride, data->decref_src.auxdata) < 0) { + return -1; + } + } + return 0; +} + + +NPY_NO_EXPORT int +any_to_object_get_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, + npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + + *flags = NPY_METH_REQUIRES_PYAPI; /* No need for floating point errors */ + + *out_loop = _strided_to_strided_any_to_object; + *out_transferdata = PyMem_Malloc(sizeof(_any_to_object_auxdata)); + if (*out_transferdata == NULL) { + return -1; + } + _any_to_object_auxdata *data = (_any_to_object_auxdata *)*out_transferdata; + data->base.free = &_any_to_object_auxdata_free; + data->base.clone = &_any_to_object_auxdata_clone; + data->arr_fields.base = NULL; + data->arr_fields.descr = context->descriptors[0]; + Py_INCREF(data->arr_fields.descr); + data->arr_fields.flags = aligned ? NPY_ARRAY_ALIGNED : 0; + data->arr_fields.nd = 0; + + data->getitem = context->descriptors[0]->f->getitem; + NPY_cast_info_init(&data->decref_src); + + if (move_references && PyDataType_REFCHK(context->descriptors[0])) { + int needs_api; + if (get_decref_transfer_function( + aligned, strides[0], context->descriptors[0], + &data->decref_src, + &needs_api) == NPY_FAIL) { + NPY_AUXDATA_FREE(*out_transferdata); + *out_transferdata = NULL; + return -1; + } + } + return 0; +} -/************************** ZERO-PADDED COPY ******************************/ -/* Does a zero-padded copy */ +/************************** OBJECT TO ANY *********************************/ + typedef struct { NpyAuxData base; - npy_intp dst_itemsize; -} _strided_zero_pad_data; + PyArray_Descr *descr; + int move_references; +} _object_to_any_auxdata; -/* zero-padded data copy function */ -static NpyAuxData *_strided_zero_pad_data_clone(NpyAuxData *data) + +static void +_object_to_any_auxdata_free(NpyAuxData *auxdata) { - _strided_zero_pad_data *newdata = - (_strided_zero_pad_data *)PyArray_malloc( - sizeof(_strided_zero_pad_data)); - if (newdata == NULL) { + _object_to_any_auxdata *data = (_object_to_any_auxdata *)auxdata; + Py_DECREF(data->descr); + PyMem_Free(data); +} + +static NpyAuxData * +_object_to_any_auxdata_clone(NpyAuxData *data) +{ + _object_to_any_auxdata *res = PyMem_Malloc(sizeof(*res)); + if (res == NULL) { return NULL; } + memcpy(res, data, sizeof(*res)); + Py_INCREF(res->descr); + return (NpyAuxData *)res; +} - memcpy(newdata, data, sizeof(_strided_zero_pad_data)); - return (NpyAuxData *)newdata; +static int +strided_to_strided_object_to_any( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) +{ + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + _object_to_any_auxdata *data = (_object_to_any_auxdata *)auxdata; + + PyObject *src_ref; + + while (N > 0) { + memcpy(&src_ref, src, sizeof(src_ref)); + if (PyArray_Pack(data->descr, dst, src_ref) < 0) { + return -1; + } + + if (data->move_references) { + Py_DECREF(src_ref); + memset(src, 0, sizeof(src_ref)); + } + + N--; + dst += dst_stride; + src += src_stride; + } + return 0; +} + + +NPY_NO_EXPORT int +object_to_any_get_loop( + PyArrayMethod_Context *context, + int NPY_UNUSED(aligned), int move_references, + npy_intp *NPY_UNUSED(strides), + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags) +{ + *flags = NPY_METH_REQUIRES_PYAPI; + + /* NOTE: auxdata is only really necessary to flag `move_references` */ + _object_to_any_auxdata *data = PyMem_Malloc(sizeof(*data)); + if (data == NULL) { + return -1; + } + data->base.free = &_object_to_any_auxdata_free; + data->base.clone = &_object_to_any_auxdata_clone; + + Py_INCREF(context->descriptors[1]); + data->descr = context->descriptors[1]; + data->move_references = move_references; + *out_transferdata = (NpyAuxData *)data; + *out_loop = &strided_to_strided_object_to_any; + return 0; } + +/************************** ZERO-PADDED COPY ******************************/ + /* * Does a strided to strided zero-padded copy for the case where * dst_itemsize > src_itemsize */ static int -_strided_to_strided_zero_pad_copy(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_zero_pad_copy( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { - _strided_zero_pad_data *d = (_strided_zero_pad_data *)data; - npy_intp dst_itemsize = d->dst_itemsize; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + npy_intp dst_itemsize = context->descriptors[1]->elsize; + npy_intp zero_size = dst_itemsize-src_itemsize; while (N > 0) { @@ -214,13 +402,15 @@ _strided_to_strided_zero_pad_copy(char *dst, npy_intp dst_stride, * dst_itemsize < src_itemsize */ static int -_strided_to_strided_truncate_copy(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_truncate_copy( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { - _strided_zero_pad_data *d = (_strided_zero_pad_data *)data; - npy_intp dst_itemsize = d->dst_itemsize; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + npy_intp dst_itemsize = context->descriptors[1]->elsize; while (N > 0) { memcpy(dst, src, dst_itemsize); @@ -236,13 +426,17 @@ _strided_to_strided_truncate_copy(char *dst, npy_intp dst_stride, * unicode swapping is needed. */ static int -_strided_to_strided_unicode_copyswap(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_unicode_copyswap( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { - _strided_zero_pad_data *d = (_strided_zero_pad_data *)data; - npy_intp dst_itemsize = d->dst_itemsize; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + npy_intp dst_itemsize = context->descriptors[1]->elsize; + npy_intp zero_size = dst_itemsize - src_itemsize; npy_intp copy_size = zero_size > 0 ? src_itemsize : dst_itemsize; char *_dst; @@ -271,26 +465,16 @@ NPY_NO_EXPORT int PyArray_GetStridedZeroPadCopyFn(int aligned, int unicode_swap, npy_intp src_stride, npy_intp dst_stride, npy_intp src_itemsize, npy_intp dst_itemsize, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata) { + *out_transferdata = NULL; if ((src_itemsize == dst_itemsize) && !unicode_swap) { *out_stransfer = PyArray_GetStridedCopyFn(aligned, src_stride, dst_stride, src_itemsize); - *out_transferdata = NULL; return (*out_stransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; } else { - _strided_zero_pad_data *d = PyArray_malloc( - sizeof(_strided_zero_pad_data)); - if (d == NULL) { - PyErr_NoMemory(); - return NPY_FAIL; - } - d->dst_itemsize = dst_itemsize; - d->base.free = (NpyAuxData_FreeFunc *)&PyArray_free; - d->base.clone = &_strided_zero_pad_data_clone; - if (unicode_swap) { *out_stransfer = &_strided_to_strided_unicode_copyswap; } @@ -300,218 +484,10 @@ PyArray_GetStridedZeroPadCopyFn(int aligned, int unicode_swap, else { *out_stransfer = &_strided_to_strided_truncate_copy; } - - *out_transferdata = (NpyAuxData *)d; return NPY_SUCCEED; } } -/***************** WRAP ALIGNED CONTIGUOUS TRANSFER FUNCTION **************/ - -/* Wraps a transfer function + data in alignment code */ -typedef struct { - NpyAuxData base; - PyArray_StridedUnaryOp *wrapped, - *tobuffer, *frombuffer; - NpyAuxData *wrappeddata, *todata, *fromdata; - npy_intp src_itemsize, dst_itemsize; - char *bufferin, *bufferout; - npy_bool init_dest, out_needs_api; -} _align_wrap_data; - -/* transfer data free function */ -static void _align_wrap_data_free(NpyAuxData *data) -{ - _align_wrap_data *d = (_align_wrap_data *)data; - NPY_AUXDATA_FREE(d->wrappeddata); - NPY_AUXDATA_FREE(d->todata); - NPY_AUXDATA_FREE(d->fromdata); - PyArray_free(data); -} - -/* transfer data copy function */ -static NpyAuxData *_align_wrap_data_clone(NpyAuxData *data) -{ - _align_wrap_data *d = (_align_wrap_data *)data; - _align_wrap_data *newdata; - npy_intp basedatasize, datasize; - - /* Round up the structure size to 16-byte boundary */ - basedatasize = (sizeof(_align_wrap_data)+15)&(-0x10); - /* Add space for two low level buffers */ - datasize = basedatasize + - NPY_LOWLEVEL_BUFFER_BLOCKSIZE*d->src_itemsize + - NPY_LOWLEVEL_BUFFER_BLOCKSIZE*d->dst_itemsize; - - /* Allocate the data, and populate it */ - newdata = (_align_wrap_data *)PyArray_malloc(datasize); - if (newdata == NULL) { - return NULL; - } - memcpy(newdata, data, basedatasize); - newdata->bufferin = (char *)newdata + basedatasize; - newdata->bufferout = newdata->bufferin + - NPY_LOWLEVEL_BUFFER_BLOCKSIZE*newdata->src_itemsize; - if (newdata->wrappeddata != NULL) { - newdata->wrappeddata = NPY_AUXDATA_CLONE(d->wrappeddata); - if (newdata->wrappeddata == NULL) { - PyArray_free(newdata); - return NULL; - } - } - if (newdata->todata != NULL) { - newdata->todata = NPY_AUXDATA_CLONE(d->todata); - if (newdata->todata == NULL) { - NPY_AUXDATA_FREE(newdata->wrappeddata); - PyArray_free(newdata); - return NULL; - } - } - if (newdata->fromdata != NULL) { - newdata->fromdata = NPY_AUXDATA_CLONE(d->fromdata); - if (newdata->fromdata == NULL) { - NPY_AUXDATA_FREE(newdata->wrappeddata); - NPY_AUXDATA_FREE(newdata->todata); - PyArray_free(newdata); - return NULL; - } - } - - newdata->init_dest = d->init_dest; - newdata->out_needs_api = d->out_needs_api; - - return (NpyAuxData *)newdata; -} - -static int -_strided_to_strided_contig_align_wrap(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) -{ - _align_wrap_data *d = (_align_wrap_data *)data; - PyArray_StridedUnaryOp *wrapped = d->wrapped, - *tobuffer = d->tobuffer, - *frombuffer = d->frombuffer; - npy_intp inner_src_itemsize = d->src_itemsize, - dst_itemsize = d->dst_itemsize; - NpyAuxData *wrappeddata = d->wrappeddata, - *todata = d->todata, - *fromdata = d->fromdata; - char *bufferin = d->bufferin, *bufferout = d->bufferout; - npy_bool init_dest = d->init_dest; - - for(;;) { - if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { - if (tobuffer( - bufferin, inner_src_itemsize, src, src_stride, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, src_itemsize, todata) < 0) { - return -1; - } - if (init_dest) { - memset(bufferout, 0, - dst_itemsize*NPY_LOWLEVEL_BUFFER_BLOCKSIZE); - } - if (wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - inner_src_itemsize, wrappeddata) < 0) { - return -1; - } - if (frombuffer(dst, dst_stride, bufferout, dst_itemsize, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - dst_itemsize, fromdata) < 0) { - return -1; - } - N -= NPY_LOWLEVEL_BUFFER_BLOCKSIZE; - src += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_stride; - dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride; - } - else { - if (tobuffer(bufferin, inner_src_itemsize, src, src_stride, - N, src_itemsize, todata) < 0) { - return -1; - } - if (init_dest) { - memset(bufferout, 0, dst_itemsize*N); - } - if (wrapped(bufferout, dst_itemsize, bufferin, inner_src_itemsize, - N, inner_src_itemsize, wrappeddata) < 0) { - return -1; - } - if (frombuffer(dst, dst_stride, bufferout, dst_itemsize, - N, dst_itemsize, fromdata) < 0) { - return -1; - } - return 0; - } - } -} - -/* - * Wraps an aligned contig to contig transfer function between either - * copies or byte swaps to temporary buffers. - * - * src_itemsize/dst_itemsize - The sizes of the src and dst datatypes. - * tobuffer - copy/swap function from src to an aligned contiguous buffer. - * todata - data for tobuffer - * frombuffer - copy/swap function from an aligned contiguous buffer to dst. - * fromdata - data for frombuffer - * wrapped - contig to contig transfer function being wrapped - * wrappeddata - data for wrapped - * init_dest - 1 means to memset the dest buffer to 0 before calling wrapped. - * out_needs_api - if NPY_TRUE, check for (and break on) Python API errors. - * - * Returns NPY_SUCCEED or NPY_FAIL. - */ -NPY_NO_EXPORT int -wrap_aligned_contig_transfer_function( - npy_intp src_itemsize, npy_intp dst_itemsize, - PyArray_StridedUnaryOp *tobuffer, NpyAuxData *todata, - PyArray_StridedUnaryOp *frombuffer, NpyAuxData *fromdata, - PyArray_StridedUnaryOp *wrapped, NpyAuxData *wrappeddata, - int init_dest, - int out_needs_api, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata) -{ - _align_wrap_data *data; - npy_intp basedatasize, datasize; - - /* Round up the structure size to 16-byte boundary */ - basedatasize = (sizeof(_align_wrap_data)+15)&(-0x10); - /* Add space for two low level buffers */ - datasize = basedatasize + - NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_itemsize + - NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_itemsize; - - /* Allocate the data, and populate it */ - data = (_align_wrap_data *)PyArray_malloc(datasize); - if (data == NULL) { - PyErr_NoMemory(); - return NPY_FAIL; - } - data->base.free = &_align_wrap_data_free; - data->base.clone = &_align_wrap_data_clone; - data->tobuffer = tobuffer; - data->todata = todata; - data->frombuffer = frombuffer; - data->fromdata = fromdata; - data->wrapped = wrapped; - data->wrappeddata = wrappeddata; - data->src_itemsize = src_itemsize; - data->dst_itemsize = dst_itemsize; - data->bufferin = (char *)data + basedatasize; - data->bufferout = data->bufferin + - NPY_LOWLEVEL_BUFFER_BLOCKSIZE*src_itemsize; - data->init_dest = (npy_bool) init_dest; - data->out_needs_api = (npy_bool) out_needs_api; - - /* Set the function and data */ - *out_stransfer = &_strided_to_strided_contig_align_wrap; - *out_transferdata = (NpyAuxData *)data; - - return NPY_SUCCEED; -} /*************************** WRAP DTYPE COPY/SWAP *************************/ /* Wraps the dtype copy swap function */ @@ -527,14 +503,14 @@ static void _wrap_copy_swap_data_free(NpyAuxData *data) { _wrap_copy_swap_data *d = (_wrap_copy_swap_data *)data; Py_DECREF(d->arr); - PyArray_free(data); + PyMem_Free(data); } /* wrap copy swap data copy function */ static NpyAuxData *_wrap_copy_swap_data_clone(NpyAuxData *data) { _wrap_copy_swap_data *newdata = - (_wrap_copy_swap_data *)PyArray_malloc(sizeof(_wrap_copy_swap_data)); + (_wrap_copy_swap_data *)PyMem_Malloc(sizeof(_wrap_copy_swap_data)); if (newdata == NULL) { return NULL; } @@ -546,32 +522,35 @@ static NpyAuxData *_wrap_copy_swap_data_clone(NpyAuxData *data) } static int -_strided_to_strided_wrap_copy_swap(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) +_strided_to_strided_wrap_copy_swap( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _wrap_copy_swap_data *d = (_wrap_copy_swap_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _wrap_copy_swap_data *d = (_wrap_copy_swap_data *)auxdata; /* We assume that d->copyswapn should not be able to error. */ d->copyswapn(dst, dst_stride, src, src_stride, N, d->swap, d->arr); return 0; } -/* This only gets used for custom data types and for Unicode when swapping */ +/* + * This function is used only via `get_wrapped_legacy_cast_function` + * when we wrap a legacy DType (or explicitly fall back to the legacy + * wrapping) for an internal cast. + */ static int -wrap_copy_swap_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *dtype, - int should_swap, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata) +wrap_copy_swap_function( + PyArray_Descr *dtype, int should_swap, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata) { - _wrap_copy_swap_data *data; - npy_intp shape = 1; - /* Allocate the data for the copy swap */ - data = (_wrap_copy_swap_data *)PyArray_malloc(sizeof(_wrap_copy_swap_data)); + _wrap_copy_swap_data *data = PyMem_Malloc(sizeof(_wrap_copy_swap_data)); if (data == NULL) { PyErr_NoMemory(); *out_stransfer = NULL; @@ -589,13 +568,14 @@ wrap_copy_swap_function(int aligned, * The copyswap functions shouldn't need that. */ Py_INCREF(dtype); + npy_intp shape = 1; data->arr = (PyArrayObject *)PyArray_NewFromDescr_int( &PyArray_Type, dtype, 1, &shape, NULL, NULL, 0, NULL, NULL, 0, 1); if (data->arr == NULL) { - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } @@ -621,14 +601,14 @@ static void _strided_cast_data_free(NpyAuxData *data) _strided_cast_data *d = (_strided_cast_data *)data; Py_DECREF(d->aip); Py_DECREF(d->aop); - PyArray_free(data); + PyMem_Free(data); } /* strided cast data copy function */ static NpyAuxData *_strided_cast_data_clone(NpyAuxData *data) { _strided_cast_data *newdata = - (_strided_cast_data *)PyArray_malloc(sizeof(_strided_cast_data)); + (_strided_cast_data *)PyMem_Malloc(sizeof(_strided_cast_data)); if (newdata == NULL) { return NULL; } @@ -641,12 +621,16 @@ static NpyAuxData *_strided_cast_data_clone(NpyAuxData *data) } static int -_aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_aligned_strided_to_strided_cast( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _strided_cast_data *d = (_strided_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _strided_cast_data *d = (_strided_cast_data *)auxdata; PyArray_VectorUnaryFunc *castfunc = d->castfunc; PyArrayObject *aip = d->aip, *aop = d->aop; npy_bool needs_api = d->needs_api; @@ -672,11 +656,16 @@ _aligned_strided_to_strided_cast(char *dst, npy_intp dst_stride, /* This one requires src be of type NPY_OBJECT */ static int -_aligned_strided_to_strided_cast_decref_src(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_aligned_strided_to_strided_cast_decref_src( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _any_to_object_auxdata *data = (_any_to_object_auxdata *)auxdata; _strided_cast_data *d = (_strided_cast_data *)data; PyArray_VectorUnaryFunc *castfunc = d->castfunc; PyArrayObject *aip = d->aip, *aop = d->aop; @@ -706,12 +695,15 @@ _aligned_strided_to_strided_cast_decref_src(char *dst, npy_intp dst_stride, } static int -_aligned_contig_to_contig_cast(char *dst, npy_intp NPY_UNUSED(dst_stride), - char *src, npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(itemsize), - NpyAuxData *data) +_aligned_contig_to_contig_cast( + PyArrayMethod_Context *NPY_UNUSED(context), char * const*args, + const npy_intp *dimensions, const npy_intp *NPY_UNUSED(strides), + NpyAuxData *auxdata) { - _strided_cast_data *d = (_strided_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + + _strided_cast_data *d = (_strided_cast_data *)auxdata; npy_bool needs_api = d->needs_api; d->castfunc(src, dst, N, d->aip, d->aop); @@ -725,43 +717,6 @@ _aligned_contig_to_contig_cast(char *dst, npy_intp NPY_UNUSED(dst_stride), return 0; } -static int -get_nbo_cast_numeric_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - int src_type_num, int dst_type_num, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata) -{ - /* Emit a warning if complex imaginary is being cast away */ - if (PyTypeNum_ISCOMPLEX(src_type_num) && - !PyTypeNum_ISCOMPLEX(dst_type_num) && - !PyTypeNum_ISBOOL(dst_type_num)) { - static PyObject *cls = NULL; - int ret; - npy_cache_import("numpy.core", "ComplexWarning", &cls); - if (cls == NULL) { - return NPY_FAIL; - } - ret = PyErr_WarnEx(cls, - "Casting complex values to real discards " - "the imaginary part", 1); - if (ret < 0) { - return NPY_FAIL; - } - } - - *out_stransfer = PyArray_GetStridedNumericCastFn(aligned, - src_stride, dst_stride, - src_type_num, dst_type_num); - *out_transferdata = NULL; - if (*out_stransfer == NULL) { - PyErr_SetString(PyExc_ValueError, - "unexpected error in GetStridedNumericCastFn"); - return NPY_FAIL; - } - - return NPY_SUCCEED; -} /* * Does a datetime->datetime, timedelta->timedelta, @@ -791,15 +746,15 @@ typedef struct { static void _strided_datetime_cast_data_free(NpyAuxData *data) { _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; - PyArray_free(d->tmp_buffer); - PyArray_free(data); + PyMem_Free(d->tmp_buffer); + PyMem_Free(data); } /* strided datetime cast data copy function */ static NpyAuxData *_strided_datetime_cast_data_clone(NpyAuxData *data) { _strided_datetime_cast_data *newdata = - (_strided_datetime_cast_data *)PyArray_malloc( + (_strided_datetime_cast_data *)PyMem_Malloc( sizeof(_strided_datetime_cast_data)); if (newdata == NULL) { return NULL; @@ -807,9 +762,9 @@ static NpyAuxData *_strided_datetime_cast_data_clone(NpyAuxData *data) memcpy(newdata, data, sizeof(_strided_datetime_cast_data)); if (newdata->tmp_buffer != NULL) { - newdata->tmp_buffer = PyArray_malloc(newdata->src_itemsize + 1); + newdata->tmp_buffer = PyMem_Malloc(newdata->src_itemsize + 1); if (newdata->tmp_buffer == NULL) { - PyArray_free(newdata); + PyMem_Free(newdata); return NULL; } } @@ -818,12 +773,16 @@ static NpyAuxData *_strided_datetime_cast_data_clone(NpyAuxData *data) } static int -_strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_datetime_general_cast( + PyArrayMethod_Context *NPY_UNUSED(context), char * const*args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)auxdata; npy_int64 dt; npy_datetimestruct dts; @@ -851,12 +810,16 @@ _strided_to_strided_datetime_general_cast(char *dst, npy_intp dst_stride, } static int -_strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_datetime_cast( + PyArrayMethod_Context *NPY_UNUSED(context), char * const*args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)auxdata; npy_int64 num = d->num, denom = d->denom; npy_int64 dt; @@ -883,13 +846,16 @@ _strided_to_strided_datetime_cast(char *dst, npy_intp dst_stride, } static int -_aligned_strided_to_strided_datetime_cast(char *dst, - npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_aligned_strided_to_strided_datetime_cast( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)auxdata; npy_int64 num = d->num, denom = d->denom; npy_int64 dt; @@ -916,12 +882,16 @@ _aligned_strided_to_strided_datetime_cast(char *dst, } static int -_strided_to_strided_datetime_to_string(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) +_strided_to_strided_datetime_to_string( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)auxdata; npy_intp dst_itemsize = d->dst_itemsize; npy_int64 dt; npy_datetimestruct dts; @@ -951,12 +921,17 @@ _strided_to_strided_datetime_to_string(char *dst, npy_intp dst_stride, } static int -_strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_string_to_datetime( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _strided_datetime_cast_data *d = (_strided_datetime_cast_data *)auxdata; npy_datetimestruct dts; char *tmp_buffer = d->tmp_buffer; char *tmp; @@ -1009,7 +984,7 @@ _strided_to_strided_string_to_datetime(char *dst, npy_intp dst_stride, NPY_NO_EXPORT int get_nbo_cast_datetime_transfer_function(int aligned, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata) { PyArray_DatetimeMetaData *src_meta, *dst_meta; @@ -1032,7 +1007,7 @@ get_nbo_cast_datetime_transfer_function(int aligned, } /* Allocate the data for the casting */ - data = (_strided_datetime_cast_data *)PyArray_malloc( + data = (_strided_datetime_cast_data *)PyMem_Malloc( sizeof(_strided_datetime_cast_data)); if (data == NULL) { PyErr_NoMemory(); @@ -1084,7 +1059,7 @@ get_nbo_cast_datetime_transfer_function(int aligned, NPY_NO_EXPORT int get_nbo_datetime_to_string_transfer_function( PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata) + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata) { PyArray_DatetimeMetaData *src_meta; _strided_datetime_cast_data *data; @@ -1095,7 +1070,7 @@ get_nbo_datetime_to_string_transfer_function( } /* Allocate the data for the casting */ - data = (_strided_datetime_cast_data *)PyArray_malloc( + data = (_strided_datetime_cast_data *)PyMem_Malloc( sizeof(_strided_datetime_cast_data)); if (data == NULL) { PyErr_NoMemory(); @@ -1124,80 +1099,53 @@ get_nbo_datetime_to_string_transfer_function( return NPY_SUCCEED; } + NPY_NO_EXPORT int get_datetime_to_unicode_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api) { - NpyAuxData *castdata = NULL, *todata = NULL, *fromdata = NULL; - PyArray_StridedUnaryOp *caststransfer, *tobuffer, *frombuffer; PyArray_Descr *str_dtype; /* Get an ASCII string data type, adapted to match the UNICODE one */ str_dtype = PyArray_DescrNewFromType(NPY_STRING); - str_dtype->elsize = dst_dtype->elsize / 4; if (str_dtype == NULL) { return NPY_FAIL; } + str_dtype->elsize = dst_dtype->elsize / 4; - /* Get the copy/swap operation to dst */ - if (PyArray_GetDTypeCopySwapFn(aligned, - src_stride, src_dtype->elsize, - src_dtype, - &tobuffer, &todata) != NPY_SUCCEED) { - Py_DECREF(str_dtype); - return NPY_FAIL; - } + /* ensured in resolve_descriptors for simplicity */ + assert(PyDataType_ISNOTSWAPPED(src_dtype)); /* Get the NBO datetime to string aligned contig function */ if (get_nbo_datetime_to_string_transfer_function( src_dtype, str_dtype, - &caststransfer, &castdata) != NPY_SUCCEED) { - Py_DECREF(str_dtype); - NPY_AUXDATA_FREE(todata); - return NPY_FAIL; - } - - /* Get the cast operation to dst */ - if (PyArray_GetDTypeTransferFunction(aligned, - str_dtype->elsize, dst_stride, - str_dtype, dst_dtype, - 0, - &frombuffer, &fromdata, - out_needs_api) != NPY_SUCCEED) { + out_stransfer, out_transferdata) != NPY_SUCCEED) { Py_DECREF(str_dtype); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(castdata); return NPY_FAIL; } - /* Wrap it all up in a new transfer function + data */ - if (wrap_aligned_contig_transfer_function( - src_dtype->elsize, str_dtype->elsize, - tobuffer, todata, - frombuffer, fromdata, - caststransfer, castdata, - PyDataType_FLAGCHK(str_dtype, NPY_NEEDS_INIT), - *out_needs_api, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); + int res = wrap_aligned_transferfunction( + aligned, 0, /* no need to ensure contiguous */ + src_stride, dst_stride, + src_dtype, dst_dtype, + src_dtype, str_dtype, + out_stransfer, out_transferdata, out_needs_api); + Py_DECREF(str_dtype); + if (res < 0) { return NPY_FAIL; } - Py_DECREF(str_dtype); - return NPY_SUCCEED; } NPY_NO_EXPORT int get_nbo_string_to_datetime_transfer_function( PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata) + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata) { PyArray_DatetimeMetaData *dst_meta; _strided_datetime_cast_data *data; @@ -1208,7 +1156,7 @@ get_nbo_string_to_datetime_transfer_function( } /* Allocate the data for the casting */ - data = (_strided_datetime_cast_data *)PyArray_malloc( + data = (_strided_datetime_cast_data *)PyMem_Malloc( sizeof(_strided_datetime_cast_data)); if (data == NULL) { PyErr_NoMemory(); @@ -1219,10 +1167,10 @@ get_nbo_string_to_datetime_transfer_function( data->base.free = &_strided_datetime_cast_data_free; data->base.clone = &_strided_datetime_cast_data_clone; data->src_itemsize = src_dtype->elsize; - data->tmp_buffer = PyArray_malloc(data->src_itemsize + 1); + data->tmp_buffer = PyMem_Malloc(data->src_itemsize + 1); if (data->tmp_buffer == NULL) { PyErr_NoMemory(); - PyArray_free(data); + PyMem_Free(data); *out_stransfer = NULL; *out_transferdata = NULL; return NPY_FAIL; @@ -1248,12 +1196,10 @@ NPY_NO_EXPORT int get_unicode_to_datetime_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api) { - NpyAuxData *castdata = NULL, *todata = NULL, *fromdata = NULL; - PyArray_StridedUnaryOp *caststransfer, *tobuffer, *frombuffer; PyArray_Descr *str_dtype; /* Get an ASCII string data type, adapted to match the UNICODE one */ @@ -1264,55 +1210,25 @@ get_unicode_to_datetime_transfer_function(int aligned, assert(src_dtype->type_num == NPY_UNICODE); str_dtype->elsize = src_dtype->elsize / 4; - /* Get the cast operation from src */ - if (PyArray_GetDTypeTransferFunction(aligned, - src_stride, str_dtype->elsize, - src_dtype, str_dtype, - 0, - &tobuffer, &todata, - out_needs_api) != NPY_SUCCEED) { - Py_DECREF(str_dtype); - return NPY_FAIL; - } - - /* Get the string to NBO datetime aligned contig function */ + /* Get the string to NBO datetime aligned function */ if (get_nbo_string_to_datetime_transfer_function( str_dtype, dst_dtype, - &caststransfer, &castdata) != NPY_SUCCEED) { + out_stransfer, out_transferdata) != NPY_SUCCEED) { Py_DECREF(str_dtype); - NPY_AUXDATA_FREE(todata); return NPY_FAIL; } - /* Get the copy/swap operation to dst */ - if (PyArray_GetDTypeCopySwapFn(aligned, - dst_dtype->elsize, dst_stride, - dst_dtype, - &frombuffer, &fromdata) != NPY_SUCCEED) { - Py_DECREF(str_dtype); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(castdata); - return NPY_FAIL; - } + int res = wrap_aligned_transferfunction( + aligned, 0, /* no need to ensure contiguous */ + src_stride, dst_stride, + src_dtype, dst_dtype, + str_dtype, dst_dtype, + out_stransfer, out_transferdata, out_needs_api); + Py_DECREF(str_dtype); - /* Wrap it all up in a new transfer function + data */ - if (wrap_aligned_contig_transfer_function( - str_dtype->elsize, dst_dtype->elsize, - tobuffer, todata, - frombuffer, fromdata, - caststransfer, castdata, - PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), - *out_needs_api, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - Py_DECREF(str_dtype); - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); + if (res < 0) { return NPY_FAIL; } - - Py_DECREF(str_dtype); - return NPY_SUCCEED; } @@ -1322,7 +1238,7 @@ get_legacy_dtype_cast_function( int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int move_references, - PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api, int *out_needs_wrap) { _strided_cast_data *data; @@ -1374,7 +1290,7 @@ get_legacy_dtype_cast_function( } /* Allocate the data for the casting */ - data = (_strided_cast_data *)PyArray_malloc(sizeof(_strided_cast_data)); + data = (_strided_cast_data *)PyMem_Malloc(sizeof(_strided_cast_data)); if (data == NULL) { PyErr_NoMemory(); *out_stransfer = NULL; @@ -1398,7 +1314,7 @@ get_legacy_dtype_cast_function( else { tmp_dtype = PyArray_DescrNewByteorder(src_dtype, NPY_NATIVE); if (tmp_dtype == NULL) { - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } } @@ -1408,7 +1324,7 @@ get_legacy_dtype_cast_function( 0, NULL, NULL, 0, 1); if (data->aip == NULL) { - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } /* @@ -1425,7 +1341,7 @@ get_legacy_dtype_cast_function( tmp_dtype = PyArray_DescrNewByteorder(dst_dtype, NPY_NATIVE); if (tmp_dtype == NULL) { Py_DECREF(data->aip); - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } } @@ -1436,7 +1352,7 @@ get_legacy_dtype_cast_function( 0, 1); if (data->aop == NULL) { Py_DECREF(data->aip); - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } @@ -1464,224 +1380,24 @@ get_legacy_dtype_cast_function( } -static int -get_nbo_cast_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api, - int *out_needs_wrap) -{ - if (PyTypeNum_ISNUMBER(src_dtype->type_num) && - PyTypeNum_ISNUMBER(dst_dtype->type_num)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_numeric_transfer_function(aligned, - src_stride, dst_stride, - src_dtype->type_num, dst_dtype->type_num, - out_stransfer, out_transferdata); - } - - if (src_dtype->type_num == NPY_DATETIME || - src_dtype->type_num == NPY_TIMEDELTA || - dst_dtype->type_num == NPY_DATETIME || - dst_dtype->type_num == NPY_TIMEDELTA) { - /* A parameterized type, datetime->datetime sometimes needs casting */ - if ((src_dtype->type_num == NPY_DATETIME && - dst_dtype->type_num == NPY_DATETIME) || - (src_dtype->type_num == NPY_TIMEDELTA && - dst_dtype->type_num == NPY_TIMEDELTA)) { - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder) || - !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_cast_datetime_transfer_function(aligned, - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - } - - /* - * Datetime <-> string conversions can be handled specially. - * The functions may raise an error if the strings have no - * space, or can't be parsed properly. - */ - if (src_dtype->type_num == NPY_DATETIME) { - switch (dst_dtype->type_num) { - case NPY_STRING: - *out_needs_api = 1; - *out_needs_wrap = !PyArray_ISNBO(src_dtype->byteorder); - return get_nbo_datetime_to_string_transfer_function( - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - - case NPY_UNICODE: - return get_datetime_to_unicode_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); - } - } - else if (dst_dtype->type_num == NPY_DATETIME) { - switch (src_dtype->type_num) { - case NPY_STRING: - *out_needs_api = 1; - *out_needs_wrap = !PyArray_ISNBO(dst_dtype->byteorder); - return get_nbo_string_to_datetime_transfer_function( - src_dtype, dst_dtype, - out_stransfer, out_transferdata); - - case NPY_UNICODE: - return get_unicode_to_datetime_transfer_function( - aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); - } - } - } - - return get_legacy_dtype_cast_function( - aligned, src_stride, dst_stride, src_dtype, dst_dtype, - move_references, out_stransfer, out_transferdata, - out_needs_api, out_needs_wrap); -} - - -NPY_NO_EXPORT int -wrap_aligned_contig_transfer_function_with_copyswapn( - int aligned, npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, NpyAuxData **out_transferdata, - int *out_needs_api, - PyArray_StridedUnaryOp *caststransfer, NpyAuxData *castdata) -{ - NpyAuxData *todata = NULL, *fromdata = NULL; - PyArray_StridedUnaryOp *tobuffer = NULL, *frombuffer = NULL; - npy_intp src_itemsize = src_dtype->elsize; - npy_intp dst_itemsize = dst_dtype->elsize; - - /* Get the copy/swap operation from src */ - PyArray_GetDTypeCopySwapFn( - aligned, src_stride, src_itemsize, src_dtype, &tobuffer, &todata); - - if (!PyDataType_REFCHK(dst_dtype)) { - /* Copying from buffer is a simple copy/swap operation */ - PyArray_GetDTypeCopySwapFn( - aligned, dst_itemsize, dst_stride, dst_dtype, - &frombuffer, &fromdata); - } - else { - /* - * Since the buffer is initialized to NULL, need to move the - * references in order to DECREF the existing data. - */ - /* Object types cannot be byte swapped */ - assert(PyDataType_ISNOTSWAPPED(dst_dtype)); - /* The loop already needs the python api if this is reached */ - assert(*out_needs_api); - - if (PyArray_GetDTypeTransferFunction( - aligned, dst_itemsize, dst_stride, - dst_dtype, dst_dtype, 1, - &frombuffer, &fromdata, out_needs_api) != NPY_SUCCEED) { - return NPY_FAIL; - } - } - - if (frombuffer == NULL || tobuffer == NULL) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); - return NPY_FAIL; - } - - *out_stransfer = caststransfer; - - /* Wrap it all up in a new transfer function + data */ - if (wrap_aligned_contig_transfer_function( - src_itemsize, dst_itemsize, - tobuffer, todata, - frombuffer, fromdata, - caststransfer, castdata, - PyDataType_FLAGCHK(dst_dtype, NPY_NEEDS_INIT), - *out_needs_api, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(castdata); - NPY_AUXDATA_FREE(todata); - NPY_AUXDATA_FREE(fromdata); - return NPY_FAIL; - } - - return NPY_SUCCEED; -} - - -static int -get_cast_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api) -{ - PyArray_StridedUnaryOp *caststransfer; - NpyAuxData *castdata; - int needs_wrap = 0; - - if (get_nbo_cast_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - &caststransfer, - &castdata, - out_needs_api, - &needs_wrap) != NPY_SUCCEED) { - return NPY_FAIL; - } - - /* - * If all native byte order and doesn't need alignment wrapping, - * return the function - */ - if (!needs_wrap) { - *out_stransfer = caststransfer; - *out_transferdata = castdata; - - return NPY_SUCCEED; - } - /* Otherwise, we have to copy and/or swap to aligned temporaries */ - else { - return wrap_aligned_contig_transfer_function_with_copyswapn( - aligned, src_stride, dst_stride, src_dtype, dst_dtype, - out_stransfer, out_transferdata, out_needs_api, - caststransfer, castdata); - } -} - /**************************** COPY 1 TO N CONTIGUOUS ************************/ /* Copies 1 element to N contiguous elements */ typedef struct { NpyAuxData base; - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *data; - npy_intp N, dst_itemsize; - /* If this is non-NULL the source type has references needing a decref */ - PyArray_StridedUnaryOp *stransfer_finish_src; - NpyAuxData *data_finish_src; + npy_intp N; + NPY_cast_info wrapped; + /* If finish->func is non-NULL the source needs a decref */ + NPY_cast_info decref_src; } _one_to_n_data; /* transfer data free function */ static void _one_to_n_data_free(NpyAuxData *data) { _one_to_n_data *d = (_one_to_n_data *)data; - NPY_AUXDATA_FREE(d->data); - NPY_AUXDATA_FREE(d->data_finish_src); - PyArray_free(data); + NPY_cast_info_xfree(&d->wrapped); + NPY_cast_info_xfree(&d->decref_src); + PyMem_Free(data); } /* transfer data copy function */ @@ -1691,44 +1407,51 @@ static NpyAuxData *_one_to_n_data_clone(NpyAuxData *data) _one_to_n_data *newdata; /* Allocate the data, and populate it */ - newdata = (_one_to_n_data *)PyArray_malloc(sizeof(_one_to_n_data)); + newdata = (_one_to_n_data *)PyMem_Malloc(sizeof(_one_to_n_data)); if (newdata == NULL) { return NULL; } - memcpy(newdata, data, sizeof(_one_to_n_data)); - if (d->data != NULL) { - newdata->data = NPY_AUXDATA_CLONE(d->data); - if (newdata->data == NULL) { - PyArray_free(newdata); - return NULL; - } + newdata->base.free = &_one_to_n_data_free; + newdata->base.clone = &_one_to_n_data_clone; + newdata->N = d->N; + /* Initialize in case of error, or if it is unused */ + NPY_cast_info_init(&newdata->decref_src); + + if (NPY_cast_info_copy(&newdata->wrapped, &d->wrapped) < 0) { + _one_to_n_data_free((NpyAuxData *)newdata); + return NULL; } - if (d->data_finish_src != NULL) { - newdata->data_finish_src = NPY_AUXDATA_CLONE(d->data_finish_src); - if (newdata->data_finish_src == NULL) { - NPY_AUXDATA_FREE(newdata->data); - PyArray_free(newdata); - return NULL; - } + if (d->decref_src.func == NULL) { + return (NpyAuxData *)newdata; + } + + if (NPY_cast_info_copy(&newdata->decref_src, &d->decref_src) < 0) { + _one_to_n_data_free((NpyAuxData *)newdata); + return NULL; } return (NpyAuxData *)newdata; } static int -_strided_to_strided_one_to_n(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_one_to_n( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _one_to_n_data *d = (_one_to_n_data *)data; - PyArray_StridedUnaryOp *subtransfer = d->stransfer; - NpyAuxData *subdata = d->data; - npy_intp subN = d->N, dst_itemsize = d->dst_itemsize; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _one_to_n_data *d = (_one_to_n_data *)auxdata; + + const npy_intp subN = d->N; + npy_intp sub_strides[2] = {0, d->wrapped.descriptors[1]->elsize}; while (N > 0) { - if (subtransfer( - dst, dst_itemsize, src, 0, subN, src_itemsize, subdata) < 0) { + char *sub_args[2] = {src, dst}; + if (d->wrapped.func(&d->wrapped.context, + sub_args, &subN, sub_strides, d->wrapped.auxdata) < 0) { return -1; } @@ -1740,25 +1463,30 @@ _strided_to_strided_one_to_n(char *dst, npy_intp dst_stride, } static int -_strided_to_strided_one_to_n_with_finish(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_one_to_n_with_finish( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _one_to_n_data *d = (_one_to_n_data *)data; - PyArray_StridedUnaryOp *subtransfer = d->stransfer, - *stransfer_finish_src = d->stransfer_finish_src; - NpyAuxData *subdata = d->data, *data_finish_src = d->data_finish_src; - npy_intp subN = d->N, dst_itemsize = d->dst_itemsize; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _one_to_n_data *d = (_one_to_n_data *)auxdata; + + const npy_intp subN = d->N; + const npy_intp one_item = 1, zero_stride = 0; + npy_intp sub_strides[2] = {0, d->wrapped.descriptors[1]->elsize}; while (N > 0) { - if (subtransfer( - dst, dst_itemsize, src, 0, subN, src_itemsize, subdata) < 0) { + char *sub_args[2] = {src, dst}; + if (d->wrapped.func(&d->wrapped.context, + sub_args, &subN, sub_strides, d->wrapped.auxdata) < 0) { return -1; } - if (stransfer_finish_src( - NULL, 0, src, 0, 1, src_itemsize, data_finish_src) < 0) { + if (d->decref_src.func(&d->decref_src.context, + &src, &one_item, &zero_stride, d->decref_src.auxdata) < 0) { return -1; } @@ -1769,27 +1497,18 @@ _strided_to_strided_one_to_n_with_finish(char *dst, npy_intp dst_stride, return 0; } -/* - * Wraps a transfer function to produce one that copies one element - * of src to N contiguous elements of dst. If stransfer_finish_src is - * not NULL, it should be a transfer function which just affects - * src, for example to do a final DECREF operation for references. - */ + static int -wrap_transfer_function_one_to_n( - PyArray_StridedUnaryOp *stransfer_inner, - NpyAuxData *data_inner, - PyArray_StridedUnaryOp *stransfer_finish_src, - NpyAuxData *data_finish_src, - npy_intp dst_itemsize, +get_one_to_n_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, npy_intp N, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata) + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api) { - _one_to_n_data *data; - - - data = PyArray_malloc(sizeof(_one_to_n_data)); + _one_to_n_data *data = PyMem_Malloc(sizeof(_one_to_n_data)); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; @@ -1797,36 +1516,8 @@ wrap_transfer_function_one_to_n( data->base.free = &_one_to_n_data_free; data->base.clone = &_one_to_n_data_clone; - data->stransfer = stransfer_inner; - data->data = data_inner; - data->stransfer_finish_src = stransfer_finish_src; - data->data_finish_src = data_finish_src; data->N = N; - data->dst_itemsize = dst_itemsize; - - if (stransfer_finish_src == NULL) { - *out_stransfer = &_strided_to_strided_one_to_n; - } - else { - *out_stransfer = &_strided_to_strided_one_to_n_with_finish; - } - *out_transferdata = (NpyAuxData *)data; - - return NPY_SUCCEED; -} - -static int -get_one_to_n_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - npy_intp N, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api) -{ - PyArray_StridedUnaryOp *stransfer, *stransfer_finish_src = NULL; - NpyAuxData *data, *data_finish_src = NULL; + NPY_cast_info_init(&data->decref_src); /* In case of error */ /* * move_references is set to 0, handled in the wrapping transfer fn, @@ -1838,33 +1529,31 @@ get_one_to_n_transfer_function(int aligned, 0, dst_dtype->elsize, src_dtype, dst_dtype, 0, - &stransfer, &data, + &data->wrapped, out_needs_api) != NPY_SUCCEED) { + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } /* If the src object will need a DECREF, set src_dtype */ if (move_references && PyDataType_REFCHK(src_dtype)) { - if (get_decsrcref_transfer_function(aligned, + if (get_decref_transfer_function(aligned, src_stride, src_dtype, - &stransfer_finish_src, - &data_finish_src, + &data->decref_src, out_needs_api) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } } - if (wrap_transfer_function_one_to_n(stransfer, data, - stransfer_finish_src, data_finish_src, - dst_dtype->elsize, - N, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(data); - NPY_AUXDATA_FREE(data_finish_src); - return NPY_FAIL; + if (data->decref_src.func == NULL) { + *out_stransfer = &_strided_to_strided_one_to_n; + } + else { + *out_stransfer = &_strided_to_strided_one_to_n_with_finish; } + *out_transferdata = (NpyAuxData *)data; return NPY_SUCCEED; } @@ -1874,17 +1563,17 @@ get_one_to_n_transfer_function(int aligned, /* Copies N contiguous elements to N contiguous elements */ typedef struct { NpyAuxData base; - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *data; - npy_intp N, src_itemsize, dst_itemsize; + NPY_cast_info wrapped; + npy_intp N; + npy_intp strides[2]; /* avoid look up on the dtype (dst can be NULL) */ } _n_to_n_data; /* transfer data free function */ static void _n_to_n_data_free(NpyAuxData *data) { _n_to_n_data *d = (_n_to_n_data *)data; - NPY_AUXDATA_FREE(d->data); - PyArray_free(data); + NPY_cast_info_xfree(&d->wrapped); + PyMem_Free(data); } /* transfer data copy function */ @@ -1894,38 +1583,47 @@ static NpyAuxData *_n_to_n_data_clone(NpyAuxData *data) _n_to_n_data *newdata; /* Allocate the data, and populate it */ - newdata = (_n_to_n_data *)PyArray_malloc(sizeof(_n_to_n_data)); + newdata = (_n_to_n_data *)PyMem_Malloc(sizeof(_n_to_n_data)); if (newdata == NULL) { return NULL; } - memcpy(newdata, data, sizeof(_n_to_n_data)); - if (newdata->data != NULL) { - newdata->data = NPY_AUXDATA_CLONE(d->data); - if (newdata->data == NULL) { - PyArray_free(newdata); - return NULL; - } + *newdata = *d; + + if (NPY_cast_info_copy(&newdata->wrapped, &d->wrapped) < 0) { + _n_to_n_data_free((NpyAuxData *)newdata); } return (NpyAuxData *)newdata; } static int -_strided_to_strided_n_to_n(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *data) +_strided_to_strided_1_to_1( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _n_to_n_data *d = (_n_to_n_data *)data; - PyArray_StridedUnaryOp *subtransfer = d->stransfer; - NpyAuxData *subdata = d->data; - npy_intp subN = d->N, src_subitemsize = d->src_itemsize, - dst_subitemsize = d->dst_itemsize; + _n_to_n_data *d = (_n_to_n_data *)auxdata; + return d->wrapped.func(&d->wrapped.context, + args, dimensions, strides, d->wrapped.auxdata); +} + +static int +_strided_to_strided_n_to_n( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) +{ + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _n_to_n_data *d = (_n_to_n_data *)auxdata; + npy_intp subN = d->N; while (N > 0) { - if (subtransfer( - dst, dst_subitemsize, src, src_subitemsize, - subN, src_subitemsize, subdata) < 0) { + char *sub_args[2] = {src, dst}; + if (d->wrapped.func(&d->wrapped.context, + sub_args, &subN, d->strides, d->wrapped.auxdata) < 0) { return -1; } src += src_stride; @@ -1936,106 +1634,93 @@ _strided_to_strided_n_to_n(char *dst, npy_intp dst_stride, } static int -_contig_to_contig_n_to_n(char *dst, npy_intp NPY_UNUSED(dst_stride), - char *src, npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) +_contig_to_contig_n_to_n( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *NPY_UNUSED(strides), + NpyAuxData *auxdata) { - _n_to_n_data *d = (_n_to_n_data *)data; - PyArray_StridedUnaryOp *subtransfer = d->stransfer; - NpyAuxData *subdata = d->data; - npy_intp subN = d->N, src_subitemsize = d->src_itemsize, - dst_subitemsize = d->dst_itemsize; - - if (subtransfer( - dst, dst_subitemsize, src, src_subitemsize, - subN*N, src_subitemsize, subdata) < 0) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + + _n_to_n_data *d = (_n_to_n_data *)auxdata; + /* Make one large transfer including both outer and inner iteration: */ + npy_intp subN = N * d->N; + + char *sub_args[2] = {src, dst}; + if (d->wrapped.func(&d->wrapped.context, + sub_args, &subN, d->strides, d->wrapped.auxdata) < 0) { return -1; } return 0; } + /* - * Wraps a transfer function to produce one that copies N contiguous elements - * of src to N contiguous elements of dst. + * Note that this function is currently both used for structured dtype + * casting as well as a decref function (with `dst_dtype == NULL`) */ static int -wrap_transfer_function_n_to_n( - PyArray_StridedUnaryOp *stransfer_inner, - NpyAuxData *data_inner, +get_n_to_n_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, - npy_intp src_itemsize, npy_intp dst_itemsize, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, npy_intp N, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata) + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api) { - _n_to_n_data *data; - - data = PyArray_malloc(sizeof(_n_to_n_data)); + _n_to_n_data *data = PyMem_Malloc(sizeof(_n_to_n_data)); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; } - data->base.free = &_n_to_n_data_free; data->base.clone = &_n_to_n_data_clone; - data->stransfer = stransfer_inner; - data->data = data_inner; data->N = N; - data->src_itemsize = src_itemsize; - data->dst_itemsize = dst_itemsize; - /* - * If the N subarray elements exactly fit in the strides, - * then can do a faster contiguous transfer. - */ - if (src_stride == N * src_itemsize && - dst_stride == N * dst_itemsize) { - *out_stransfer = &_contig_to_contig_n_to_n; - } - else { - *out_stransfer = &_strided_to_strided_n_to_n; + if (N != 1) { + /* + * If N == 1, we can use the original strides, + * otherwise fields are contiguous + */ + src_stride = src_dtype->elsize; + dst_stride = dst_dtype != NULL ? dst_dtype->elsize : 0; + /* Store the wrapped strides for easier access */ + data->strides[0] = src_stride; + data->strides[1] = dst_stride; } - *out_transferdata = (NpyAuxData *)data; - - return NPY_SUCCEED; -} - -static int -get_n_to_n_transfer_function(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - npy_intp N, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api) -{ - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *data; /* * src_stride and dst_stride are set to contiguous, because * subarrays are always contiguous. */ if (PyArray_GetDTypeTransferFunction(aligned, - src_dtype->elsize, dst_dtype->elsize, + src_stride, dst_stride, src_dtype, dst_dtype, move_references, - &stransfer, &data, + &data->wrapped, out_needs_api) != NPY_SUCCEED) { + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } - if (wrap_transfer_function_n_to_n(stransfer, data, - src_stride, dst_stride, - src_dtype->elsize, dst_dtype->elsize, - N, - out_stransfer, - out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(data); - return NPY_FAIL; + if (N == 1) { + /* + * No need for wrapping, we can just copy directly. In principle + * this step could be optimized away entirely, but it requires + * replacing the context (to have the unpacked dtypes). + */ + *out_stransfer = &_strided_to_strided_1_to_1; + } + else if (src_stride == N * src_stride && + dst_stride == N * dst_stride) { + /* The subarrays can be coalesced (probably very rarely) */ + *out_stransfer = &_contig_to_contig_n_to_n; + } + else { + *out_stransfer = &_strided_to_strided_n_to_n; } + *out_transferdata = (NpyAuxData *)data; return NPY_SUCCEED; } @@ -2049,16 +1734,13 @@ typedef struct { /* Copies element with subarray broadcasting */ typedef struct { NpyAuxData base; - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *data; - npy_intp src_N, dst_N, src_itemsize, dst_itemsize; - PyArray_StridedUnaryOp *stransfer_decsrcref; - NpyAuxData *data_decsrcref; - PyArray_StridedUnaryOp *stransfer_decdstref; - NpyAuxData *data_decdstref; + NPY_cast_info wrapped; + NPY_cast_info decref_src; + NPY_cast_info decref_dst; /* The use-case should probably be deprecated */ + npy_intp src_N, dst_N; /* This gets a run-length encoded representation of the transfer */ npy_intp run_count; - _subarray_broadcast_offsetrun offsetruns; + _subarray_broadcast_offsetrun offsetruns[]; } _subarray_broadcast_data; @@ -2066,49 +1748,48 @@ typedef struct { static void _subarray_broadcast_data_free(NpyAuxData *data) { _subarray_broadcast_data *d = (_subarray_broadcast_data *)data; - NPY_AUXDATA_FREE(d->data); - NPY_AUXDATA_FREE(d->data_decsrcref); - NPY_AUXDATA_FREE(d->data_decdstref); - PyArray_free(data); + NPY_cast_info_xfree(&d->wrapped); + NPY_cast_info_xfree(&d->decref_src); + NPY_cast_info_xfree(&d->decref_dst); + PyMem_Free(data); } /* transfer data copy function */ -static NpyAuxData *_subarray_broadcast_data_clone( NpyAuxData *data) +static NpyAuxData *_subarray_broadcast_data_clone(NpyAuxData *data) { _subarray_broadcast_data *d = (_subarray_broadcast_data *)data; - _subarray_broadcast_data *newdata; - npy_intp run_count = d->run_count, structsize; - structsize = sizeof(_subarray_broadcast_data) + - run_count*sizeof(_subarray_broadcast_offsetrun); + npy_intp offsetruns_size = d->run_count*sizeof(_subarray_broadcast_offsetrun); + npy_intp structsize = sizeof(_subarray_broadcast_data) + offsetruns_size; /* Allocate the data and populate it */ - newdata = (_subarray_broadcast_data *)PyArray_malloc(structsize); + _subarray_broadcast_data *newdata = PyMem_Malloc(structsize); if (newdata == NULL) { return NULL; } - memcpy(newdata, data, structsize); - if (d->data != NULL) { - newdata->data = NPY_AUXDATA_CLONE(d->data); - if (newdata->data == NULL) { - PyArray_free(newdata); - return NULL; - } + newdata->base.free = &_subarray_broadcast_data_free; + newdata->base.clone = &_subarray_broadcast_data_clone; + newdata->src_N = d->src_N; + newdata->dst_N = d->dst_N; + newdata->run_count = d->run_count; + memcpy(newdata->offsetruns, d->offsetruns, offsetruns_size); + + NPY_cast_info_init(&newdata->decref_src); + NPY_cast_info_init(&newdata->decref_dst); + + if (NPY_cast_info_copy(&newdata->wrapped, &d->wrapped) < 0) { + _subarray_broadcast_data_free((NpyAuxData *)newdata); + return NULL; } - if (d->data_decsrcref != NULL) { - newdata->data_decsrcref = NPY_AUXDATA_CLONE(d->data_decsrcref); - if (newdata->data_decsrcref == NULL) { - NPY_AUXDATA_FREE(newdata->data); - PyArray_free(newdata); + if (d->decref_src.func != NULL) { + if (NPY_cast_info_copy(&newdata->decref_src, &d->decref_src) < 0) { + _subarray_broadcast_data_free((NpyAuxData *) newdata); return NULL; } } - if (d->data_decdstref != NULL) { - newdata->data_decdstref = NPY_AUXDATA_CLONE(d->data_decdstref); - if (newdata->data_decdstref == NULL) { - NPY_AUXDATA_FREE(newdata->data); - NPY_AUXDATA_FREE(newdata->data_decsrcref); - PyArray_free(newdata); + if (d->decref_dst.func != NULL) { + if (NPY_cast_info_copy(&newdata->decref_dst, &d->decref_dst) < 0) { + _subarray_broadcast_data_free((NpyAuxData *) newdata); return NULL; } } @@ -2117,31 +1798,34 @@ static NpyAuxData *_subarray_broadcast_data_clone( NpyAuxData *data) } static int -_strided_to_strided_subarray_broadcast(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) +_strided_to_strided_subarray_broadcast( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _subarray_broadcast_data *d = (_subarray_broadcast_data *)data; - PyArray_StridedUnaryOp *subtransfer = d->stransfer; - NpyAuxData *subdata = d->data; - npy_intp run, run_count = d->run_count, - src_subitemsize = d->src_itemsize, - dst_subitemsize = d->dst_itemsize; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _subarray_broadcast_data *d = (_subarray_broadcast_data *)auxdata; + npy_intp run, run_count = d->run_count; npy_intp loop_index, offset, count; - char *dst_ptr; - _subarray_broadcast_offsetrun *offsetruns = &d->offsetruns; + + npy_intp src_subitemsize = d->wrapped.descriptors[0]->elsize; + npy_intp dst_subitemsize = d->wrapped.descriptors[1]->elsize; + + npy_intp sub_strides[2] = {src_subitemsize, dst_subitemsize}; while (N > 0) { loop_index = 0; for (run = 0; run < run_count; ++run) { - offset = offsetruns[run].offset; - count = offsetruns[run].count; - dst_ptr = dst + loop_index*dst_subitemsize; + offset = d->offsetruns[run].offset; + count = d->offsetruns[run].count; + char *dst_ptr = dst + loop_index*dst_subitemsize; + char *sub_args[2] = {src + offset, dst_ptr}; if (offset != -1) { - if (subtransfer( - dst_ptr, dst_subitemsize, src + offset, src_subitemsize, - count, src_subitemsize, subdata) < 0) { + if (d->wrapped.func(&d->wrapped.context, + sub_args, &count, sub_strides, d->wrapped.auxdata) < 0) { return -1; } } @@ -2160,44 +1844,42 @@ _strided_to_strided_subarray_broadcast(char *dst, npy_intp dst_stride, static int -_strided_to_strided_subarray_broadcast_withrefs(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) +_strided_to_strided_subarray_broadcast_withrefs( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _subarray_broadcast_data *d = (_subarray_broadcast_data *)data; - PyArray_StridedUnaryOp *subtransfer = d->stransfer; - NpyAuxData *subdata = d->data; - PyArray_StridedUnaryOp *stransfer_decsrcref = d->stransfer_decsrcref; - NpyAuxData *data_decsrcref = d->data_decsrcref; - PyArray_StridedUnaryOp *stransfer_decdstref = d->stransfer_decdstref; - NpyAuxData *data_decdstref = d->data_decdstref; - npy_intp run, run_count = d->run_count, - src_subitemsize = d->src_itemsize, - dst_subitemsize = d->dst_itemsize, - src_subN = d->src_N; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _subarray_broadcast_data *d = (_subarray_broadcast_data *)auxdata; + npy_intp run, run_count = d->run_count; npy_intp loop_index, offset, count; - char *dst_ptr; - _subarray_broadcast_offsetrun *offsetruns = &d->offsetruns; + + npy_intp src_subitemsize = d->wrapped.descriptors[0]->elsize; + npy_intp dst_subitemsize = d->wrapped.descriptors[1]->elsize; + + npy_intp sub_strides[2] = {src_subitemsize, dst_subitemsize}; while (N > 0) { loop_index = 0; for (run = 0; run < run_count; ++run) { - offset = offsetruns[run].offset; - count = offsetruns[run].count; - dst_ptr = dst + loop_index*dst_subitemsize; + offset = d->offsetruns[run].offset; + count = d->offsetruns[run].count; + char *dst_ptr = dst + loop_index*dst_subitemsize; + char *sub_args[2] = {src + offset, dst_ptr}; if (offset != -1) { - if (subtransfer( - dst_ptr, dst_subitemsize, src + offset, src_subitemsize, - count, src_subitemsize, subdata) < 0) { + if (d->wrapped.func(&d->wrapped.context, + sub_args, &count, sub_strides, d->wrapped.auxdata) < 0) { return -1; } } else { - if (stransfer_decdstref != NULL) { - if (stransfer_decdstref( - NULL, 0, dst_ptr, dst_subitemsize, - count, dst_subitemsize, data_decdstref) < 0) { + if (d->decref_dst.func != NULL) { + if (d->decref_dst.func(&d->decref_dst.context, + &dst_ptr, &count, &dst_subitemsize, + d->decref_dst.auxdata) < 0) { return -1; } } @@ -2206,10 +1888,10 @@ _strided_to_strided_subarray_broadcast_withrefs(char *dst, npy_intp dst_stride, loop_index += count; } - if (stransfer_decsrcref != NULL) { - if (stransfer_decsrcref( - NULL, 0, src, src_subitemsize, - src_subN, src_subitemsize, data_decsrcref) < 0) { + if (d->decref_src.func != NULL) { + if (d->decref_src.func(&d->decref_src.context, + &src, &d->src_N, &src_subitemsize, + d->decref_src.auxdata) < 0) { return -1; } } @@ -2229,24 +1911,30 @@ get_subarray_broadcast_transfer_function(int aligned, npy_intp src_size, npy_intp dst_size, PyArray_Dims src_shape, PyArray_Dims dst_shape, int move_references, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api) { _subarray_broadcast_data *data; npy_intp structsize, loop_index, run, run_size, src_index, dst_index, i, ndim; - _subarray_broadcast_offsetrun *offsetruns; structsize = sizeof(_subarray_broadcast_data) + dst_size*sizeof(_subarray_broadcast_offsetrun); /* Allocate the data and populate it */ - data = (_subarray_broadcast_data *)PyArray_malloc(structsize); + data = (_subarray_broadcast_data *)PyMem_Malloc(structsize); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; } + data->base.free = &_subarray_broadcast_data_free; + data->base.clone = &_subarray_broadcast_data_clone; + data->src_N = src_size; + data->dst_N = dst_size; + + NPY_cast_info_init(&data->decref_src); + NPY_cast_info_init(&data->decref_dst); /* * move_references is set to 0, handled in the wrapping transfer fn, @@ -2257,17 +1945,11 @@ get_subarray_broadcast_transfer_function(int aligned, src_dtype->elsize, dst_dtype->elsize, src_dtype, dst_dtype, 0, - &data->stransfer, &data->data, + &data->wrapped, out_needs_api) != NPY_SUCCEED) { - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } - data->base.free = &_subarray_broadcast_data_free; - data->base.clone = &_subarray_broadcast_data_clone; - data->src_N = src_size; - data->dst_N = dst_size; - data->src_itemsize = src_dtype->elsize; - data->dst_itemsize = dst_dtype->elsize; /* If the src object will need a DECREF */ if (move_references && PyDataType_REFCHK(src_dtype)) { @@ -2275,18 +1957,12 @@ get_subarray_broadcast_transfer_function(int aligned, src_dtype->elsize, 0, src_dtype, NULL, 1, - &data->stransfer_decsrcref, - &data->data_decsrcref, + &data->decref_src, out_needs_api) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(data->data); - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } } - else { - data->stransfer_decsrcref = NULL; - data->data_decsrcref = NULL; - } /* If the dst object needs a DECREF to set it to NULL */ if (PyDataType_REFCHK(dst_dtype)) { @@ -2294,22 +1970,15 @@ get_subarray_broadcast_transfer_function(int aligned, dst_dtype->elsize, 0, dst_dtype, NULL, 1, - &data->stransfer_decdstref, - &data->data_decdstref, + &data->decref_dst, out_needs_api) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(data->data); - NPY_AUXDATA_FREE(data->data_decsrcref); - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } } - else { - data->stransfer_decdstref = NULL; - data->data_decdstref = NULL; - } /* Calculate the broadcasting and set the offsets */ - offsetruns = &data->offsetruns; + _subarray_broadcast_offsetrun *offsetruns = data->offsetruns; ndim = (src_shape.len > dst_shape.len) ? src_shape.len : dst_shape.len; for (loop_index = 0; loop_index < dst_size; ++loop_index) { npy_intp src_factor = 1; @@ -2395,8 +2064,8 @@ get_subarray_broadcast_transfer_function(int aligned, } } - if (data->stransfer_decsrcref == NULL && - data->stransfer_decdstref == NULL) { + if (data->decref_src.func == NULL && + data->decref_dst.func == NULL) { *out_stransfer = &_strided_to_strided_subarray_broadcast; } else { @@ -2411,12 +2080,12 @@ get_subarray_broadcast_transfer_function(int aligned, * Handles subarray transfer. To call this, at least one of the dtype's * subarrays must be non-NULL */ -static int +NPY_NO_EXPORT int get_subarray_transfer_function(int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int move_references, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api) { @@ -2447,46 +2116,36 @@ get_subarray_transfer_function(int aligned, } /* - * Just a straight one-element copy. + * Copy the src value to all the dst values, the size one can be + * special cased for speed. */ - if (dst_size == 1 && src_size == 1) { - npy_free_cache_dim_obj(src_shape); - npy_free_cache_dim_obj(dst_shape); + if ((dst_size == 1 && src_size == 1) || ( + src_shape.len == dst_shape.len && PyArray_CompareLists( + src_shape.ptr, dst_shape.ptr, src_shape.len))) { - return PyArray_GetDTypeTransferFunction(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - out_stransfer, out_transferdata, - out_needs_api); - } - /* Copy the src value to all the dst values */ - else if (src_size == 1) { npy_free_cache_dim_obj(src_shape); npy_free_cache_dim_obj(dst_shape); - return get_one_to_n_transfer_function(aligned, + return get_n_to_n_transfer_function(aligned, src_stride, dst_stride, src_dtype, dst_dtype, move_references, - dst_size, + src_size, out_stransfer, out_transferdata, out_needs_api); } - /* If the shapes match exactly, do an n to n copy */ - else if (src_shape.len == dst_shape.len && - PyArray_CompareLists(src_shape.ptr, dst_shape.ptr, - src_shape.len)) { + /* Copy the src value to all the dst values */ + else if (src_size == 1) { npy_free_cache_dim_obj(src_shape); npy_free_cache_dim_obj(dst_shape); - return get_n_to_n_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - src_size, - out_stransfer, out_transferdata, - out_needs_api); + return get_one_to_n_transfer_function(aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + move_references, + dst_size, + out_stransfer, out_transferdata, + out_needs_api); } /* * Copy the subarray with broadcasting, truncating, and zero-padding @@ -2510,91 +2169,82 @@ get_subarray_transfer_function(int aligned, /**************************** COPY FIELDS *******************************/ typedef struct { - npy_intp src_offset, dst_offset, src_itemsize; - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *data; + npy_intp src_offset, dst_offset; + NPY_cast_info info; } _single_field_transfer; typedef struct { NpyAuxData base; npy_intp field_count; - - _single_field_transfer fields; + _single_field_transfer fields[]; } _field_transfer_data; + /* transfer data free function */ static void _field_transfer_data_free(NpyAuxData *data) { _field_transfer_data *d = (_field_transfer_data *)data; - npy_intp i, field_count; - _single_field_transfer *fields; - - field_count = d->field_count; - fields = &d->fields; - for (i = 0; i < field_count; ++i) { - NPY_AUXDATA_FREE(fields[i].data); + for (npy_intp i = 0; i < d->field_count; ++i) { + NPY_cast_info_xfree(&d->fields[i].info); } - PyArray_free(d); + PyMem_Free(d); } /* transfer data copy function */ static NpyAuxData *_field_transfer_data_clone(NpyAuxData *data) { _field_transfer_data *d = (_field_transfer_data *)data; - _field_transfer_data *newdata; - npy_intp i, field_count = d->field_count, structsize; - _single_field_transfer *fields, *newfields; - structsize = sizeof(_field_transfer_data) + + npy_intp field_count = d->field_count; + npy_intp structsize = sizeof(_field_transfer_data) + field_count * sizeof(_single_field_transfer); /* Allocate the data and populate it */ - newdata = (_field_transfer_data *)PyArray_malloc(structsize); + _field_transfer_data *newdata = PyMem_Malloc(structsize); if (newdata == NULL) { return NULL; } - memcpy(newdata, d, structsize); + newdata->base = d->base; + newdata->field_count = 0; + /* Copy all the fields transfer data */ - fields = &d->fields; - newfields = &newdata->fields; - for (i = 0; i < field_count; ++i) { - if (fields[i].data != NULL) { - newfields[i].data = NPY_AUXDATA_CLONE(fields[i].data); - if (newfields[i].data == NULL) { - for (i = i-1; i >= 0; --i) { - NPY_AUXDATA_FREE(newfields[i].data); - } - PyArray_free(newdata); - return NULL; - } + for (npy_intp i = 0; i < field_count; ++i) { + if (NPY_cast_info_copy(&newdata->fields[i].info, &d->fields[i].info) < 0) { + NPY_AUXDATA_FREE((NpyAuxData *)newdata); + return NULL; } - + newdata->fields[i].src_offset = d->fields[i].src_offset; + newdata->fields[i].dst_offset = d->fields[i].dst_offset; + newdata->field_count++; } return (NpyAuxData *)newdata; } + static int -_strided_to_strided_field_transfer(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) +_strided_to_strided_field_transfer( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) { - _field_transfer_data *d = (_field_transfer_data *)data; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + _field_transfer_data *d = (_field_transfer_data *)auxdata; npy_intp i, field_count = d->field_count; - _single_field_transfer *field; + const npy_intp blocksize = NPY_LOWLEVEL_BUFFER_BLOCKSIZE; /* Do the transfer a block at a time */ for (;;) { - field = &d->fields; - if (N > NPY_LOWLEVEL_BUFFER_BLOCKSIZE) { - for (i = 0; i < field_count; ++i, ++field) { - if (field->stransfer( - dst + field->dst_offset, dst_stride, - src + field->src_offset, src_stride, - NPY_LOWLEVEL_BUFFER_BLOCKSIZE, - field->src_itemsize, field->data) < 0) { + if (N > blocksize) { + for (i = 0; i < field_count; ++i) { + _single_field_transfer field = d->fields[i]; + char *fargs[2] = {src + field.src_offset, dst + field.dst_offset}; + if (field.info.func(&field.info.context, + fargs, &blocksize, strides, field.info.auxdata) < 0) { return -1; } } @@ -2603,12 +2253,11 @@ _strided_to_strided_field_transfer(char *dst, npy_intp dst_stride, dst += NPY_LOWLEVEL_BUFFER_BLOCKSIZE*dst_stride; } else { - for (i = 0; i < field_count; ++i, ++field) { - if (field->stransfer( - dst + field->dst_offset, dst_stride, - src + field->src_offset, src_stride, - N, - field->src_itemsize, field->data) < 0) { + for (i = 0; i < field_count; ++i) { + _single_field_transfer field = d->fields[i]; + char *fargs[2] = {src + field.src_offset, dst + field.dst_offset}; + if (field.info.func(&field.info.context, + fargs, &N, strides, field.info.auxdata) < 0) { return -1; } } @@ -2621,22 +2270,22 @@ _strided_to_strided_field_transfer(char *dst, npy_intp dst_stride, * Handles fields transfer. To call this, at least one of the dtypes * must have fields. Does not take care of object<->structure conversion */ -static int -get_fields_transfer_function(int aligned, +NPY_NO_EXPORT int +get_fields_transfer_function(int NPY_UNUSED(aligned), npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int move_references, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api) { PyObject *key, *tup, *title; PyArray_Descr *src_fld_dtype, *dst_fld_dtype; - npy_int i, field_count, structsize; + npy_int i; + size_t structsize; + Py_ssize_t field_count; int src_offset, dst_offset; _field_transfer_data *data; - _single_field_transfer *fields; - int failed = 0; /* * There are three cases to take care of: 1. src is non-structured, @@ -2650,64 +2299,55 @@ get_fields_transfer_function(int aligned, /* Allocate the field-data structure and populate it */ structsize = sizeof(_field_transfer_data) + (field_count + 1) * sizeof(_single_field_transfer); - data = (_field_transfer_data *)PyArray_malloc(structsize); + data = PyMem_Malloc(structsize); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; } data->base.free = &_field_transfer_data_free; data->base.clone = &_field_transfer_data_clone; - fields = &data->fields; + data->field_count = 0; for (i = 0; i < field_count; ++i) { key = PyTuple_GET_ITEM(dst_dtype->names, i); tup = PyDict_GetItem(dst_dtype->fields, key); if (!PyArg_ParseTuple(tup, "Oi|O", &dst_fld_dtype, &dst_offset, &title)) { - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } if (PyArray_GetDTypeTransferFunction(0, src_stride, dst_stride, src_dtype, dst_fld_dtype, 0, - &fields[i].stransfer, - &fields[i].data, + &data->fields[i].info, out_needs_api) != NPY_SUCCEED) { - for (i = i-1; i >= 0; --i) { - NPY_AUXDATA_FREE(fields[i].data); - } - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } - fields[i].src_offset = 0; - fields[i].dst_offset = dst_offset; - fields[i].src_itemsize = src_dtype->elsize; + data->fields[i].src_offset = 0; + data->fields[i].dst_offset = dst_offset; + data->field_count++; } /* - * If references should be decrefd in src, add - * another transfer function to do that. + * If references should be decrefd in src, add another transfer + * function to do that. Since a decref function only uses a single + * input, the second one (normally output) just does not matter here. */ if (move_references && PyDataType_REFCHK(src_dtype)) { - if (get_decsrcref_transfer_function(0, + if (get_decref_transfer_function(0, src_stride, src_dtype, - &fields[field_count].stransfer, - &fields[field_count].data, + &data->fields[field_count].info, out_needs_api) != NPY_SUCCEED) { - for (i = 0; i < field_count; ++i) { - NPY_AUXDATA_FREE(fields[i].data); - } - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } - fields[field_count].src_offset = 0; - fields[field_count].dst_offset = 0; - fields[field_count].src_itemsize = src_dtype->elsize; - field_count++; + data->fields[field_count].src_offset = 0; + data->fields[field_count].dst_offset = 0; + data->field_count = field_count; } - data->field_count = field_count; *out_stransfer = &_strided_to_strided_field_transfer; *out_transferdata = (NpyAuxData *)data; @@ -2727,19 +2367,19 @@ get_fields_transfer_function(int aligned, /* Allocate the field-data structure and populate it */ structsize = sizeof(_field_transfer_data) + 1 * sizeof(_single_field_transfer); - data = (_field_transfer_data *)PyArray_malloc(structsize); + data = PyMem_Malloc(structsize); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; } data->base.free = &_field_transfer_data_free; data->base.clone = &_field_transfer_data_clone; - fields = &data->fields; key = PyTuple_GET_ITEM(src_dtype->names, 0); tup = PyDict_GetItem(src_dtype->fields, key); if (!PyArg_ParseTuple(tup, "Oi|O", &src_fld_dtype, &src_offset, &title)) { + PyMem_Free(data); return NPY_FAIL; } @@ -2747,16 +2387,13 @@ get_fields_transfer_function(int aligned, src_stride, dst_stride, src_fld_dtype, dst_dtype, move_references, - &fields[0].stransfer, - &fields[0].data, + &data->fields[0].info, out_needs_api) != NPY_SUCCEED) { - PyArray_free(data); + PyMem_Free(data); return NPY_FAIL; } - fields[0].src_offset = src_offset; - fields[0].dst_offset = 0; - fields[0].src_itemsize = src_fld_dtype->elsize; - + data->fields[0].src_offset = src_offset; + data->fields[0].dst_offset = 0; data->field_count = 1; *out_stransfer = &_strided_to_strided_field_transfer; @@ -2777,14 +2414,14 @@ get_fields_transfer_function(int aligned, /* Allocate the field-data structure and populate it */ structsize = sizeof(_field_transfer_data) + field_count * sizeof(_single_field_transfer); - data = (_field_transfer_data *)PyArray_malloc(structsize); + data = PyMem_Malloc(structsize); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; } data->base.free = &_field_transfer_data_free; data->base.clone = &_field_transfer_data_clone; - fields = &data->fields; + data->field_count = 0; /* set up the transfer function for each field */ for (i = 0; i < field_count; ++i) { @@ -2792,42 +2429,31 @@ get_fields_transfer_function(int aligned, tup = PyDict_GetItem(dst_dtype->fields, key); if (!PyArg_ParseTuple(tup, "Oi|O", &dst_fld_dtype, &dst_offset, &title)) { - failed = 1; - break; + NPY_AUXDATA_FREE((NpyAuxData *)data); + return NPY_FAIL; } key = PyTuple_GET_ITEM(src_dtype->names, i); tup = PyDict_GetItem(src_dtype->fields, key); if (!PyArg_ParseTuple(tup, "Oi|O", &src_fld_dtype, &src_offset, &title)) { - failed = 1; - break; + NPY_AUXDATA_FREE((NpyAuxData *)data); + return NPY_FAIL; } if (PyArray_GetDTypeTransferFunction(0, src_stride, dst_stride, src_fld_dtype, dst_fld_dtype, move_references, - &fields[i].stransfer, - &fields[i].data, + &data->fields[i].info, out_needs_api) != NPY_SUCCEED) { - failed = 1; - break; - } - fields[i].src_offset = src_offset; - fields[i].dst_offset = dst_offset; - fields[i].src_itemsize = src_fld_dtype->elsize; - } - - if (failed) { - for (i = i-1; i >= 0; --i) { - NPY_AUXDATA_FREE(fields[i].data); + NPY_AUXDATA_FREE((NpyAuxData *)data); + return NPY_FAIL; } - PyArray_free(data); - return NPY_FAIL; + data->fields[i].src_offset = src_offset; + data->fields[i].dst_offset = dst_offset; + data->field_count++; } - data->field_count = field_count; - *out_stransfer = &_strided_to_strided_field_transfer; *out_transferdata = (NpyAuxData *)data; @@ -2835,190 +2461,110 @@ get_fields_transfer_function(int aligned, } static int -get_decsrcref_fields_transfer_function(int aligned, +get_decref_fields_transfer_function(int NPY_UNUSED(aligned), npy_intp src_stride, PyArray_Descr *src_dtype, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, int *out_needs_api) { PyObject *names, *key, *tup, *title; PyArray_Descr *src_fld_dtype; - npy_int i, names_size, field_count, structsize; + npy_int i, structsize; + Py_ssize_t field_count; int src_offset; - _field_transfer_data *data; - _single_field_transfer *fields; names = src_dtype->names; - names_size = PyTuple_GET_SIZE(src_dtype->names); + field_count = PyTuple_GET_SIZE(src_dtype->names); - field_count = names_size; + /* Over-allocating here: less fields may be used */ structsize = sizeof(_field_transfer_data) + field_count * sizeof(_single_field_transfer); /* Allocate the data and populate it */ - data = (_field_transfer_data *)PyArray_malloc(structsize); + _field_transfer_data *data = PyMem_Malloc(structsize); if (data == NULL) { PyErr_NoMemory(); return NPY_FAIL; } data->base.free = &_field_transfer_data_free; data->base.clone = &_field_transfer_data_clone; - fields = &data->fields; + data->field_count = 0; - field_count = 0; - for (i = 0; i < names_size; ++i) { + _single_field_transfer *field = data->fields; + for (i = 0; i < field_count; ++i) { key = PyTuple_GET_ITEM(names, i); tup = PyDict_GetItem(src_dtype->fields, key); if (!PyArg_ParseTuple(tup, "Oi|O", &src_fld_dtype, &src_offset, &title)) { - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } if (PyDataType_REFCHK(src_fld_dtype)) { if (out_needs_api) { *out_needs_api = 1; } - if (get_decsrcref_transfer_function(0, + if (get_decref_transfer_function(0, src_stride, src_fld_dtype, - &fields[field_count].stransfer, - &fields[field_count].data, + &field->info, out_needs_api) != NPY_SUCCEED) { - for (i = field_count-1; i >= 0; --i) { - NPY_AUXDATA_FREE(fields[i].data); - } - PyArray_free(data); + NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } - fields[field_count].src_offset = src_offset; - fields[field_count].dst_offset = 0; - fields[field_count].src_itemsize = src_dtype->elsize; - field_count++; + field->src_offset = src_offset; + data->field_count++; + field++; } } - data->field_count = field_count; - *out_stransfer = &_strided_to_strided_field_transfer; *out_transferdata = (NpyAuxData *)data; return NPY_SUCCEED; } -static int -get_setdestzero_fields_transfer_function(int aligned, - npy_intp dst_stride, - PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api) -{ - PyObject *names, *key, *tup, *title; - PyArray_Descr *dst_fld_dtype; - npy_int i, names_size, field_count, structsize; - int dst_offset; - _field_transfer_data *data; - _single_field_transfer *fields; - - names = dst_dtype->names; - names_size = PyTuple_GET_SIZE(dst_dtype->names); - - field_count = names_size; - structsize = sizeof(_field_transfer_data) + - field_count * sizeof(_single_field_transfer); - /* Allocate the data and populate it */ - data = (_field_transfer_data *)PyArray_malloc(structsize); - if (data == NULL) { - PyErr_NoMemory(); - return NPY_FAIL; - } - data->base.free = &_field_transfer_data_free; - data->base.clone = &_field_transfer_data_clone; - fields = &data->fields; - - for (i = 0; i < names_size; ++i) { - key = PyTuple_GET_ITEM(names, i); - tup = PyDict_GetItem(dst_dtype->fields, key); - if (!PyArg_ParseTuple(tup, "Oi|O", &dst_fld_dtype, - &dst_offset, &title)) { - PyArray_free(data); - return NPY_FAIL; - } - if (get_setdstzero_transfer_function(0, - dst_stride, - dst_fld_dtype, - &fields[i].stransfer, - &fields[i].data, - out_needs_api) != NPY_SUCCEED) { - for (i = i-1; i >= 0; --i) { - NPY_AUXDATA_FREE(fields[i].data); - } - PyArray_free(data); - return NPY_FAIL; - } - fields[i].src_offset = 0; - fields[i].dst_offset = dst_offset; - fields[i].src_itemsize = 0; - } - - data->field_count = field_count; - - *out_stransfer = &_strided_to_strided_field_transfer; - *out_transferdata = (NpyAuxData *)data; - - return NPY_SUCCEED; -} /************************* MASKED TRANSFER WRAPPER *************************/ typedef struct { NpyAuxData base; - /* The transfer function being wrapped */ - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *transferdata; - + /* The transfer function being wrapped (could likely be stored directly) */ + NPY_cast_info wrapped; /* The src decref function if necessary */ - PyArray_StridedUnaryOp *decsrcref_stransfer; - NpyAuxData *decsrcref_transferdata; + NPY_cast_info decref_src; } _masked_wrapper_transfer_data; /* transfer data free function */ -static void _masked_wrapper_transfer_data_free(NpyAuxData *data) +static void +_masked_wrapper_transfer_data_free(NpyAuxData *data) { _masked_wrapper_transfer_data *d = (_masked_wrapper_transfer_data *)data; - NPY_AUXDATA_FREE(d->transferdata); - NPY_AUXDATA_FREE(d->decsrcref_transferdata); - PyArray_free(data); + NPY_cast_info_xfree(&d->wrapped); + NPY_cast_info_xfree(&d->decref_src); + PyMem_Free(data); } /* transfer data copy function */ -static NpyAuxData *_masked_wrapper_transfer_data_clone(NpyAuxData *data) +static NpyAuxData * +_masked_wrapper_transfer_data_clone(NpyAuxData *data) { _masked_wrapper_transfer_data *d = (_masked_wrapper_transfer_data *)data; _masked_wrapper_transfer_data *newdata; /* Allocate the data and populate it */ - newdata = (_masked_wrapper_transfer_data *)PyArray_malloc( - sizeof(_masked_wrapper_transfer_data)); + newdata = PyMem_Malloc(sizeof(*newdata)); if (newdata == NULL) { return NULL; } - memcpy(newdata, d, sizeof(_masked_wrapper_transfer_data)); + newdata->base = d->base; - /* Clone all the owned auxdata as well */ - if (newdata->transferdata != NULL) { - newdata->transferdata = NPY_AUXDATA_CLONE(newdata->transferdata); - if (newdata->transferdata == NULL) { - PyArray_free(newdata); - return NULL; - } + if (NPY_cast_info_copy(&newdata->wrapped, &d->wrapped) < 0) { + PyMem_Free(newdata); + return NULL; } - if (newdata->decsrcref_transferdata != NULL) { - newdata->decsrcref_transferdata = - NPY_AUXDATA_CLONE(newdata->decsrcref_transferdata); - if (newdata->decsrcref_transferdata == NULL) { - NPY_AUXDATA_FREE(newdata->transferdata); - PyArray_free(newdata); + if (d->decref_src.func != NULL) { + if (NPY_cast_info_copy(&newdata->decref_src, &d->decref_src) < 0) { + NPY_AUXDATA_FREE((NpyAuxData *)newdata); return NULL; } } @@ -3027,31 +2573,25 @@ static NpyAuxData *_masked_wrapper_transfer_data_clone(NpyAuxData *data) } static int -_strided_masked_wrapper_decsrcref_transfer_function( - char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_bool *mask, npy_intp mask_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *transferdata) +_strided_masked_wrapper_decref_transfer_function( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + npy_bool *mask, npy_intp mask_stride, + NpyAuxData *auxdata) { - _masked_wrapper_transfer_data *d = - (_masked_wrapper_transfer_data *)transferdata; - npy_intp subloopsize; - PyArray_StridedUnaryOp *unmasked_stransfer, *decsrcref_stransfer; - NpyAuxData *unmasked_transferdata, *decsrcref_transferdata; + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; - unmasked_stransfer = d->stransfer; - unmasked_transferdata = d->transferdata; - decsrcref_stransfer = d->decsrcref_stransfer; - decsrcref_transferdata = d->decsrcref_transferdata; + _masked_wrapper_transfer_data *d = (_masked_wrapper_transfer_data *)auxdata; + npy_intp subloopsize; while (N > 0) { - /* Skip masked values, still calling decsrcref for move_references */ + /* Skip masked values, still calling decref for move_references */ mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N, &subloopsize, 1); - if (decsrcref_stransfer( - NULL, 0, src, src_stride, - subloopsize, src_itemsize, decsrcref_transferdata) < 0) { + if (d->decref_src.func(&d->decref_src.context, + &src, &subloopsize, &src_stride, d->decref_src.auxdata) < 0) { return -1; } dst += subloopsize * dst_stride; @@ -3064,9 +2604,9 @@ _strided_masked_wrapper_decsrcref_transfer_function( /* Process unmasked values */ mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N, &subloopsize, 0); - if (unmasked_stransfer( - dst, dst_stride, src, src_stride, - subloopsize, src_itemsize, unmasked_transferdata) < 0) { + char *wrapped_args[2] = {src, dst}; + if (d->wrapped.func(&d->wrapped.context, + wrapped_args, &subloopsize, strides, d->wrapped.auxdata) < 0) { return -1; } dst += subloopsize * dst_stride; @@ -3078,21 +2618,17 @@ _strided_masked_wrapper_decsrcref_transfer_function( static int _strided_masked_wrapper_transfer_function( - char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_bool *mask, npy_intp mask_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *transferdata) + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + npy_bool *mask, npy_intp mask_stride, + NpyAuxData *auxdata) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; - _masked_wrapper_transfer_data *d = - (_masked_wrapper_transfer_data *)transferdata; + _masked_wrapper_transfer_data *d = (_masked_wrapper_transfer_data *)auxdata; npy_intp subloopsize; - PyArray_StridedUnaryOp *unmasked_stransfer; - NpyAuxData *unmasked_transferdata; - - unmasked_stransfer = d->stransfer; - unmasked_transferdata = d->transferdata; while (N > 0) { /* Skip masked values */ @@ -3108,9 +2644,9 @@ _strided_masked_wrapper_transfer_function( /* Process unmasked values */ mask = (npy_bool*)npy_memchr((char *)mask, 0, mask_stride, N, &subloopsize, 0); - if (unmasked_stransfer( - dst, dst_stride, src, src_stride, - subloopsize, src_itemsize, unmasked_transferdata) < 0) { + char *wrapped_args[2] = {src, dst}; + if (d->wrapped.func(&d->wrapped.context, + wrapped_args, &subloopsize, strides, d->wrapped.auxdata) < 0) { return -1; } dst += subloopsize * dst_stride; @@ -3121,249 +2657,28 @@ _strided_masked_wrapper_transfer_function( } -/************************* DEST BOOL SETONE *******************************/ - -static int -_null_to_strided_set_bool_one(char *dst, - npy_intp dst_stride, - char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) -{ - /* bool type is one byte, so can just use the char */ - - while (N > 0) { - *dst = 1; - - dst += dst_stride; - --N; - } - return 0; -} - -static int -_null_to_contig_set_bool_one(char *dst, - npy_intp NPY_UNUSED(dst_stride), - char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) -{ - /* bool type is one byte, so can just use the char */ - - memset(dst, 1, N); - return 0; -} - -/* Only for the bool type, sets the destination to 1 */ -NPY_NO_EXPORT int -get_bool_setdstone_transfer_function(npy_intp dst_stride, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *NPY_UNUSED(out_needs_api)) -{ - if (dst_stride == 1) { - *out_stransfer = &_null_to_contig_set_bool_one; - } - else { - *out_stransfer = &_null_to_strided_set_bool_one; - } - *out_transferdata = NULL; - - return NPY_SUCCEED; -} - -/*************************** DEST SETZERO *******************************/ - -/* Sets dest to zero */ -typedef struct { - NpyAuxData base; - npy_intp dst_itemsize; -} _dst_memset_zero_data; - -/* zero-padded data copy function */ -static NpyAuxData *_dst_memset_zero_data_clone(NpyAuxData *data) -{ - _dst_memset_zero_data *newdata = - (_dst_memset_zero_data *)PyArray_malloc( - sizeof(_dst_memset_zero_data)); - if (newdata == NULL) { - return NULL; - } - - memcpy(newdata, data, sizeof(_dst_memset_zero_data)); - - return (NpyAuxData *)newdata; -} - -static int -_null_to_strided_memset_zero(char *dst, - npy_intp dst_stride, - char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) -{ - _dst_memset_zero_data *d = (_dst_memset_zero_data *)data; - npy_intp dst_itemsize = d->dst_itemsize; - - while (N > 0) { - memset(dst, 0, dst_itemsize); - dst += dst_stride; - --N; - } - return 0; -} - -static int -_null_to_contig_memset_zero(char *dst, - npy_intp dst_stride, - char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *data) -{ - _dst_memset_zero_data *d = (_dst_memset_zero_data *)data; - npy_intp dst_itemsize = d->dst_itemsize; - - memset(dst, 0, N*dst_itemsize); - return 0; -} - -static int -_null_to_strided_reference_setzero(char *dst, - npy_intp dst_stride, - char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) -{ - PyObject *dst_ref = NULL; - - while (N > 0) { - memcpy(&dst_ref, dst, sizeof(dst_ref)); - - /* Release the reference in dst and set it to NULL */ - NPY_DT_DBG_REFTRACE("dec dest ref (to set zero)", dst_ref); - Py_XDECREF(dst_ref); - memset(dst, 0, sizeof(PyObject *)); - - dst += dst_stride; - --N; - } - return 0; -} - -NPY_NO_EXPORT int -get_setdstzero_transfer_function(int aligned, - npy_intp dst_stride, - PyArray_Descr *dst_dtype, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api) -{ - _dst_memset_zero_data *data; - - /* If there are no references, just set the whole thing to zero */ - if (!PyDataType_REFCHK(dst_dtype)) { - data = (_dst_memset_zero_data *) - PyArray_malloc(sizeof(_dst_memset_zero_data)); - if (data == NULL) { - PyErr_NoMemory(); - return NPY_FAIL; - } - - data->base.free = (NpyAuxData_FreeFunc *)(&PyArray_free); - data->base.clone = &_dst_memset_zero_data_clone; - data->dst_itemsize = dst_dtype->elsize; - - if (dst_stride == data->dst_itemsize) { - *out_stransfer = &_null_to_contig_memset_zero; - } - else { - *out_stransfer = &_null_to_strided_memset_zero; - } - *out_transferdata = (NpyAuxData *)data; - } - /* If it's exactly one reference, use the decref function */ - else if (dst_dtype->type_num == NPY_OBJECT) { - if (out_needs_api) { - *out_needs_api = 1; - } - - *out_stransfer = &_null_to_strided_reference_setzero; - *out_transferdata = NULL; - } - /* If there are subarrays, need to wrap it */ - else if (PyDataType_HASSUBARRAY(dst_dtype)) { - PyArray_Dims dst_shape = {NULL, -1}; - npy_intp dst_size = 1; - PyArray_StridedUnaryOp *contig_stransfer; - NpyAuxData *contig_data; - - if (out_needs_api) { - *out_needs_api = 1; - } - - if (!(PyArray_IntpConverter(dst_dtype->subarray->shape, - &dst_shape))) { - PyErr_SetString(PyExc_ValueError, - "invalid subarray shape"); - return NPY_FAIL; - } - dst_size = PyArray_MultiplyList(dst_shape.ptr, dst_shape.len); - npy_free_cache_dim_obj(dst_shape); - - /* Get a function for contiguous dst of the subarray type */ - if (get_setdstzero_transfer_function(aligned, - dst_dtype->subarray->base->elsize, - dst_dtype->subarray->base, - &contig_stransfer, &contig_data, - out_needs_api) != NPY_SUCCEED) { - return NPY_FAIL; - } - - if (wrap_transfer_function_n_to_n(contig_stransfer, contig_data, - 0, dst_stride, - 0, dst_dtype->subarray->base->elsize, - dst_size, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(contig_data); - return NPY_FAIL; - } - } - /* If there are fields, need to do each field */ - else if (PyDataType_HASFIELDS(dst_dtype)) { - if (out_needs_api) { - *out_needs_api = 1; - } - - return get_setdestzero_fields_transfer_function(aligned, - dst_stride, dst_dtype, - out_stransfer, - out_transferdata, - out_needs_api); - } - - return NPY_SUCCEED; -} +/*************************** CLEAR SRC *******************************/ static int -_dec_src_ref_nop(char *NPY_UNUSED(dst), - npy_intp NPY_UNUSED(dst_stride), - char *NPY_UNUSED(src), npy_intp NPY_UNUSED(src_stride), - npy_intp NPY_UNUSED(N), - npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) +_dec_src_ref_nop( + PyArrayMethod_Context *NPY_UNUSED(context), + char *const *NPY_UNUSED(args), const npy_intp *NPY_UNUSED(dimensions), + const npy_intp *NPY_UNUSED(strides), NpyAuxData *NPY_UNUSED(auxdata)) { /* NOP */ return 0; } static int -_strided_to_null_dec_src_ref_reference(char *NPY_UNUSED(dst), - npy_intp NPY_UNUSED(dst_stride), - char *src, npy_intp src_stride, - npy_intp N, - npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) +_strided_to_null_dec_src_ref_reference( + PyArrayMethod_Context *NPY_UNUSED(context), + char *const *args, const npy_intp *dimensions, + const npy_intp *strides, NpyAuxData *NPY_UNUSED(auxdata)) { + char *src = args[0]; + npy_intp N = dimensions[0]; + npy_intp stride = strides[0]; + PyObject *src_ref = NULL; while (N > 0) { /* Release the reference in src and set it to NULL */ @@ -3372,27 +2687,38 @@ _strided_to_null_dec_src_ref_reference(char *NPY_UNUSED(dst), Py_XDECREF(src_ref); memset(src, 0, sizeof(PyObject *)); - src += src_stride; + src += stride; --N; } return 0; } -NPY_NO_EXPORT int -get_decsrcref_transfer_function(int aligned, +/* + * Get a function to decref. Currently, this uses a cast info slot, which + * means that the second (destination) descriptor is always set to NULL + * and generally does not have to be passed. + * Since we do not currently have an `ArrayMethod` representing this, the + * method is also set to NULL. + * + * TODO: this function should probably be moved onto the DType eventually, + * which would allow for user DTypes to include dynamic allocated + * memory or Python objects. + */ +static int +get_decref_transfer_function(int aligned, npy_intp src_stride, PyArray_Descr *src_dtype, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, + NPY_cast_info *cast_info, int *out_needs_api) { + NPY_cast_info_init(cast_info); + /* If there are no references, it's a nop */ if (!PyDataType_REFCHK(src_dtype)) { - *out_stransfer = &_dec_src_ref_nop; - *out_transferdata = NULL; - - return NPY_SUCCEED; + cast_info->func = &_dec_src_ref_nop; + cast_info->auxdata = NULL; + goto finalize; } /* If it's a single reference, it's one decref */ else if (src_dtype->type_num == NPY_OBJECT) { @@ -3400,17 +2726,14 @@ get_decsrcref_transfer_function(int aligned, *out_needs_api = 1; } - *out_stransfer = &_strided_to_null_dec_src_ref_reference; - *out_transferdata = NULL; - - return NPY_SUCCEED; + cast_info->func = &_strided_to_null_dec_src_ref_reference; + cast_info->auxdata = NULL; + goto finalize; } /* If there are subarrays, need to wrap it */ else if (PyDataType_HASSUBARRAY(src_dtype)) { PyArray_Dims src_shape = {NULL, -1}; npy_intp src_size; - PyArray_StridedUnaryOp *stransfer; - NpyAuxData *data; if (out_needs_api) { *out_needs_api = 1; @@ -3425,354 +2748,658 @@ get_decsrcref_transfer_function(int aligned, src_size = PyArray_MultiplyList(src_shape.ptr, src_shape.len); npy_free_cache_dim_obj(src_shape); - /* Get a function for contiguous src of the subarray type */ - if (get_decsrcref_transfer_function(aligned, - src_dtype->subarray->base->elsize, - src_dtype->subarray->base, - &stransfer, &data, - out_needs_api) != NPY_SUCCEED) { - return NPY_FAIL; - } - - if (wrap_transfer_function_n_to_n(stransfer, data, - src_stride, 0, - src_dtype->subarray->base->elsize, 0, - src_size, - out_stransfer, out_transferdata) != NPY_SUCCEED) { - NPY_AUXDATA_FREE(data); + if (get_n_to_n_transfer_function(aligned, + src_stride, 0, + src_dtype->subarray->base, NULL, 1, src_size, + &cast_info->func, &cast_info->auxdata, + out_needs_api) != NPY_SUCCEED) { return NPY_FAIL; } - return NPY_SUCCEED; + goto finalize; } /* If there are fields, need to do each field */ - else { + else if (PyDataType_HASFIELDS(src_dtype)) { if (out_needs_api) { *out_needs_api = 1; } - return get_decsrcref_fields_transfer_function(aligned, + if (get_decref_fields_transfer_function(aligned, src_stride, src_dtype, - out_stransfer, - out_transferdata, - out_needs_api); + &cast_info->func, &cast_info->auxdata, + out_needs_api) < 0) { + return NPY_FAIL; + } + goto finalize; + } + else { + PyErr_Format(PyExc_RuntimeError, + "Internal error, tried to fetch decref function for the " + "unsupported DType '%S'.", src_dtype); + return NPY_FAIL; } -} - -/********************* DTYPE COPY SWAP FUNCTION ***********************/ -NPY_NO_EXPORT int -PyArray_GetDTypeCopySwapFn(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *dtype, - PyArray_StridedUnaryOp **outstransfer, - NpyAuxData **outtransferdata) + finalize: + /* Make sure all important fields are either set or cleared */ + Py_INCREF(src_dtype); + cast_info->descriptors[0] = src_dtype; + cast_info->descriptors[1] = NULL; + cast_info->context.method = NULL; + cast_info->context.caller = NULL; + return NPY_SUCCEED; +} + + +/* + * ********************* Generalized Multistep Cast ************************ + * + * New general purpose multiple step cast function when resolve descriptors + * implies that multiple cast steps are necessary. + */ + +typedef struct { + NpyAuxData base; + /* Information for main cast */ + NPY_cast_info main; + /* Information for input preparation cast */ + NPY_cast_info from; + /* Information for output finalization cast */ + NPY_cast_info to; + char *from_buffer; + char *to_buffer; +} _multistep_castdata; + + +/* zero-padded data copy function */ +static void +_multistep_cast_auxdata_free(NpyAuxData *auxdata) { - npy_intp itemsize = dtype->elsize; - - /* If it's a custom data type, wrap its copy swap function */ - if (dtype->type_num >= NPY_NTYPES) { - *outstransfer = NULL; - wrap_copy_swap_function(aligned, - src_stride, dst_stride, - dtype, - !PyArray_ISNBO(dtype->byteorder), - outstransfer, outtransferdata); - } - /* A straight copy */ - else if (itemsize == 1 || PyArray_ISNBO(dtype->byteorder)) { - *outstransfer = PyArray_GetStridedCopyFn(aligned, - src_stride, dst_stride, - itemsize); - *outtransferdata = NULL; + _multistep_castdata *data = (_multistep_castdata *)auxdata; + NPY_cast_info_xfree(&data->main); + if (data->from.func != NULL) { + NPY_cast_info_xfree(&data->from); } - else if (dtype->kind == 'U') { - return wrap_copy_swap_function(aligned, - src_stride, dst_stride, dtype, 1, - outstransfer, outtransferdata); + if (data->to.func != NULL) { + NPY_cast_info_xfree(&data->to); } - /* If it's not complex, one swap */ - else if (dtype->kind != 'c') { - *outstransfer = PyArray_GetStridedCopySwapFn(aligned, - src_stride, dst_stride, - itemsize); - *outtransferdata = NULL; + PyMem_Free(data); +} + + +static NpyAuxData * +_multistep_cast_auxdata_clone(NpyAuxData *auxdata_old); + + +static NpyAuxData * +_multistep_cast_auxdata_clone_int(_multistep_castdata *castdata, int move_info) +{ + /* Round up the structure size to 16-byte boundary for the buffers */ + Py_ssize_t datasize = (sizeof(_multistep_castdata) + 15) & ~0xf; + + Py_ssize_t from_buffer_offset = datasize; + if (castdata->from.func != NULL) { + Py_ssize_t src_itemsize = castdata->main.context.descriptors[0]->elsize; + datasize += NPY_LOWLEVEL_BUFFER_BLOCKSIZE * src_itemsize; + datasize = (datasize + 15) & ~0xf; } - /* If complex, a paired swap */ - else { - *outstransfer = PyArray_GetStridedCopySwapPairFn(aligned, - src_stride, dst_stride, - itemsize); - *outtransferdata = NULL; + Py_ssize_t to_buffer_offset = datasize; + if (castdata->to.func != NULL) { + Py_ssize_t dst_itemsize = castdata->main.context.descriptors[1]->elsize; + datasize += NPY_LOWLEVEL_BUFFER_BLOCKSIZE * dst_itemsize; } - return (*outstransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; -} + char *char_data = PyMem_Malloc(datasize); + if (char_data == NULL) { + return NULL; + } -/********************* MAIN DTYPE TRANSFER FUNCTION ***********************/ + _multistep_castdata *newdata = (_multistep_castdata *)char_data; -NPY_NO_EXPORT int -PyArray_GetDTypeTransferFunction(int aligned, - npy_intp src_stride, npy_intp dst_stride, - PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, - int move_references, - PyArray_StridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, - int *out_needs_api) -{ - npy_intp src_itemsize, dst_itemsize; - int src_type_num, dst_type_num; - int is_builtin; + /* Fix up the basic information: */ + newdata->base.free = &_multistep_cast_auxdata_free; + newdata->base.clone = &_multistep_cast_auxdata_clone; + /* And buffer information: */ + newdata->from_buffer = char_data + from_buffer_offset; + newdata->to_buffer = char_data + to_buffer_offset; -#if NPY_DT_DBG_TRACING - printf("Calculating dtype transfer from "); - if (PyObject_Print((PyObject *)src_dtype, stdout, 0) < 0) { - return NPY_FAIL; + /* Initialize funcs to NULL to signal no-cleanup in case of an error. */ + newdata->from.func = NULL; + newdata->to.func = NULL; + + if (move_info) { + NPY_cast_info_move(&newdata->main, &castdata->main); } - printf(" to "); - if (PyObject_Print((PyObject *)dst_dtype, stdout, 0) < 0) { - return NPY_FAIL; + else if (NPY_cast_info_copy(&newdata->main, &castdata->main) < 0) { + goto fail; } - printf("\n"); -#endif - /* - * If one of the dtypes is NULL, we give back either a src decref - * function or a dst setzero function - */ - if (dst_dtype == NULL) { - if (move_references) { - return get_decsrcref_transfer_function(aligned, - src_dtype->elsize, - src_dtype, - out_stransfer, out_transferdata, - out_needs_api); + if (castdata->from.func != NULL) { + if (move_info) { + NPY_cast_info_move(&newdata->from, &castdata->from); } - else { - *out_stransfer = &_dec_src_ref_nop; - *out_transferdata = NULL; - return NPY_SUCCEED; + else if (NPY_cast_info_copy(&newdata->from, &castdata->from) < 0) { + goto fail; + } + + if (PyDataType_FLAGCHK(newdata->main.descriptors[0], NPY_NEEDS_INIT)) { + memset(newdata->from_buffer, 0, to_buffer_offset - from_buffer_offset); } } - else if (src_dtype == NULL) { - return get_setdstzero_transfer_function(aligned, - dst_dtype->elsize, - dst_dtype, - out_stransfer, out_transferdata, - out_needs_api); + if (castdata->to.func != NULL) { + if (move_info) { + NPY_cast_info_move(&newdata->to, &castdata->to); + } + else if (NPY_cast_info_copy(&newdata->to, &castdata->to) < 0) { + goto fail; + } + + if (PyDataType_FLAGCHK(newdata->main.descriptors[1], NPY_NEEDS_INIT)) { + memset(newdata->to_buffer, 0, datasize - to_buffer_offset); + } } - src_itemsize = src_dtype->elsize; - dst_itemsize = dst_dtype->elsize; - src_type_num = src_dtype->type_num; - dst_type_num = dst_dtype->type_num; - is_builtin = src_type_num < NPY_NTYPES && dst_type_num < NPY_NTYPES; + return (NpyAuxData *)newdata; - /* Common special case - number -> number NBO cast */ - if (PyTypeNum_ISNUMBER(src_type_num) && - PyTypeNum_ISNUMBER(dst_type_num) && - PyArray_ISNBO(src_dtype->byteorder) && - PyArray_ISNBO(dst_dtype->byteorder)) { + fail: + NPY_AUXDATA_FREE((NpyAuxData *)newdata); + return NULL; +} - if (PyArray_EquivTypenums(src_type_num, dst_type_num)) { - *out_stransfer = PyArray_GetStridedCopyFn(aligned, - src_stride, dst_stride, - src_itemsize); - *out_transferdata = NULL; - return (*out_stransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; + +static NpyAuxData * +_multistep_cast_auxdata_clone(NpyAuxData *auxdata_old) +{ + return _multistep_cast_auxdata_clone_int( + (_multistep_castdata *)auxdata_old, 0); +} + + +static int +_strided_to_strided_multistep_cast( + /* The context is always stored explicitly in auxdata */ + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *auxdata) +{ + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + _multistep_castdata *castdata = (_multistep_castdata *)auxdata; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + + char *main_src, *main_dst; + npy_intp main_src_stride, main_dst_stride; + + npy_intp block_size = NPY_LOWLEVEL_BUFFER_BLOCKSIZE; + while (N > 0) { + if (block_size > N) { + block_size = N; + } + + if (castdata->from.func != NULL) { + npy_intp out_stride = castdata->from.descriptors[1]->elsize; + if (castdata->from.func(&castdata->from.context, + (char *[2]){src, castdata->from_buffer}, &block_size, + (npy_intp [2]){src_stride, out_stride}, + castdata->from.auxdata) != 0) { + /* TODO: Internal buffer may require cleanup on error. */ + return -1; + } + main_src = castdata->from_buffer; + main_src_stride = out_stride; } else { - return get_nbo_cast_numeric_transfer_function (aligned, - src_stride, dst_stride, - src_type_num, dst_type_num, - out_stransfer, out_transferdata); + main_src = src; + main_src_stride = src_stride; } + + if (castdata->to.func != NULL) { + main_dst = castdata->to_buffer; + main_dst_stride = castdata->main.descriptors[1]->elsize; + } + else { + main_dst = dst; + main_dst_stride = dst_stride; + } + + if (castdata->main.func(&castdata->main.context, + (char *[2]){main_src, main_dst}, &block_size, + (npy_intp [2]){main_src_stride, main_dst_stride}, + castdata->main.auxdata) != 0) { + /* TODO: Internal buffer may require cleanup on error. */ + return -1; + } + + if (castdata->to.func != NULL) { + if (castdata->to.func(&castdata->to.context, + (char *[2]){main_dst, dst}, &block_size, + (npy_intp [2]){main_dst_stride, dst_stride}, + castdata->to.auxdata) != 0) { + return -1; + } + } + + N -= block_size; + src += block_size * src_stride; + dst += block_size * dst_stride; } + return 0; +} - /* - * If there are no references and the data types are equivalent and builtin, - * return a simple copy - */ - if (PyArray_EquivTypes(src_dtype, dst_dtype) && - !PyDataType_REFCHK(src_dtype) && !PyDataType_REFCHK(dst_dtype) && - ( !PyDataType_HASFIELDS(dst_dtype) || - is_dtype_struct_simple_unaligned_layout(dst_dtype)) && - is_builtin) { + +/* + * Initialize most of a cast-info structure, this step does not fetch the + * transferfunction and transferdata. + */ +static NPY_INLINE int +init_cast_info(NPY_cast_info *cast_info, NPY_CASTING *casting, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int main_step) +{ + PyObject *meth = PyArray_GetCastingImpl( + NPY_DTYPE(src_dtype), NPY_DTYPE(dst_dtype)); + if (meth == NULL) { + return -1; + } + if (meth == Py_None) { + Py_DECREF(Py_None); + PyErr_Format(PyExc_TypeError, + "Cannot cast data from %S to %S.", src_dtype, dst_dtype); + return -1; + } + /* Initialize the context and related data */ + NPY_cast_info_init(cast_info); + cast_info->auxdata = NULL; + + cast_info->context.caller = NULL; + cast_info->context.method = (PyArrayMethodObject *)meth; + + PyArray_DTypeMeta *dtypes[2] = {NPY_DTYPE(src_dtype), NPY_DTYPE(dst_dtype)}; + PyArray_Descr *in_descr[2] = {src_dtype, dst_dtype}; + + *casting = cast_info->context.method->resolve_descriptors( + cast_info->context.method, dtypes, in_descr, cast_info->descriptors); + if (NPY_UNLIKELY(*casting < 0)) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_TypeError, + "Cannot cast array data from %R to %R.", src_dtype, dst_dtype); + Py_DECREF(meth); + return -1; + } + } + assert(PyArray_DescrCheck(cast_info->descriptors[0])); + assert(PyArray_DescrCheck(cast_info->descriptors[1])); + + if (!main_step && NPY_UNLIKELY(src_dtype != cast_info->descriptors[0] || + dst_dtype != cast_info->descriptors[1])) { /* - * We can't pass through the aligned flag because it's not - * appropriate. Consider a size-8 string, it will say it's - * aligned because strings only need alignment 1, but the - * copy function wants to know if it's alignment 8. - * - * TODO: Change align from a flag to a "best power of 2 alignment" - * which holds the strongest alignment value for all - * the data which will be used. + * We currently do not resolve recursively, but require a non + * main cast (within the same DType) to be done in a single step. + * This could be expanded at some point if the need arises. */ - *out_stransfer = PyArray_GetStridedCopyFn(0, - src_stride, dst_stride, - src_dtype->elsize); - *out_transferdata = NULL; - return NPY_SUCCEED; + PyErr_Format(PyExc_RuntimeError, + "Required internal cast from %R to %R was not done in a single " + "step (a secondary cast must currently be between instances of " + "the same DType class and such a cast must currently return " + "the input descriptors unmodified).", + src_dtype, dst_dtype); + NPY_cast_info_xfree(cast_info); + return -1; } - /* First look at the possibilities of just a copy or swap */ - if (src_itemsize == dst_itemsize && src_dtype->kind == dst_dtype->kind && - !PyDataType_HASFIELDS(src_dtype) && - !PyDataType_HASFIELDS(dst_dtype) && - !PyDataType_HASSUBARRAY(src_dtype) && - !PyDataType_HASSUBARRAY(dst_dtype) && - src_type_num != NPY_DATETIME && src_type_num != NPY_TIMEDELTA) { - /* A custom data type requires that we use its copy/swap */ - if (!is_builtin) { - /* - * If the sizes and kinds are identical, but they're different - * custom types, then get a cast function - */ - if (src_type_num != dst_type_num) { - return get_cast_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - out_stransfer, out_transferdata, - out_needs_api); - } - else { - return wrap_copy_swap_function(aligned, - src_stride, dst_stride, - src_dtype, - PyArray_ISNBO(src_dtype->byteorder) != - PyArray_ISNBO(dst_dtype->byteorder), - out_stransfer, out_transferdata); - } - } + return 0; +} - /* The special types, which have no or subelement byte-order */ - switch (src_type_num) { - case NPY_UNICODE: - /* Wrap the copy swap function when swapping is necessary */ - if (PyArray_ISNBO(src_dtype->byteorder) != - PyArray_ISNBO(dst_dtype->byteorder)) { - return wrap_copy_swap_function(aligned, - src_stride, dst_stride, - src_dtype, 1, - out_stransfer, out_transferdata); - } - case NPY_VOID: - case NPY_STRING: - *out_stransfer = PyArray_GetStridedCopyFn(0, - src_stride, dst_stride, - src_itemsize); - *out_transferdata = NULL; - return NPY_SUCCEED; - case NPY_OBJECT: - if (out_needs_api) { - *out_needs_api = 1; - } - if (move_references) { - *out_stransfer = &_strided_to_strided_move_references; - *out_transferdata = NULL; - } - else { - *out_stransfer = &_strided_to_strided_copy_references; - *out_transferdata = NULL; - } - return NPY_SUCCEED; + +/* + * When there is a failure in ArrayMethod.get_loop(...) we still have + * to clean up references, but assume that `auxdata` and `func` + * have undefined values. + * NOTE: This should possibly be moved, but is only necessary here + */ +static void +_clear_cast_info_after_get_loop_failure(NPY_cast_info *cast_info) +{ + /* As public API we could choose to clear auxdata != NULL */ + assert(cast_info->auxdata == NULL); + /* Set func to be non-null so that `NPY_cats_info_xfree` does not skip */ + cast_info->func = &_dec_src_ref_nop; + NPY_cast_info_xfree(cast_info); +} + + +/* + * Helper for PyArray_GetDTypeTransferFunction, which fetches a single + * transfer function from the each casting implementation (ArrayMethod). + * May set the transfer function to NULL when the cast can be achieved using + * a view. + * The `out_needs_api` flag must be initialized. + * + * NOTE: In theory casting errors here could be slightly misleading in case + * of a multi-step casting scenario. It should be possible to improve + * this in the future. + * + * Note about `move_references`: Move references means stealing of + * references. It is useful to clear buffers immediately. No matter the + * input all copies from a buffer must use `move_references`. Move references + * is thus used: + * * For the added initial "from" cast if it was passed in + * * Always in the main step if a "from" cast is made (it casts from a buffer) + * * Always for the "to" cast, as it always cast from a buffer to the output. + * + * Returns -1 on failure, 0 on success + */ +static int +define_cast_for_descrs( + int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + NPY_cast_info *cast_info, int *out_needs_api) +{ + /* Storage for all cast info in case multi-step casting is necessary */ + _multistep_castdata castdata; + /* Initialize funcs to NULL to simplify cleanup on error. */ + castdata.main.func = NULL; + castdata.to.func = NULL; + castdata.from.func = NULL; + NPY_CASTING casting = -1; + + if (init_cast_info(cast_info, &casting, src_dtype, dst_dtype, 1) < 0) { + return -1; + } + + /* + * Both input and output must be wrapped in case they may be unaligned + * and the method does not support unaligned data. + * NOTE: It is probable that most/all legacy loops actually do support + * unaligned output, we could move the wrapping there if we wanted + * to. It probably isn't speed relevant though and they should be + * deleted in any case. + */ + int must_wrap = (!aligned && + (cast_info->context.method->flags & NPY_METH_SUPPORTS_UNALIGNED) == 0); + + /* + * Wrap the input with an additional cast if necessary. + */ + if (NPY_UNLIKELY(src_dtype != cast_info->descriptors[0] || must_wrap)) { + NPY_CASTING from_casting = -1; + /* Cast function may not support the input, wrap if necessary */ + if (init_cast_info( + &castdata.from, &from_casting, + src_dtype, cast_info->descriptors[0], 0) < 0) { + goto fail; + } + casting = PyArray_MinCastSafety(casting, from_casting); + + /* Prepare the actual cast (if necessary): */ + if (from_casting & _NPY_CAST_IS_VIEW && !must_wrap) { + /* This step is not necessary and can be skipped. */ + castdata.from.func = &_dec_src_ref_nop; /* avoid NULL */ + NPY_cast_info_xfree(&castdata.from); } + else { + /* Fetch the cast function and set up */ + PyArrayMethod_Context *context = &castdata.from.context; + npy_intp strides[2] = {src_stride, cast_info->descriptors[0]->elsize}; + NPY_ARRAYMETHOD_FLAGS flags; + if (context->method->get_strided_loop( + context, aligned, move_references, strides, + &castdata.from.func, &castdata.from.auxdata, &flags) < 0) { + _clear_cast_info_after_get_loop_failure(&castdata.from); + goto fail; + } + assert(castdata.from.func != NULL); - /* This is a straight copy */ - if (src_itemsize == 1 || PyArray_ISNBO(src_dtype->byteorder) == - PyArray_ISNBO(dst_dtype->byteorder)) { - *out_stransfer = PyArray_GetStridedCopyFn(aligned, - src_stride, dst_stride, - src_itemsize); - *out_transferdata = NULL; - return (*out_stransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; + *out_needs_api |= (flags & NPY_METH_REQUIRES_PYAPI) != 0; + /* The main cast now uses a buffered input: */ + src_stride = strides[1]; + move_references = 1; /* main cast has to clear the buffer */ } - /* This is a straight copy + byte swap */ - else if (!PyTypeNum_ISCOMPLEX(src_type_num)) { - *out_stransfer = PyArray_GetStridedCopySwapFn(aligned, - src_stride, dst_stride, - src_itemsize); - *out_transferdata = NULL; - return (*out_stransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; + } + /* + * Wrap the output with an additional cast if necessary. + */ + if (NPY_UNLIKELY(dst_dtype != cast_info->descriptors[1] || must_wrap)) { + NPY_CASTING to_casting = -1; + /* Cast function may not support the output, wrap if necessary */ + if (init_cast_info( + &castdata.to, &to_casting, + cast_info->descriptors[1], dst_dtype, 0) < 0) { + goto fail; + } + casting = PyArray_MinCastSafety(casting, to_casting); + + /* Prepare the actual cast (if necessary): */ + if (to_casting & _NPY_CAST_IS_VIEW && !must_wrap) { + /* This step is not necessary and can be skipped. */ + castdata.to.func = &_dec_src_ref_nop; /* avoid NULL */ + NPY_cast_info_xfree(&castdata.to); } - /* This is a straight copy + element pair byte swap */ else { - *out_stransfer = PyArray_GetStridedCopySwapPairFn(aligned, - src_stride, dst_stride, - src_itemsize); - *out_transferdata = NULL; - return (*out_stransfer == NULL) ? NPY_FAIL : NPY_SUCCEED; + /* Fetch the cast function and set up */ + PyArrayMethod_Context *context = &castdata.to.context; + npy_intp strides[2] = {cast_info->descriptors[1]->elsize, dst_stride}; + NPY_ARRAYMETHOD_FLAGS flags; + if (context->method->get_strided_loop( + context, aligned, 1 /* clear buffer */, strides, + &castdata.to.func, &castdata.to.auxdata, &flags) < 0) { + _clear_cast_info_after_get_loop_failure(&castdata.to); + goto fail; + } + assert(castdata.to.func != NULL); + + *out_needs_api |= (flags & NPY_METH_REQUIRES_PYAPI) != 0; + /* The main cast now uses a buffered input: */ + dst_stride = strides[0]; + if (castdata.from.func != NULL) { + /* Both input and output are wrapped, now always aligned */ + aligned = 1; + } } } - /* Handle subarrays */ - if (PyDataType_HASSUBARRAY(src_dtype) || - PyDataType_HASSUBARRAY(dst_dtype)) { - return get_subarray_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - out_stransfer, out_transferdata, - out_needs_api); + /* Fetch the main cast function (with updated values) */ + PyArrayMethod_Context *context = &cast_info->context; + npy_intp strides[2] = {src_stride, dst_stride}; + NPY_ARRAYMETHOD_FLAGS flags; + if (context->method->get_strided_loop( + context, aligned, move_references, strides, + &cast_info->func, &cast_info->auxdata, &flags) < 0) { + _clear_cast_info_after_get_loop_failure(cast_info); + goto fail; } - /* Handle fields */ - if ((PyDataType_HASFIELDS(src_dtype) || PyDataType_HASFIELDS(dst_dtype)) && - src_type_num != NPY_OBJECT && dst_type_num != NPY_OBJECT) { - return get_fields_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - out_stransfer, out_transferdata, - out_needs_api); + *out_needs_api |= (flags & NPY_METH_REQUIRES_PYAPI) != 0; + + if (castdata.from.func == NULL && castdata.to.func == NULL) { + /* Most of the time, there will be only one step required. */ + return 0; + } + /* The full cast passed in is only the "main" step, copy cast_info there */ + NPY_cast_info_move(&castdata.main, cast_info); + Py_INCREF(src_dtype); + cast_info->descriptors[0] = src_dtype; + Py_INCREF(dst_dtype); + cast_info->descriptors[1] = dst_dtype; + cast_info->context.method = NULL; + + cast_info->func = &_strided_to_strided_multistep_cast; + cast_info->auxdata = _multistep_cast_auxdata_clone_int(&castdata, 1); + if (cast_info->auxdata == NULL) { + PyErr_NoMemory(); + goto fail; } - /* Check for different-sized strings, unicodes, or voids */ - if (src_type_num == dst_type_num) { - switch (src_type_num) { - case NPY_UNICODE: - if (PyArray_ISNBO(src_dtype->byteorder) != - PyArray_ISNBO(dst_dtype->byteorder)) { - return PyArray_GetStridedZeroPadCopyFn(0, 1, - src_stride, dst_stride, - src_dtype->elsize, dst_dtype->elsize, - out_stransfer, out_transferdata); - } - case NPY_STRING: - case NPY_VOID: - return PyArray_GetStridedZeroPadCopyFn(0, 0, - src_stride, dst_stride, - src_dtype->elsize, dst_dtype->elsize, - out_stransfer, out_transferdata); + return 0; + + fail: + NPY_cast_info_xfree(&castdata.main); + NPY_cast_info_xfree(&castdata.from); + NPY_cast_info_xfree(&castdata.to); + return -1; +} + + +NPY_NO_EXPORT int +PyArray_GetDTypeTransferFunction(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + NPY_cast_info *cast_info, + int *out_needs_api) +{ + assert(src_dtype != NULL); + + /* + * If one of the dtypes is NULL, we give back either a src decref + * function or a dst setzero function + * + * TODO: Eventually, we may wish to support user dtype with references + * (including and beyond bare `PyObject *` this may require extending + * the ArrayMethod API and those paths should likely be split out + * from this function.) + */ + if (dst_dtype == NULL) { + assert(move_references); + return get_decref_transfer_function(aligned, + src_dtype->elsize, + src_dtype, + cast_info, + out_needs_api); + } + + if (define_cast_for_descrs(aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, move_references, + cast_info, out_needs_api) < 0) { + return NPY_FAIL; + } + + return NPY_SUCCEED; +} + + +/* + * Internal wrapping of casts that have to be performed in a "single" + * function (i.e. not by the generic multi-step-cast), but rely on it + * internally. There are only two occasions where this is used: + * + * 1. Void advertises that it handles unaligned casts, but has to wrap the + * legacy cast which (probably) does not. + * 2. Datetime to unicode casts are implemented via bytes "U" vs. "S". If + * we relax the chaining rules to allow "recursive" cast chaining where + * `resolve_descriptors` can return a descriptor with a different type, + * this would become unnecessary. + * 3. Time <-> Time casts, which currently must support byte swapping, but + * have a non-trivial inner-loop (due to units) which does not support + * it. + * + * When wrapping is performed (guaranteed for `aligned == 0` and if the + * wrapped dtype is not identical to the input dtype), the wrapped transfer + * function can assume a contiguous input. + * Otherwise use `must_wrap` to ensure that wrapping occurs, which guarantees + * a contiguous, aligned, call of the wrapped function. + */ +NPY_NO_EXPORT int +wrap_aligned_transferfunction( + int aligned, int must_wrap, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_Descr *src_wrapped_dtype, PyArray_Descr *dst_wrapped_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, int *out_needs_api) +{ + must_wrap = must_wrap | !aligned; + + _multistep_castdata castdata; + NPY_cast_info_init(&castdata.main); + NPY_cast_info_init(&castdata.from); + NPY_cast_info_init(&castdata.to); + + /* Finalize the existing cast information: */ + castdata.main.func = *out_stransfer; + *out_stransfer = NULL; + castdata.main.auxdata = *out_transferdata; + *out_transferdata = NULL; + castdata.main.context.method = NULL; + /* These are always legacy casts that only support native-byte-order: */ + Py_INCREF(src_wrapped_dtype); + castdata.main.descriptors[0] = src_wrapped_dtype; + if (castdata.main.descriptors[0] == NULL) { + castdata.main.descriptors[1] = NULL; + goto fail; + } + Py_INCREF(dst_wrapped_dtype); + castdata.main.descriptors[1] = dst_wrapped_dtype; + if (castdata.main.descriptors[1] == NULL) { + goto fail; + } + + /* + * Similar to the normal multi-step cast, but we always have to wrap + * it all up, but we can simply do this via a "recursive" call. + * TODO: This is slightly wasteful, since it unnecessarily checks casting, + * but this whole function is about corner cases, which should rather + * have an explicit implementation instead if we want performance. + */ + if (must_wrap || src_wrapped_dtype != src_dtype) { + if (PyArray_GetDTypeTransferFunction(aligned, + src_stride, castdata.main.descriptors[0]->elsize, + src_dtype, castdata.main.descriptors[0], 0, + &castdata.from, out_needs_api) != NPY_SUCCEED) { + goto fail; + } + } + if (must_wrap || dst_wrapped_dtype != dst_dtype) { + if (PyArray_GetDTypeTransferFunction(aligned, + castdata.main.descriptors[1]->elsize, dst_stride, + castdata.main.descriptors[1], dst_dtype, + 1, /* clear buffer if it includes references */ + &castdata.to, out_needs_api) != NPY_SUCCEED) { + goto fail; } } - /* Otherwise a cast is necessary */ - return get_cast_transfer_function(aligned, - src_stride, dst_stride, - src_dtype, dst_dtype, - move_references, - out_stransfer, out_transferdata, - out_needs_api); + *out_transferdata = _multistep_cast_auxdata_clone_int(&castdata, 1); + if (*out_transferdata == NULL) { + PyErr_NoMemory(); + goto fail; + } + *out_stransfer = &_strided_to_strided_multistep_cast; + return 0; + + fail: + NPY_cast_info_xfree(&castdata.main); + NPY_cast_info_xfree(&castdata.from); + NPY_cast_info_xfree(&castdata.to); + + return -1; } /* - * Basic version of PyArray_GetDTypeTransferFunction for legacy dtype - * support. - * It supports only wrapping the copyswapn functions and the legacy - * cast functions registered with `PyArray_RegisterCastFunc`. - * This function takes the easy way out: It does not wrap + * This function wraps the legacy casts stored on the `dtype->f->cast` + * or registered with `PyArray_RegisterCastFunc`. + * For casts between two dtypes with the same type (within DType casts) + * it also wraps the `copyswapn` function. + * + * This function is called called from `ArrayMethod.get_loop()` when a + * specialized cast function is missing. + * + * In general, the legacy cast functions do not support unaligned access, + * so an ArrayMethod using this must signal that. In a few places we do + * signal support for unaligned access (or byte swapping). + * In this case `allow_wrapped=1` will wrap it into an additional multi-step + * cast as necessary. */ NPY_NO_EXPORT int -PyArray_GetLegacyDTypeTransferFunction(int aligned, +get_wrapped_legacy_cast_function(int aligned, npy_intp src_stride, npy_intp dst_stride, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int move_references, - PyArray_StridedUnaryOp **out_stransfer, + PyArrayMethod_StridedLoop **out_stransfer, NpyAuxData **out_transferdata, - int *out_needs_api) + int *out_needs_api, int allow_wrapped) { /* Note: We ignore `needs_wrap`; needs-wrap is handled by another cast */ int needs_wrap = 0; @@ -3782,15 +3409,14 @@ PyArray_GetLegacyDTypeTransferFunction(int aligned, * This is a cast within the same dtype. For legacy user-dtypes, * it is always valid to handle this using the copy swap function. */ - return wrap_copy_swap_function(aligned, - src_stride, dst_stride, - src_dtype, - PyArray_ISNBO(src_dtype->byteorder) != - PyArray_ISNBO(dst_dtype->byteorder), + return wrap_copy_swap_function(src_dtype, + PyDataType_ISNOTSWAPPED(src_dtype) != + PyDataType_ISNOTSWAPPED(dst_dtype), out_stransfer, out_transferdata); } - if (get_legacy_dtype_cast_function(aligned, + if (get_legacy_dtype_cast_function( + aligned, src_stride, dst_stride, src_dtype, dst_dtype, move_references, @@ -3798,9 +3424,51 @@ PyArray_GetLegacyDTypeTransferFunction(int aligned, out_transferdata, out_needs_api, &needs_wrap) != NPY_SUCCEED) { - return NPY_FAIL; + return -1; } - return NPY_SUCCEED; + if (!needs_wrap) { + return 0; + } + if (NPY_UNLIKELY(!allow_wrapped)) { + /* + * Legacy casts do not support unaligned which requires wrapping. + * However, normally we ensure that wrapping happens before calling + * this function, so this path should never happen. + */ + PyErr_Format(PyExc_RuntimeError, + "Internal NumPy error, casting %S to %S required wrapping, " + "probably it incorrectly flagged support for unaligned data. " + "(aligned passed to discovery is %d)", + src_dtype, dst_dtype, aligned); + goto fail; + } + + /* + * If we are here, use the legacy code to wrap the above cast (which + * does not support unaligned data) into copyswapn. + */ + PyArray_Descr *src_wrapped_dtype = ensure_dtype_nbo(src_dtype); + if (src_wrapped_dtype == NULL) { + goto fail; + } + PyArray_Descr *dst_wrapped_dtype = ensure_dtype_nbo(dst_dtype); + if (dst_wrapped_dtype == NULL) { + goto fail; + } + int res = wrap_aligned_transferfunction( + aligned, 1, /* We assume wrapped is contiguous here */ + src_stride, dst_stride, + src_dtype, dst_dtype, + src_wrapped_dtype, dst_wrapped_dtype, + out_stransfer, out_transferdata, out_needs_api); + Py_DECREF(src_wrapped_dtype); + Py_DECREF(dst_wrapped_dtype); + return res; + + fail: + NPY_AUXDATA_FREE(*out_transferdata); + *out_transferdata = NULL; + return -1; } @@ -3813,71 +3481,66 @@ PyArray_GetMaskedDTypeTransferFunction(int aligned, PyArray_Descr *dst_dtype, PyArray_Descr *mask_dtype, int move_references, - PyArray_MaskedStridedUnaryOp **out_stransfer, - NpyAuxData **out_transferdata, + NPY_cast_info *cast_info, int *out_needs_api) { - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - _masked_wrapper_transfer_data *data; + NPY_cast_info_init(cast_info); - /* TODO: Add struct-based mask_dtype support later */ if (mask_dtype->type_num != NPY_BOOL && mask_dtype->type_num != NPY_UINT8) { PyErr_SetString(PyExc_TypeError, - "Only bool and uint8 masks are supported at the moment, " - "structs of bool/uint8 is planned for the future"); + "Only bool and uint8 masks are supported."); return NPY_FAIL; } - /* TODO: Special case some important cases so they're fast */ + /* Create the wrapper function's auxdata */ + _masked_wrapper_transfer_data *data; + data = PyMem_Malloc(sizeof(_masked_wrapper_transfer_data)); + if (data == NULL) { + PyErr_NoMemory(); + return NPY_FAIL; + } + data->base.free = &_masked_wrapper_transfer_data_free; + data->base.clone = &_masked_wrapper_transfer_data_clone; /* Fall back to wrapping a non-masked transfer function */ + assert(dst_dtype != NULL); if (PyArray_GetDTypeTransferFunction(aligned, src_stride, dst_stride, src_dtype, dst_dtype, move_references, - &stransfer, &transferdata, + &data->wrapped, out_needs_api) != NPY_SUCCEED) { + PyMem_Free(data); return NPY_FAIL; } - /* Create the wrapper function's auxdata */ - data = (_masked_wrapper_transfer_data *)PyArray_malloc( - sizeof(_masked_wrapper_transfer_data)); - if (data == NULL) { - PyErr_NoMemory(); - NPY_AUXDATA_FREE(transferdata); - return NPY_FAIL; - } - - /* Fill in the auxdata object */ - memset(data, 0, sizeof(_masked_wrapper_transfer_data)); - data->base.free = &_masked_wrapper_transfer_data_free; - data->base.clone = &_masked_wrapper_transfer_data_clone; - - data->stransfer = stransfer; - data->transferdata = transferdata; - /* If the src object will need a DECREF, get a function to handle that */ if (move_references && PyDataType_REFCHK(src_dtype)) { - if (get_decsrcref_transfer_function(aligned, + if (get_decref_transfer_function(aligned, src_stride, src_dtype, - &data->decsrcref_stransfer, - &data->decsrcref_transferdata, + &data->decref_src, out_needs_api) != NPY_SUCCEED) { NPY_AUXDATA_FREE((NpyAuxData *)data); return NPY_FAIL; } - - *out_stransfer = &_strided_masked_wrapper_decsrcref_transfer_function; + cast_info->func = (PyArrayMethod_StridedLoop *) + &_strided_masked_wrapper_decref_transfer_function; } else { - *out_stransfer = &_strided_masked_wrapper_transfer_function; - } - - *out_transferdata = (NpyAuxData *)data; + NPY_cast_info_init(&data->decref_src); + cast_info->func = (PyArrayMethod_StridedLoop *) + &_strided_masked_wrapper_transfer_function; + } + cast_info->auxdata = (NpyAuxData *)data; + /* The context is almost unused, but clear it for cleanup. */ + Py_INCREF(src_dtype); + cast_info->descriptors[0] = src_dtype; + Py_INCREF(dst_dtype); + cast_info->descriptors[1] = dst_dtype; + cast_info->context.caller = NULL; + cast_info->context.method = NULL; return NPY_SUCCEED; } @@ -3889,8 +3552,6 @@ PyArray_CastRawArrays(npy_intp count, PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, int move_references) { - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; int aligned = 1, needs_api = 0; /* Make sure the copy is reasonable */ @@ -3914,21 +3575,23 @@ PyArray_CastRawArrays(npy_intp count, src_dtype->alignment); /* Get the function to do the casting */ + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction(aligned, src_stride, dst_stride, src_dtype, dst_dtype, move_references, - &stransfer, &transferdata, + &cast_info, &needs_api) != NPY_SUCCEED) { return NPY_FAIL; } /* Cast */ - stransfer(dst, dst_stride, src, src_stride, count, - src_dtype->elsize, transferdata); + char *args[2] = {src, dst}; + npy_intp strides[2] = {src_stride, dst_stride}; + cast_info.func(&cast_info.context, args, &count, strides, cast_info.auxdata); /* Cleanup */ - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); /* If needs_api was set to 1, it may have raised a Python exception */ return (needs_api && PyErr_Occurred()) ? NPY_FAIL : NPY_SUCCEED; diff --git a/numpy/core/src/multiarray/dtype_transfer.h b/numpy/core/src/multiarray/dtype_transfer.h new file mode 100644 index 000000000000..e29ac40b8900 --- /dev/null +++ b/numpy/core/src/multiarray/dtype_transfer.h @@ -0,0 +1,205 @@ +#ifndef _NPY_DTYPE_TRANSFER_H +#define _NPY_DTYPE_TRANSFER_H + +#include "array_method.h" + + +/* + * More than for most functions, cast information needs to be stored in + * a few places. Most importantly, in many cases we need to chain or wrap + * casts (e.g. structured dtypes). + * + * This struct provides a place to store all necessary information as + * compact as possible. It must be used with the inline functions below + * to ensure correct setup and teardown. + * + * In general, the casting machinery currently handles the correct set up + * of the struct. + */ +typedef struct { + PyArrayMethod_StridedLoop *func; + NpyAuxData *auxdata; + PyArrayMethod_Context context; + /* Storage to be linked from "context" */ + PyArray_Descr *descriptors[2]; +} NPY_cast_info; + + +/* + * Create a new cast-info struct with cast_info->context.descriptors linked. + * Compilers should inline this to ensure the whole struct is not actually + * copied. + * If set up otherwise, func must be NULL'ed to indicate no-cleanup necessary. + */ +static NPY_INLINE void +NPY_cast_info_init(NPY_cast_info *cast_info) +{ + cast_info->func = NULL; /* mark as uninitialized. */ + /* + * Support for auxdata being unchanged, in the future, we might add + * a scratch space to `NPY_cast_info` and link to that instead. + */ + cast_info->auxdata = NULL; + cast_info->context.descriptors = cast_info->descriptors; + + // TODO: Delete this again probably maybe create a new minimal init macro + cast_info->context.caller = NULL; +} + + +/* + * Free's all references and data held inside the struct (not the struct). + * First checks whether `cast_info.func == NULL`, and assume it is + * uninitialized in that case. + */ +static NPY_INLINE void +NPY_cast_info_xfree(NPY_cast_info *cast_info) +{ + if (cast_info->func == NULL) { + return; + } + assert(cast_info->context.descriptors == cast_info->descriptors); + NPY_AUXDATA_FREE(cast_info->auxdata); + Py_DECREF(cast_info->descriptors[0]); + Py_XDECREF(cast_info->descriptors[1]); + Py_XDECREF(cast_info->context.method); + cast_info->func = NULL; +} + + +/* + * Move the data from `original` to `cast_info`. Original is cleared + * (its func set to NULL). + */ +static NPY_INLINE void +NPY_cast_info_move(NPY_cast_info *cast_info, NPY_cast_info *original) +{ + *cast_info = *original; + /* Fix internal pointer: */ + cast_info->context.descriptors = cast_info->descriptors; + /* Mark original to not be cleaned up: */ + original->func = NULL; +} + +/* + * Finalize a copy (INCREF+auxdata clone). This assumes a previous `memcpy` + * of the struct. + * NOTE: It is acceptable to call this with the same struct if the struct + * has been filled by a valid memcpy from an initialized one. + */ +static NPY_INLINE int +NPY_cast_info_copy(NPY_cast_info *cast_info, NPY_cast_info *original) +{ + cast_info->context.descriptors = cast_info->descriptors; + + assert(original->func != NULL); + cast_info->func = original->func; + cast_info->descriptors[0] = original->descriptors[0]; + Py_XINCREF(cast_info->descriptors[0]); + cast_info->descriptors[1] = original->descriptors[1]; + Py_XINCREF(cast_info->descriptors[1]); + cast_info->context.caller = original->context.caller; + Py_XINCREF(cast_info->context.caller); + cast_info->context.method = original->context.method; + Py_XINCREF(cast_info->context.method); + if (original->auxdata == NULL) { + cast_info->auxdata = NULL; + return 0; + } + cast_info->auxdata = NPY_AUXDATA_CLONE(original->auxdata); + if (NPY_UNLIKELY(cast_info->auxdata == NULL)) { + /* No need for cleanup, everything but auxdata is initialized fine. */ + return -1; + } + return 0; +} + + +NPY_NO_EXPORT int +_strided_to_strided_move_references( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)); + +NPY_NO_EXPORT int +_strided_to_strided_copy_references( + PyArrayMethod_Context *NPY_UNUSED(context), char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)); + + +NPY_NO_EXPORT int +any_to_object_get_loop( + PyArrayMethod_Context *context, + int aligned, int move_references, + npy_intp *strides, + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags); + +NPY_NO_EXPORT int +object_to_any_get_loop( + PyArrayMethod_Context *context, + int NPY_UNUSED(aligned), int move_references, + npy_intp *NPY_UNUSED(strides), + PyArrayMethod_StridedLoop **out_loop, + NpyAuxData **out_transferdata, + NPY_ARRAYMETHOD_FLAGS *flags); + + +NPY_NO_EXPORT int +wrap_aligned_transferfunction( + int aligned, int must_wrap, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArray_Descr *src_wrapped_dtype, PyArray_Descr *dst_wrapped_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, int *out_needs_api); + + +NPY_NO_EXPORT int +get_nbo_cast_datetime_transfer_function(int aligned, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata); + +NPY_NO_EXPORT int +get_nbo_datetime_to_string_transfer_function( + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata); + +NPY_NO_EXPORT int +get_nbo_string_to_datetime_transfer_function( + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata); + +NPY_NO_EXPORT int +get_datetime_to_unicode_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api); + +NPY_NO_EXPORT int +get_unicode_to_datetime_transfer_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api); + +/* Creates a wrapper around copyswapn or legacy cast functions */ +NPY_NO_EXPORT int +get_wrapped_legacy_cast_function(int aligned, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references, + PyArrayMethod_StridedLoop **out_stransfer, + NpyAuxData **out_transferdata, + int *out_needs_api, int allow_wrapped); + + +#endif /* _NPY_DTYPE_TRANSFER_H */ diff --git a/numpy/core/src/multiarray/dtypemeta.c b/numpy/core/src/multiarray/dtypemeta.c index 4c11723e7dd1..4ee721964a91 100644 --- a/numpy/core/src/multiarray/dtypemeta.c +++ b/numpy/core/src/multiarray/dtypemeta.c @@ -31,6 +31,14 @@ dtypemeta_dealloc(PyArray_DTypeMeta *self) { PyType_Type.tp_dealloc((PyObject *) self); } +static PyObject * +dtypemeta_alloc(PyTypeObject *NPY_UNUSED(type), Py_ssize_t NPY_UNUSED(items)) +{ + PyErr_SetString(PyExc_TypeError, + "DTypes can only be created using the NumPy API."); + return NULL; +} + static PyObject * dtypemeta_new(PyTypeObject *NPY_UNUSED(type), PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) @@ -162,12 +170,12 @@ void_discover_descr_from_pyobject( } if (PyBytes_Check(obj)) { PyArray_Descr *descr = PyArray_DescrNewFromType(NPY_VOID); - Py_ssize_t itemsize = (int)PyBytes_Size(obj); + Py_ssize_t itemsize = PyBytes_Size(obj); if (itemsize > NPY_MAX_INT) { PyErr_SetString(PyExc_TypeError, "byte-like to large to store inside array."); } - descr->elsize = itemsize; + descr->elsize = (int)itemsize; return descr; } PyErr_Format(PyExc_TypeError, @@ -375,7 +383,10 @@ default_builtin_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) { assert(cls->type_num < NPY_NTYPES); if (!other->legacy || other->type_num > cls->type_num) { - /* Let the more generic (larger type number) DType handle this */ + /* + * Let the more generic (larger type number) DType handle this + * (note that half is after all others, which works out here.) + */ Py_INCREF(Py_NotImplemented); return (PyArray_DTypeMeta *)Py_NotImplemented; } @@ -397,10 +408,10 @@ default_builtin_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) static PyArray_DTypeMeta * string_unicode_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other) { - assert(cls->type_num < NPY_NTYPES); - if (!other->legacy || other->type_num > cls->type_num || - other->type_num == NPY_OBJECT) { - /* Let the more generic (larger type number) DType handle this */ + assert(cls->type_num < NPY_NTYPES && cls != other); + if (!other->legacy || (!PyTypeNum_ISNUMBER(other->type_num) && + /* Not numeric so defer unless cls is unicode and other is string */ + !(cls->type_num == NPY_UNICODE && other->type_num == NPY_STRING))) { Py_INCREF(Py_NotImplemented); return (PyArray_DTypeMeta *)Py_NotImplemented; } @@ -523,7 +534,7 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr) if (dot) { scalar_name = dot + 1; } - ssize_t name_length = strlen(scalar_name) + 14; + Py_ssize_t name_length = strlen(scalar_name) + 14; char *tp_name = malloc(name_length); if (tp_name == NULL) { @@ -674,6 +685,7 @@ NPY_NO_EXPORT PyTypeObject PyArrayDTypeMeta_Type = { .tp_doc = "Preliminary NumPy API: The Type of NumPy DTypes (metaclass)", .tp_members = dtypemeta_members, .tp_base = NULL, /* set to PyType_Type at import time */ + .tp_alloc = dtypemeta_alloc, .tp_init = (initproc)dtypemeta_init, .tp_new = dtypemeta_new, .tp_is_gc = dtypemeta_is_gc, diff --git a/numpy/core/src/multiarray/einsum.c.src b/numpy/core/src/multiarray/einsum.c.src index 6ad375f670a5..85806fab3612 100644 --- a/numpy/core/src/multiarray/einsum.c.src +++ b/numpy/core/src/multiarray/einsum.c.src @@ -1100,6 +1100,7 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop, char **dataptr; npy_intp *stride; npy_intp *countptr; + int needs_api; NPY_BEGIN_THREADS_DEF; iternext = NpyIter_GetIterNext(iter, NULL); @@ -1110,12 +1111,13 @@ PyArray_EinsteinSum(char *subscripts, npy_intp nop, dataptr = NpyIter_GetDataPtrArray(iter); stride = NpyIter_GetInnerStrideArray(iter); countptr = NpyIter_GetInnerLoopSizePtr(iter); + needs_api = NpyIter_IterationNeedsAPI(iter); NPY_BEGIN_THREADS_NDITER(iter); NPY_EINSUM_DBG_PRINT("Einsum loop\n"); do { sop(nop, dataptr, stride, *countptr); - } while(iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); NPY_END_THREADS; /* If the API was needed, it may have thrown an error */ diff --git a/numpy/core/src/multiarray/einsum_sumprod.c.src b/numpy/core/src/multiarray/einsum_sumprod.c.src index c58e742874d0..333b8e188355 100644 --- a/numpy/core/src/multiarray/einsum_sumprod.c.src +++ b/numpy/core/src/multiarray/einsum_sumprod.c.src @@ -17,30 +17,16 @@ #include "einsum_sumprod.h" #include "einsum_debug.h" +#include "simd/simd.h" +#include "common.h" - -#ifdef NPY_HAVE_SSE_INTRINSICS -#define EINSUM_USE_SSE1 1 -#else -#define EINSUM_USE_SSE1 0 -#endif - -#ifdef NPY_HAVE_SSE2_INTRINSICS -#define EINSUM_USE_SSE2 1 +// ARM/Neon don't have instructions for aligned memory access +#ifdef NPY_HAVE_NEON + #define EINSUM_IS_ALIGNED(x) 0 #else -#define EINSUM_USE_SSE2 0 -#endif - -#if EINSUM_USE_SSE1 -#include -#endif - -#if EINSUM_USE_SSE2 -#include + #define EINSUM_IS_ALIGNED(x) npy_is_aligned(x, NPY_SIMD_WIDTH) #endif -#define EINSUM_IS_SSE_ALIGNED(x) ((((npy_intp)x)&0xf) == 0) - /**********************************************/ /**begin repeat @@ -56,6 +42,10 @@ * npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong, * npy_float, npy_float, npy_double, npy_longdouble, * npy_float, npy_double, npy_longdouble# + * #sfx = s8, s16, s32, long, s64, + * u8, u16, u32, ulong, u64, + * half, f32, f64, longdouble, + * f32, f64, clongdouble# * #to = ,,,,, * ,,,,, * npy_float_to_half,,,, @@ -76,8 +66,64 @@ * 0*5, * 0,0,1,0, * 0*3# + * #NPYV_CHK = 0*5, + * 0*5, + * 0, NPY_SIMD, NPY_SIMD_F64, 0, + * 0*3# */ +#if !@complex@ +static NPY_GCC_OPT_3 @temptype@ @name@_sum_of_arr(@type@ *data, npy_intp count) +{ + @temptype@ accum = 0; +#if @NPYV_CHK@ // NPYV check for @type@ + /* Use aligned instructions if possible */ + const int is_aligned = EINSUM_IS_ALIGNED(data); + const int vstep = npyv_nlanes_@sfx@; + npyv_@sfx@ vaccum = npyv_zero_@sfx@(); + const npy_intp vstepx4 = vstep * 4; + + /**begin repeat1 + * #cond = if(is_aligned), else# + * #ld = loada, load# + * #st = storea, store# + */ + @cond@ { + for (; count >= vstepx4; count -= vstepx4, data += vstepx4) { + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + npyv_@sfx@ a@i@ = npyv_@ld@_@sfx@(data + vstep * @i@); + /**end repeat2**/ + npyv_@sfx@ a01 = npyv_add_@sfx@(a0, a1); + npyv_@sfx@ a23 = npyv_add_@sfx@(a2, a3); + npyv_@sfx@ a0123 = npyv_add_@sfx@(a01, a23); + vaccum = npyv_add_@sfx@(a0123, vaccum); + } + } + /**end repeat1**/ + for (; count > 0; count -= vstep, data += vstep) { + npyv_@sfx@ a = npyv_load_tillz_@sfx@(data, count); + vaccum = npyv_add_@sfx@(a, vaccum); + } + accum = npyv_sum_@sfx@(vaccum); + npyv_cleanup(); +#else +#ifndef NPY_DISABLE_OPTIMIZATION + for (; count > 4; count -= 4, data += 4) { + const @temptype@ a01 = @from@(*data) + @from@(data[1]); + const @temptype@ a23 = @from@(data[2]) + @from@(data[3]); + accum += a01 + a23; + } +#endif // !NPY_DISABLE_OPTIMIZATION + for (; count > 0; --count, data++) { + accum += @from@(*data); + } +#endif // NPYV check for @type@ + return accum; +} +#endif + /**begin repeat1 * #nop = 1, 2, 3, 1000# * #noplabel = one, two, three, any# @@ -243,6 +289,77 @@ finish_after_unrolled_loop: #elif @nop@ == 2 && !@complex@ +// calculate the multiply and add operation such as dataout = data*scalar+dataout +static NPY_GCC_OPT_3 void +@name@_sum_of_products_muladd(@type@ *data, @type@ *data_out, @temptype@ scalar, npy_intp count) +{ +#if @NPYV_CHK@ // NPYV check for @type@ + /* Use aligned instructions if possible */ + const int is_aligned = EINSUM_IS_ALIGNED(data) && EINSUM_IS_ALIGNED(data_out); + const int vstep = npyv_nlanes_@sfx@; + const npyv_@sfx@ v_scalar = npyv_setall_@sfx@(scalar); + /**begin repeat2 + * #cond = if(is_aligned), else# + * #ld = loada, load# + * #st = storea, store# + */ + @cond@ { + const npy_intp vstepx4 = vstep * 4; + for (; count >= vstepx4; count -= vstepx4, data += vstepx4, data_out += vstepx4) { + /**begin repeat3 + * #i = 0, 1, 2, 3# + */ + npyv_@sfx@ b@i@ = npyv_@ld@_@sfx@(data + vstep * @i@); + npyv_@sfx@ c@i@ = npyv_@ld@_@sfx@(data_out + vstep * @i@); + /**end repeat3**/ + /**begin repeat3 + * #i = 0, 1, 2, 3# + */ + npyv_@sfx@ abc@i@ = npyv_muladd_@sfx@(v_scalar, b@i@, c@i@); + /**end repeat3**/ + /**begin repeat3 + * #i = 0, 1, 2, 3# + */ + npyv_@st@_@sfx@(data_out + vstep * @i@, abc@i@); + /**end repeat3**/ + } + } + /**end repeat2**/ + for (; count > 0; count -= vstep, data += vstep, data_out += vstep) { + npyv_@sfx@ a = npyv_load_tillz_@sfx@(data, count); + npyv_@sfx@ b = npyv_load_tillz_@sfx@(data_out, count); + npyv_store_till_@sfx@(data_out, count, npyv_muladd_@sfx@(a, v_scalar, b)); + } + npyv_cleanup(); +#else +#ifndef NPY_DISABLE_OPTIMIZATION + for (; count >= 4; count -= 4, data += 4, data_out += 4) { + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + const @type@ b@i@ = @from@(data[@i@]); + const @type@ c@i@ = @from@(data_out[@i@]); + /**end repeat2**/ + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + const @type@ abc@i@ = scalar * b@i@ + c@i@; + /**end repeat2**/ + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + data_out[@i@] = @to@(abc@i@); + /**end repeat2**/ + } +#endif // !NPY_DISABLE_OPTIMIZATION + for (; count > 0; --count, ++data, ++data_out) { + const @type@ b = @from@(*data); + const @type@ c = @from@(*data_out); + *data_out = @to@(scalar * b + c); + } +#endif // NPYV check for @type@ +} + static void @name@_sum_of_products_contig_two(int nop, char **dataptr, npy_intp const *NPY_UNUSED(strides), npy_intp count) @@ -250,115 +367,80 @@ static void @type@ *data0 = (@type@ *)dataptr[0]; @type@ *data1 = (@type@ *)dataptr[1]; @type@ *data_out = (@type@ *)dataptr[2]; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, b; -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, b; -#endif - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_two (%d)\n", (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - data_out[@i@] = @to@(@from@(data0[@i@]) * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ - case 0: - return; - } - -#if EINSUM_USE_SSE1 && @float32@ + // NPYV check for @type@ +#if @NPYV_CHK@ /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1) && - EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_load_ps(data0+@i@), _mm_load_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); - _mm_store_ps(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data1 += 8; - data_out += 8; + const int is_aligned = EINSUM_IS_ALIGNED(data0) && EINSUM_IS_ALIGNED(data1) && + EINSUM_IS_ALIGNED(data_out); + const int vstep = npyv_nlanes_@sfx@; + + /**begin repeat2 + * #cond = if(is_aligned), else# + * #ld = loada, load# + * #st = storea, store# + */ + @cond@ { + const npy_intp vstepx4 = vstep * 4; + for (; count >= vstepx4; count -= vstepx4, data0 += vstepx4, data1 += vstepx4, data_out += vstepx4) { + /**begin repeat3 + * #i = 0, 1, 2, 3# + */ + npyv_@sfx@ a@i@ = npyv_@ld@_@sfx@(data0 + vstep * @i@); + npyv_@sfx@ b@i@ = npyv_@ld@_@sfx@(data1 + vstep * @i@); + npyv_@sfx@ c@i@ = npyv_@ld@_@sfx@(data_out + vstep * @i@); + /**end repeat3**/ + /**begin repeat3 + * #i = 0, 1, 2, 3# + */ + npyv_@sfx@ abc@i@ = npyv_muladd_@sfx@(a@i@, b@i@, c@i@); + /**end repeat3**/ + /**begin repeat3 + * #i = 0, 1, 2, 3# + */ + npyv_@st@_@sfx@(data_out + vstep * @i@, abc@i@); + /**end repeat3**/ } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1) && - EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_load_pd(data0+@i@), _mm_load_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); - _mm_store_pd(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + /**end repeat2**/ + for (; count > 0; count -= vstep, data0 += vstep, data1 += vstep, data_out += vstep) { + npyv_@sfx@ a = npyv_load_tillz_@sfx@(data0, count); + npyv_@sfx@ b = npyv_load_tillz_@sfx@(data1, count); + npyv_@sfx@ c = npyv_load_tillz_@sfx@(data_out, count); + npyv_store_till_@sfx@(data_out, count, npyv_muladd_@sfx@(a, b, c)); } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), _mm_loadu_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); - _mm_storeu_ps(data_out+@i@, b); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), _mm_loadu_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); - _mm_storeu_pd(data_out+@i@, b); -/**end repeat2**/ + npyv_cleanup(); #else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(@from@(data0[@i@]) * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ -#endif - data0 += 8; - data1 += 8; - data_out += 8; +#ifndef NPY_DISABLE_OPTIMIZATION + for (; count >= 4; count -= 4, data0 += 4, data1 += 4, data_out += 4) { + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + const @type@ a@i@ = @from@(data0[@i@]); + const @type@ b@i@ = @from@(data1[@i@]); + const @type@ c@i@ = @from@(data_out[@i@]); + /**end repeat2**/ + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + const @type@ abc@i@ = a@i@ * b@i@ + c@i@; + /**end repeat2**/ + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + data_out[@i@] = @to@(abc@i@); + /**end repeat2**/ + } +#endif // !NPY_DISABLE_OPTIMIZATION + for (; count > 0; --count, ++data0, ++data1, ++data_out) { + const @type@ a = @from@(*data0); + const @type@ b = @from@(*data1); + const @type@ c = @from@(*data_out); + *data_out = @to@(a * b + c); } +#endif // NPYV check for @type@ - /* Finish off the loop */ - goto finish_after_unrolled_loop; } /* Some extra specializations for the two operand case */ @@ -370,245 +452,26 @@ static void @type@ *data1 = (@type@ *)dataptr[1]; @type@ *data_out = (@type@ *)dataptr[2]; -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, b, value0_sse; -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, b, value0_sse; -#endif - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_stride0_contig_outcontig_two (%d)\n", (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - data_out[@i@] = @to@(value0 * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ - case 0: - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - value0_sse = _mm_set_ps1(value0); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(value0_sse, _mm_load_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); - _mm_store_ps(data_out+@i@, b); -/**end repeat2**/ - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - if (count > 0) { - goto finish_after_unrolled_loop; - } - else { - return; - } - } -#elif EINSUM_USE_SSE2 && @float64@ - value0_sse = _mm_set1_pd(value0); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(value0_sse, _mm_load_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); - _mm_store_pd(data_out+@i@, b); -/**end repeat2**/ - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - if (count > 0) { - goto finish_after_unrolled_loop; - } - else { - return; - } - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(value0_sse, _mm_loadu_ps(data1+@i@)); - b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); - _mm_storeu_ps(data_out+@i@, b); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(value0_sse, _mm_loadu_pd(data1+@i@)); - b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); - _mm_storeu_pd(data_out+@i@, b); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(value0 * - @from@(data1[@i@]) + - @from@(data_out[@i@])); -/**end repeat2**/ -#endif - data1 += 8; - data_out += 8; - } - - /* Finish off the loop */ - if (count > 0) { - goto finish_after_unrolled_loop; - } + @name@_sum_of_products_muladd(data1, data_out, value0, count); + } static void @name@_sum_of_products_contig_stride0_outcontig_two(int nop, char **dataptr, npy_intp const *NPY_UNUSED(strides), npy_intp count) { - @type@ *data0 = (@type@ *)dataptr[0]; @temptype@ value1 = @from@(*(@type@ *)dataptr[1]); + @type@ *data0 = (@type@ *)dataptr[0]; @type@ *data_out = (@type@ *)dataptr[2]; -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, b, value1_sse; -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, b, value1_sse; -#endif - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_stride0_outcontig_two (%d)\n", (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - data_out[@i@] = @to@(@from@(data0[@i@])* - value1 + - @from@(data_out[@i@])); -/**end repeat2**/ - case 0: - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - value1_sse = _mm_set_ps1(value1); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_load_ps(data0+@i@), value1_sse); - b = _mm_add_ps(a, _mm_load_ps(data_out+@i@)); - _mm_store_ps(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - value1_sse = _mm_set1_pd(value1); - - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data_out)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_load_pd(data0+@i@), value1_sse); - b = _mm_add_pd(a, _mm_load_pd(data_out+@i@)); - _mm_store_pd(data_out+@i@, b); -/**end repeat2**/ - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), value1_sse); - b = _mm_add_ps(a, _mm_loadu_ps(data_out+@i@)); - _mm_storeu_ps(data_out+@i@, b); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), value1_sse); - b = _mm_add_pd(a, _mm_loadu_pd(data_out+@i@)); - _mm_storeu_pd(data_out+@i@, b); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - data_out[@i@] = @to@(@from@(data0[@i@])* - value1 + - @from@(data_out[@i@])); -/**end repeat2**/ -#endif - data0 += 8; - data_out += 8; - } - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + @name@_sum_of_products_muladd(data0, data_out, value1, count); } -static void +static NPY_GCC_OPT_3 void @name@_sum_of_products_contig_contig_outstride0_two(int nop, char **dataptr, npy_intp const *NPY_UNUSED(strides), npy_intp count) { @@ -616,432 +479,80 @@ static void @type@ *data1 = (@type@ *)dataptr[1]; @temptype@ accum = 0; -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_contig_outstride0_two (%d)\n", (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - accum += @from@(data0[@i@]) * @from@(data1[@i@]); -/**end repeat2**/ - case 0: - *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum); - return; - } - -#if EINSUM_USE_SSE1 && @float32@ +#if @NPYV_CHK@ // NPYV check for @type@ /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. + const int is_aligned = EINSUM_IS_ALIGNED(data0) && EINSUM_IS_ALIGNED(data1); + const int vstep = npyv_nlanes_@sfx@; + npyv_@sfx@ vaccum = npyv_zero_@sfx@(); + + /**begin repeat2 + * #cond = if(is_aligned), else# + * #ld = loada, load# + * #st = storea, store# + */ + @cond@ { + const npy_intp vstepx4 = vstep * 4; + for (; count >= vstepx4; count -= vstepx4, data0 += vstepx4, data1 += vstepx4) { + /**begin repeat3 + * #i = 0, 1, 2, 3# */ - a = _mm_mul_ps(_mm_load_ps(data0+@i@), _mm_load_ps(data1+@i@)); - accum_sse = _mm_add_ps(accum_sse, a); -/**end repeat2**/ - data0 += 8; - data1 += 8; + npyv_@sfx@ a@i@ = npyv_@ld@_@sfx@(data0 + vstep * @i@); + npyv_@sfx@ b@i@ = npyv_@ld@_@sfx@(data1 + vstep * @i@); + /**end repeat3**/ + npyv_@sfx@ ab3 = npyv_muladd_@sfx@(a3, b3, vaccum); + npyv_@sfx@ ab2 = npyv_muladd_@sfx@(a2, b2, ab3); + npyv_@sfx@ ab1 = npyv_muladd_@sfx@(a1, b1, ab2); + vaccum = npyv_muladd_@sfx@(a0, b0, ab1); } - - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0) && EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_pd(_mm_load_pd(data0+@i@), _mm_load_pd(data1+@i@)); - accum_sse = _mm_add_pd(accum_sse, a); -/**end repeat2**/ - data0 += 8; - data1 += 8; - } - - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + /**end repeat2**/ + for (; count > 0; count -= vstep, data0 += vstep, data1 += vstep) { + npyv_@sfx@ a = npyv_load_tillz_@sfx@(data0, count); + npyv_@sfx@ b = npyv_load_tillz_@sfx@(data1, count); + vaccum = npyv_muladd_@sfx@(a, b, vaccum); } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_ps(_mm_loadu_ps(data0+@i@), _mm_loadu_ps(data1+@i@)); - accum_sse = _mm_add_ps(accum_sse, a); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - _mm_prefetch(data1 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - a = _mm_mul_pd(_mm_loadu_pd(data0+@i@), _mm_loadu_pd(data1+@i@)); - accum_sse = _mm_add_pd(accum_sse, a); -/**end repeat2**/ + accum = npyv_sum_@sfx@(vaccum); + npyv_cleanup(); #else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - accum += @from@(data0[@i@]) * @from@(data1[@i@]); -/**end repeat2**/ -#endif - data0 += 8; - data1 += 8; +#ifndef NPY_DISABLE_OPTIMIZATION + for (; count >= 4; count -= 4, data0 += 4, data1 += 4) { + /**begin repeat2 + * #i = 0, 1, 2, 3# + */ + const @type@ ab@i@ = @from@(data0[@i@]) * @from@(data1[@i@]); + /**end repeat2**/ + accum += ab0 + ab1 + ab2 + ab3; } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; +#endif // !NPY_DISABLE_OPTIMIZATION + for (; count > 0; --count, ++data0, ++data1) { + const @type@ a = @from@(*data0); + const @type@ b = @from@(*data1); + accum += a * b; + } +#endif // NPYV check for @type@ + *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum); } -static void +static NPY_GCC_OPT_3 void @name@_sum_of_products_stride0_contig_outstride0_two(int nop, char **dataptr, npy_intp const *NPY_UNUSED(strides), npy_intp count) { - @temptype@ value0 = @from@(*(@type@ *)dataptr[0]); @type@ *data1 = (@type@ *)dataptr[1]; - @temptype@ accum = 0; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_stride0_contig_outstride0_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - accum += @from@(data1[@i@]); -/**end repeat2**/ - case 0: - *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + value0 * accum); - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data1+@i@)); -/**end repeat2**/ - data1 += 8; - } - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data1)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data1+@i@)); -/**end repeat2**/ - data1 += 8; - } - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data1+@i@)); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data1+@i@)); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - accum += @from@(data1[@i@]); -/**end repeat2**/ -#endif - data1 += 8; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + @temptype@ value0 = @from@(*(@type@ *)dataptr[0]); + @temptype@ accum = @name@_sum_of_arr(data1, count); + *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + value0 * accum); } -static void +static NPY_GCC_OPT_3 void @name@_sum_of_products_contig_stride0_outstride0_two(int nop, char **dataptr, npy_intp const *NPY_UNUSED(strides), npy_intp count) { @type@ *data0 = (@type@ *)dataptr[0]; @temptype@ value1 = @from@(*(@type@ *)dataptr[1]); - @temptype@ accum = 0; - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_stride0_outstride0_two (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: - accum += @from@(data0[@i@]); -/**end repeat2**/ - case 0: - *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + accum * value1); - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data0+@i@)); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data0+@i@)); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - accum += @from@(data0[@i@]); -/**end repeat2**/ -#endif - data0 += 8; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + @temptype@ accum = @name@_sum_of_arr(data0, count); + *(@type@ *)dataptr[2] = @to@(@from@(*(@type@ *)dataptr[2]) + value1 * accum); } #elif @nop@ == 3 && !@complex@ @@ -1143,179 +654,35 @@ static void #if @nop@ == 1 -static void +static NPY_GCC_OPT_3 void @name@_sum_of_products_contig_outstride0_one(int nop, char **dataptr, npy_intp const *strides, npy_intp count) { -#if @complex@ - @temptype@ accum_re = 0, accum_im = 0; - @temptype@ *data0 = (@temptype@ *)dataptr[0]; -#else - @temptype@ accum = 0; - @type@ *data0 = (@type@ *)dataptr[0]; -#endif - -#if EINSUM_USE_SSE1 && @float32@ - __m128 a, accum_sse = _mm_setzero_ps(); -#elif EINSUM_USE_SSE2 && @float64@ - __m128d a, accum_sse = _mm_setzero_pd(); -#endif - - - NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_outstride0_one (%d)\n", - (int)count); - -/* This is placed before the main loop to make small counts faster */ -finish_after_unrolled_loop: - switch (count) { -/**begin repeat2 - * #i = 6, 5, 4, 3, 2, 1, 0# - */ - case @i@+1: + NPY_EINSUM_DBG_PRINT1("@name@_sum_of_products_contig_outstride0_one (%d)\n", (int)count); #if !@complex@ - accum += @from@(data0[@i@]); -#else /* complex */ - accum_re += data0[2*@i@+0]; - accum_im += data0[2*@i@+1]; -#endif -/**end repeat2**/ - case 0: -#if @complex@ - ((@temptype@ *)dataptr[1])[0] += accum_re; - ((@temptype@ *)dataptr[1])[1] += accum_im; + @type@ *data = (@type@ *)dataptr[0]; + @temptype@ accum = @name@_sum_of_arr(data, count); + *((@type@ *)dataptr[1]) = @to@(accum + @from@(*((@type@ *)dataptr[1]))); #else - *((@type@ *)dataptr[1]) = @to@(accum + - @from@(*((@type@ *)dataptr[1]))); -#endif - return; - } - -#if EINSUM_USE_SSE1 && @float32@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_load_ps(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; - } -#elif EINSUM_USE_SSE2 && @float64@ - /* Use aligned instructions if possible */ - if (EINSUM_IS_SSE_ALIGNED(data0)) { - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_load_pd(data0+@i@)); -/**end repeat2**/ - data0 += 8; - } - - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + @temptype@ accum_re = 0, accum_im = 0; + @temptype@ *data0 = (@temptype@ *)dataptr[0]; +#ifndef NPY_DISABLE_OPTIMIZATION + for (; count > 4; count -= 4, data0 += 4*2) { + const @temptype@ re01 = data0[0] + data0[2]; + const @temptype@ re23 = data0[4] + data0[6]; + const @temptype@ im13 = data0[1] + data0[3]; + const @temptype@ im57 = data0[5] + data0[7]; + accum_re += re01 + re23; + accum_im += im13 + im57; } -#endif - - /* Unroll the loop by 8 */ - while (count >= 8) { - count -= 8; - -#if EINSUM_USE_SSE1 && @float32@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 4# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_ps(accum_sse, _mm_loadu_ps(data0+@i@)); -/**end repeat2**/ -#elif EINSUM_USE_SSE2 && @float64@ - _mm_prefetch(data0 + 512, _MM_HINT_T0); - -/**begin repeat2 - * #i = 0, 2, 4, 6# - */ - /* - * NOTE: This accumulation changes the order, so will likely - * produce slightly different results. - */ - accum_sse = _mm_add_pd(accum_sse, _mm_loadu_pd(data0+@i@)); -/**end repeat2**/ -#else -/**begin repeat2 - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ -# if !@complex@ - accum += @from@(data0[@i@]); -# else /* complex */ - accum_re += data0[2*@i@+0]; - accum_im += data0[2*@i@+1]; -# endif -/**end repeat2**/ -#endif - -#if !@complex@ - data0 += 8; -#else - data0 += 8*2; -#endif +#endif // !NPY_DISABLE_OPTIMIZATION + for (; count > 0; --count, data0 += 2) { + accum_re += data0[0]; + accum_im += data0[1]; } - -#if EINSUM_USE_SSE1 && @float32@ - /* Add the four SSE values and put in accum */ - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(2,3,0,1)); - accum_sse = _mm_add_ps(a, accum_sse); - a = _mm_shuffle_ps(accum_sse, accum_sse, _MM_SHUFFLE(1,0,3,2)); - accum_sse = _mm_add_ps(a, accum_sse); - _mm_store_ss(&accum, accum_sse); -#elif EINSUM_USE_SSE2 && @float64@ - /* Add the two SSE2 values and put in accum */ - a = _mm_shuffle_pd(accum_sse, accum_sse, _MM_SHUFFLE2(0,1)); - accum_sse = _mm_add_pd(a, accum_sse); - _mm_store_sd(&accum, accum_sse); -#endif - - /* Finish off the loop */ - goto finish_after_unrolled_loop; + ((@temptype@ *)dataptr[1])[0] += accum_re; + ((@temptype@ *)dataptr[1])[1] += accum_im; +#endif // !@complex@ } #endif /* @nop@ == 1 */ diff --git a/numpy/core/src/multiarray/item_selection.c b/numpy/core/src/multiarray/item_selection.c index 8052e24e424b..fb354ce5473a 100644 --- a/numpy/core/src/multiarray/item_selection.c +++ b/numpy/core/src/multiarray/item_selection.c @@ -4,6 +4,7 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION #define _MULTIARRAYMODULE + #include "numpy/arrayobject.h" #include "numpy/arrayscalars.h" @@ -27,7 +28,7 @@ #include "alloc.h" #include "arraytypes.h" #include "array_coercion.h" - +#include "simd/simd.h" static NPY_GCC_OPT_3 NPY_INLINE int npy_fasttake_impl( @@ -576,6 +577,10 @@ PyArray_PutMask(PyArrayObject *self, PyObject* values0, PyObject* mask0) return NULL; } + if (PyArray_FailUnlessWriteable(self, "putmask: output array") < 0) { + return NULL; + } + mask = (PyArrayObject *)PyArray_FROM_OTF(mask0, NPY_BOOL, NPY_ARRAY_CARRAY | NPY_ARRAY_FORCECAST); if (mask == NULL) { @@ -2124,22 +2129,199 @@ count_nonzero_bytes_384(const npy_uint64 * w) return r; } +#if NPY_SIMD +/* Count the zero bytes between `*d` and `end`, updating `*d` to point to where to keep counting from. */ +static NPY_INLINE NPY_GCC_OPT_3 npyv_u8 +count_zero_bytes_u8(const npy_uint8 **d, const npy_uint8 *end, npy_uint8 max_count) +{ + const npyv_u8 vone = npyv_setall_u8(1); + const npyv_u8 vzero = npyv_zero_u8(); + + npy_intp lane_max = 0; + npyv_u8 vsum8 = npyv_zero_u8(); + while (*d < end && lane_max <= max_count - 1) { + // we count zeros because `cmpeq` cheaper than `cmpneq` for most archs + npyv_u8 vt = npyv_cvt_u8_b8(npyv_cmpeq_u8(npyv_load_u8(*d), vzero)); + vt = npyv_and_u8(vt, vone); + vsum8 = npyv_add_u8(vsum8, vt); + *d += npyv_nlanes_u8; + lane_max += 1; + } + return vsum8; +} + +static NPY_INLINE NPY_GCC_OPT_3 npyv_u16x2 +count_zero_bytes_u16(const npy_uint8 **d, const npy_uint8 *end, npy_uint16 max_count) +{ + npyv_u16x2 vsum16; + vsum16.val[0] = vsum16.val[1] = npyv_zero_u16(); + npy_intp lane_max = 0; + while (*d < end && lane_max <= max_count - NPY_MAX_UINT8) { + npyv_u8 vsum8 = count_zero_bytes_u8(d, end, NPY_MAX_UINT8); + npyv_u16x2 part = npyv_expand_u16_u8(vsum8); + vsum16.val[0] = npyv_add_u16(vsum16.val[0], part.val[0]); + vsum16.val[1] = npyv_add_u16(vsum16.val[1], part.val[1]); + lane_max += NPY_MAX_UINT8; + } + return vsum16; +} +#endif // NPY_SIMD +/* + * Counts the number of non-zero values in a raw array. + * The one loop process is shown below(take SSE2 with 128bits vector for example): + * |------------16 lanes---------| + *[vsum8] 255 255 255 ... 255 255 255 255 count_zero_bytes_u8: counting 255*16 elements + * !! + * |------------8 lanes---------| + *[vsum16] 65535 65535 65535 ... 65535 count_zero_bytes_u16: counting (2*16-1)*16 elements + * 65535 65535 65535 ... 65535 + * !! + * |------------4 lanes---------| + *[sum_32_0] 65535 65535 65535 65535 count_nonzero_bytes + * 65535 65535 65535 65535 + *[sum_32_1] 65535 65535 65535 65535 + * 65535 65535 65535 65535 + * !! + * (2*16-1)*16 +*/ +static NPY_INLINE NPY_GCC_OPT_3 npy_intp +count_nonzero_u8(const char *data, npy_intp bstride, npy_uintp len) +{ + npy_intp count = 0; + if (bstride == 1) { + #if NPY_SIMD + npy_uintp len_m = len & -npyv_nlanes_u8; + npy_uintp zcount = 0; + for (const char *end = data + len_m; data < end;) { + npyv_u16x2 vsum16 = count_zero_bytes_u16((const npy_uint8**)&data, (const npy_uint8*)end, NPY_MAX_UINT16); + npyv_u32x2 sum_32_0 = npyv_expand_u32_u16(vsum16.val[0]); + npyv_u32x2 sum_32_1 = npyv_expand_u32_u16(vsum16.val[1]); + zcount += npyv_sum_u32(npyv_add_u32( + npyv_add_u32(sum_32_0.val[0], sum_32_0.val[1]), + npyv_add_u32(sum_32_1.val[0], sum_32_1.val[1]) + )); + } + len -= len_m; + count = len_m - zcount; + #else + if (!NPY_ALIGNMENT_REQUIRED || npy_is_aligned(data, sizeof(npy_uint64))) { + int step = 6 * sizeof(npy_uint64); + int left_bytes = len % step; + for (const char *end = data + len; data < end - left_bytes; data += step) { + count += count_nonzero_bytes_384((const npy_uint64 *)data); + } + len = left_bytes; + } + #endif // NPY_SIMD + } + for (; len > 0; --len, data += bstride) { + count += (*data != 0); + } + return count; +} + +static NPY_INLINE NPY_GCC_OPT_3 npy_intp +count_nonzero_u16(const char *data, npy_intp bstride, npy_uintp len) +{ + npy_intp count = 0; +#if NPY_SIMD + if (bstride == sizeof(npy_uint16)) { + npy_uintp zcount = 0, len_m = len & -npyv_nlanes_u16; + const npyv_u16 vone = npyv_setall_u16(1); + const npyv_u16 vzero = npyv_zero_u16(); + + for (npy_uintp lenx = len_m; lenx > 0;) { + npyv_u16 vsum16 = npyv_zero_u16(); + npy_uintp max16 = PyArray_MIN(lenx, NPY_MAX_UINT16*npyv_nlanes_u16); + + for (const char *end = data + max16*bstride; data < end; data += NPY_SIMD_WIDTH) { + npyv_u16 mask = npyv_cvt_u16_b16(npyv_cmpeq_u16(npyv_load_u16((npy_uint16*)data), vzero)); + mask = npyv_and_u16(mask, vone); + vsum16 = npyv_add_u16(vsum16, mask); + } + lenx -= max16; + zcount += npyv_sumup_u16(vsum16); + } + len -= len_m; + count = len_m - zcount; + } +#endif + for (; len > 0; --len, data += bstride) { + count += (*(npy_uint16*)data != 0); + } + return count; +} + +static NPY_INLINE NPY_GCC_OPT_3 npy_intp +count_nonzero_u32(const char *data, npy_intp bstride, npy_uintp len) +{ + npy_intp count = 0; +#if NPY_SIMD + if (bstride == sizeof(npy_uint32)) { + const npy_uintp max_iter = NPY_MAX_UINT32*npyv_nlanes_u32; + const npy_uintp len_m = (len > max_iter ? max_iter : len) & -npyv_nlanes_u32; + const npyv_u32 vone = npyv_setall_u32(1); + const npyv_u32 vzero = npyv_zero_u32(); + + npyv_u32 vsum32 = npyv_zero_u32(); + for (const char *end = data + len_m*bstride; data < end; data += NPY_SIMD_WIDTH) { + npyv_u32 mask = npyv_cvt_u32_b32(npyv_cmpeq_u32(npyv_load_u32((npy_uint32*)data), vzero)); + mask = npyv_and_u32(mask, vone); + vsum32 = npyv_add_u32(vsum32, mask); + } + const npyv_u32 maskevn = npyv_reinterpret_u32_u64(npyv_setall_u64(0xffffffffULL)); + npyv_u64 odd = npyv_shri_u64(npyv_reinterpret_u64_u32(vsum32), 32); + npyv_u64 even = npyv_reinterpret_u64_u32(npyv_and_u32(vsum32, maskevn)); + count = len_m - npyv_sum_u64(npyv_add_u64(odd, even)); + len -= len_m; + } +#endif + for (; len > 0; --len, data += bstride) { + count += (*(npy_uint32*)data != 0); + } + return count; +} + +static NPY_INLINE NPY_GCC_OPT_3 npy_intp +count_nonzero_u64(const char *data, npy_intp bstride, npy_uintp len) +{ + npy_intp count = 0; +#if NPY_SIMD + if (bstride == sizeof(npy_uint64)) { + const npy_uintp len_m = len & -npyv_nlanes_u64; + const npyv_u64 vone = npyv_setall_u64(1); + const npyv_u64 vzero = npyv_zero_u64(); + + npyv_u64 vsum64 = npyv_zero_u64(); + for (const char *end = data + len_m*bstride; data < end; data += NPY_SIMD_WIDTH) { + npyv_u64 mask = npyv_cvt_u64_b64(npyv_cmpeq_u64(npyv_load_u64((npy_uint64*)data), vzero)); + mask = npyv_and_u64(mask, vone); + vsum64 = npyv_add_u64(vsum64, mask); + } + len -= len_m; + count = len_m - npyv_sum_u64(vsum64); + } +#endif + for (; len > 0; --len, data += bstride) { + count += (*(npy_uint64*)data != 0); + } + return count; +} /* * Counts the number of True values in a raw boolean array. This * is a low-overhead function which does no heap allocations. * * Returns -1 on error. */ -NPY_NO_EXPORT npy_intp -count_boolean_trues(int ndim, char *data, npy_intp const *ashape, npy_intp const *astrides) +static NPY_GCC_OPT_3 npy_intp +count_nonzero_int(int ndim, char *data, const npy_intp *ashape, const npy_intp *astrides, int elsize) { + assert(elsize <= 8); int idim; npy_intp shape[NPY_MAXDIMS], strides[NPY_MAXDIMS]; - npy_intp i, coord[NPY_MAXDIMS]; - npy_intp count = 0; - NPY_BEGIN_THREADS_DEF; + npy_intp coord[NPY_MAXDIMS]; - /* Use raw iteration with no heap memory allocation */ + // Use raw iteration with no heap memory allocation if (PyArray_PrepareOneRawArrayIter( ndim, ashape, data, astrides, @@ -2148,46 +2330,44 @@ count_boolean_trues(int ndim, char *data, npy_intp const *ashape, npy_intp const return -1; } - /* Handle zero-sized array */ + // Handle zero-sized array if (shape[0] == 0) { return 0; } + NPY_BEGIN_THREADS_DEF; NPY_BEGIN_THREADS_THRESHOLDED(shape[0]); - /* Special case for contiguous inner loop */ - if (strides[0] == 1) { - NPY_RAW_ITER_START(idim, ndim, coord, shape) { - /* Process the innermost dimension */ - const char *d = data; - const char *e = data + shape[0]; - if (NPY_CPU_HAVE_UNALIGNED_ACCESS || - npy_is_aligned(d, sizeof(npy_uint64))) { - npy_uintp stride = 6 * sizeof(npy_uint64); - for (; d < e - (shape[0] % stride); d += stride) { - count += count_nonzero_bytes_384((const npy_uint64 *)d); - } - } - for (; d < e; ++d) { - count += (*d != 0); - } - } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape, data, strides); - } - /* General inner loop */ - else { - NPY_RAW_ITER_START(idim, ndim, coord, shape) { - char *d = data; - /* Process the innermost dimension */ - for (i = 0; i < shape[0]; ++i, d += strides[0]) { - count += (*d != 0); - } - } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape, data, strides); + #define NONZERO_CASE(LEN, SFX) \ + case LEN: \ + NPY_RAW_ITER_START(idim, ndim, coord, shape) { \ + count += count_nonzero_##SFX(data, strides[0], shape[0]); \ + } NPY_RAW_ITER_ONE_NEXT(idim, ndim, coord, shape, data, strides); \ + break + + npy_intp count = 0; + switch(elsize) { + NONZERO_CASE(1, u8); + NONZERO_CASE(2, u16); + NONZERO_CASE(4, u32); + NONZERO_CASE(8, u64); } + #undef NONZERO_CASE NPY_END_THREADS; - return count; } +/* + * Counts the number of True values in a raw boolean array. This + * is a low-overhead function which does no heap allocations. + * + * Returns -1 on error. + */ +NPY_NO_EXPORT NPY_GCC_OPT_3 npy_intp +count_boolean_trues(int ndim, char *data, npy_intp const *ashape, npy_intp const *astrides) +{ + return count_nonzero_int(ndim, data, ashape, astrides, 1); +} /*NUMPY_API * Counts the number of non-zero elements in the array. @@ -2210,14 +2390,22 @@ PyArray_CountNonzero(PyArrayObject *self) npy_intp *strideptr, *innersizeptr; NPY_BEGIN_THREADS_DEF; - /* Special low-overhead version specific to the boolean type */ + // Special low-overhead version specific to the boolean/int types dtype = PyArray_DESCR(self); - if (dtype->type_num == NPY_BOOL) { - return count_boolean_trues(PyArray_NDIM(self), PyArray_DATA(self), - PyArray_DIMS(self), PyArray_STRIDES(self)); + switch(dtype->kind) { + case 'u': + case 'i': + case 'b': + if (dtype->elsize > 8) { + break; + } + return count_nonzero_int( + PyArray_NDIM(self), PyArray_BYTES(self), PyArray_DIMS(self), + PyArray_STRIDES(self), dtype->elsize + ); } - nonzero = PyArray_DESCR(self)->f->nonzero; + nonzero = PyArray_DESCR(self)->f->nonzero; /* If it's a trivial one-dimensional loop, don't use an iterator */ if (PyArray_TRIVIALLY_ITERABLE(self)) { needs_api = PyDataType_FLAGCHK(dtype, NPY_NEEDS_PYAPI); diff --git a/numpy/core/src/multiarray/legacy_dtype_implementation.c b/numpy/core/src/multiarray/legacy_dtype_implementation.c index 3ce4710fddb4..9b4946da3c7c 100644 --- a/numpy/core/src/multiarray/legacy_dtype_implementation.c +++ b/numpy/core/src/multiarray/legacy_dtype_implementation.c @@ -1,10 +1,10 @@ /* - * This file hosts legacy implementations of certain functions for - * which alternatives exists, but the old functions are still required - * in certain code paths, or until the code transition is finalized. + * The only function exported here is `PyArray_LegacyCanCastTypeTo`, which + * is currently still in use when first registering a userdtype. * - * This code should typically not require modification, and if modified - * similar changes may be necessary in the new version. + * The extremely limited use means that it can probably remain unmaintained + * until such a time where legay user dtypes are deprecated and removed + * entirely. */ #define NPY_NO_DEPRECATED_API NPY_API_VERSION @@ -78,7 +78,7 @@ _equivalent_subarrays(PyArray_ArrayDescr *sub1, PyArray_ArrayDescr *sub2) } -NPY_NO_EXPORT unsigned char +static unsigned char PyArray_LegacyEquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) { int type_num1, type_num2, size1, size2; @@ -116,7 +116,7 @@ PyArray_LegacyEquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) } -NPY_NO_EXPORT unsigned char +static unsigned char PyArray_LegacyEquivTypenums(int typenum1, int typenum2) { PyArray_Descr *d1, *d2; @@ -135,7 +135,7 @@ PyArray_LegacyEquivTypenums(int typenum1, int typenum2) } -NPY_NO_EXPORT int +static int PyArray_LegacyCanCastSafely(int fromtype, int totype) { PyArray_Descr *from; @@ -161,15 +161,17 @@ PyArray_LegacyCanCastSafely(int fromtype, int totype) while (*curtype != NPY_NOTYPE) { if (*curtype++ == totype) { + Py_DECREF(from); return 1; } } } + Py_DECREF(from); return 0; } -NPY_NO_EXPORT npy_bool +static npy_bool PyArray_LegacyCanCastTo(PyArray_Descr *from, PyArray_Descr *to) { int from_type_num = from->type_num; @@ -549,168 +551,3 @@ PyArray_LegacyCanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, } } - -/* - * Legacy function to find the correct dtype when casting from any built-in - * dtype to NPY_STRING, NPY_UNICODE, NPY_VOID, and NPY_DATETIME with generic - * units. - * - * This function returns a dtype based on flex_dtype and the values in - * data_dtype. It also calls Py_DECREF on the flex_dtype. If the - * flex_dtype is not flexible, it returns it as-is. - * - * Usually, if data_obj is not an array, dtype should be the result - * given by the PyArray_GetArrayParamsFromObject function. - * - * If *flex_dtype is NULL, returns immediately, without setting an - * exception, leaving any previous error handling intact. - */ -NPY_NO_EXPORT PyArray_Descr * -PyArray_AdaptFlexibleDType(PyArray_Descr *data_dtype, PyArray_Descr *flex_dtype) -{ - PyArray_DatetimeMetaData *meta; - PyArray_Descr *retval = NULL; - int flex_type_num; - - if (flex_dtype == NULL) { - return retval; - } - - flex_type_num = flex_dtype->type_num; - - /* Flexible types with expandable size */ - if (PyDataType_ISUNSIZED(flex_dtype)) { - /* First replace the flex_dtype */ - retval = PyArray_DescrNew(flex_dtype); - Py_DECREF(flex_dtype); - if (retval == NULL) { - return retval; - } - - if (data_dtype->type_num == flex_type_num || - flex_type_num == NPY_VOID) { - (retval)->elsize = data_dtype->elsize; - } - else if (flex_type_num == NPY_STRING || flex_type_num == NPY_UNICODE) { - npy_intp size = 8; - - /* - * Get a string-size estimate of the input. These - * are generallly the size needed, rounded up to - * a multiple of eight. - */ - switch (data_dtype->type_num) { - case NPY_BOOL: - case NPY_UBYTE: - case NPY_BYTE: - case NPY_USHORT: - case NPY_SHORT: - case NPY_UINT: - case NPY_INT: - case NPY_ULONG: - case NPY_LONG: - case NPY_ULONGLONG: - case NPY_LONGLONG: - if (data_dtype->kind == 'b') { - /* 5 chars needed for cast to 'True' or 'False' */ - size = 5; - } - else if (data_dtype->elsize > 8 || - data_dtype->elsize < 0) { - /* - * Element size should never be greater than 8 or - * less than 0 for integer type, but just in case... - */ - break; - } - else if (data_dtype->kind == 'u') { - size = REQUIRED_STR_LEN[data_dtype->elsize]; - } - else if (data_dtype->kind == 'i') { - /* Add character for sign symbol */ - size = REQUIRED_STR_LEN[data_dtype->elsize] + 1; - } - break; - case NPY_HALF: - case NPY_FLOAT: - case NPY_DOUBLE: - size = 32; - break; - case NPY_LONGDOUBLE: - size = 48; - break; - case NPY_CFLOAT: - case NPY_CDOUBLE: - size = 2 * 32; - break; - case NPY_CLONGDOUBLE: - size = 2 * 48; - break; - case NPY_OBJECT: - size = 64; - break; - case NPY_STRING: - case NPY_VOID: - size = data_dtype->elsize; - break; - case NPY_UNICODE: - size = data_dtype->elsize / 4; - break; - case NPY_DATETIME: - meta = get_datetime_metadata_from_dtype(data_dtype); - if (meta == NULL) { - Py_DECREF(retval); - return NULL; - } - size = get_datetime_iso_8601_strlen(0, meta->base); - break; - case NPY_TIMEDELTA: - size = 21; - break; - } - - if (flex_type_num == NPY_STRING) { - retval->elsize = size; - } - else if (flex_type_num == NPY_UNICODE) { - retval->elsize = size * 4; - } - } - else { - /* - * We should never get here, but just in case someone adds - * a new flex dtype... - */ - PyErr_SetString(PyExc_TypeError, - "don't know how to adapt flex dtype"); - Py_DECREF(retval); - return NULL; - } - } - /* Flexible type with generic time unit that adapts */ - else if (flex_type_num == NPY_DATETIME || - flex_type_num == NPY_TIMEDELTA) { - meta = get_datetime_metadata_from_dtype(flex_dtype); - retval = flex_dtype; - if (meta == NULL) { - return NULL; - } - - if (meta->base == NPY_FR_GENERIC) { - if (data_dtype->type_num == NPY_DATETIME || - data_dtype->type_num == NPY_TIMEDELTA) { - meta = get_datetime_metadata_from_dtype(data_dtype); - if (meta == NULL) { - return NULL; - } - - retval = create_datetime_dtype(flex_type_num, meta); - Py_DECREF(flex_dtype); - } - } - } - else { - retval = flex_dtype; - } - return retval; -} diff --git a/numpy/core/src/multiarray/legacy_dtype_implementation.h b/numpy/core/src/multiarray/legacy_dtype_implementation.h index ca171d7733e6..b36eb019a452 100644 --- a/numpy/core/src/multiarray/legacy_dtype_implementation.h +++ b/numpy/core/src/multiarray/legacy_dtype_implementation.h @@ -1,40 +1,8 @@ #ifndef _NPY_LEGACY_DTYPE_IMPLEMENTATION_H #define _NPY_LEGACY_DTYPE_IMPLEMENTATION_H - -NPY_NO_EXPORT unsigned char -PyArray_LegacyEquivTypes(PyArray_Descr *type1, PyArray_Descr *type2); - -NPY_NO_EXPORT unsigned char -PyArray_LegacyEquivTypenums(int typenum1, int typenum2); - -NPY_NO_EXPORT int -PyArray_LegacyCanCastSafely(int fromtype, int totype); - -NPY_NO_EXPORT npy_bool -PyArray_LegacyCanCastTo(PyArray_Descr *from, PyArray_Descr *to); - NPY_NO_EXPORT npy_bool PyArray_LegacyCanCastTypeTo(PyArray_Descr *from, PyArray_Descr *to, NPY_CASTING casting); -/* - * This function calls Py_DECREF on flex_dtype, and replaces it with - * a new dtype that has been adapted based on the values in data_dtype - * and data_obj. If the flex_dtype is not flexible, it returns it as-is. - * - * Usually, if data_obj is not an array, dtype should be the result - * given by the PyArray_GetArrayParamsFromObject function. - * - * The data_obj may be NULL if just a dtype is known for the source. - * - * If *flex_dtype is NULL, returns immediately, without setting an - * exception, leaving any previous error handling intact. - * - * The current flexible dtypes include NPY_STRING, NPY_UNICODE, NPY_VOID, - * and NPY_DATETIME with generic units. - */ -NPY_NO_EXPORT PyArray_Descr * -PyArray_AdaptFlexibleDType(PyArray_Descr *data_dtype, PyArray_Descr *flex_dtype); - #endif /*_NPY_LEGACY_DTYPE_IMPLEMENTATION_H*/ diff --git a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src index 0590558be2f5..631042dae1d7 100644 --- a/numpy/core/src/multiarray/lowlevel_strided_loops.c.src +++ b/numpy/core/src/multiarray/lowlevel_strided_loops.c.src @@ -19,6 +19,8 @@ #include "lowlevel_strided_loops.h" #include "array_assign.h" +#include "array_method.h" +#include "usertypes.h" /* @@ -29,7 +31,7 @@ * instructions (16 byte). * So this flag can only be enabled if autovectorization is disabled. */ -#if NPY_CPU_HAVE_UNALIGNED_ACCESS +#if NPY_ALIGNMENT_REQUIRED # define NPY_USE_UNALIGNED_ACCESS 0 #else # define NPY_USE_UNALIGNED_ACCESS 0 @@ -114,11 +116,20 @@ static int #if @is_aligned@ && @is_swap@ == 0 && @elsize@ <= NPY_SIZEOF_INTP NPY_GCC_UNROLL_LOOPS #endif -@prefix@_@oper@_size@elsize@(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) +@prefix@_@oper@_size@elsize@( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; +#if !@src_contig@ + npy_intp src_stride = strides[0]; +#endif +#if !@dst_contig@ + npy_intp dst_stride = strides[1]; +#endif + #if @is_aligned@ /* sanity check */ assert(N == 0 || npy_is_aligned(dst, _UINT_ALIGN(@type@))); @@ -184,12 +195,17 @@ static int */ #if (@src_contig@ == 0) && @is_aligned@ static NPY_GCC_OPT_3 int -@prefix@_@oper@_size@elsize@_srcstride0(char *dst, - npy_intp dst_stride, - char *src, npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) +@prefix@_@oper@_size@elsize@_srcstride0( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(auxdata)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; +#if !@dst_contig@ + npy_intp dst_stride = strides[1]; +#endif + #if @elsize@ != 16 # if !(@elsize@ == 1 && @dst_contig@) @type@ temp; @@ -250,11 +266,16 @@ static NPY_GCC_OPT_3 int /**end repeat**/ static int -_strided_to_strided(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *NPY_UNUSED(data)) +_strided_to_strided( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(data)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + while (N > 0) { memmove(dst, src, src_itemsize); dst += dst_stride; @@ -264,12 +285,22 @@ _strided_to_strided(char *dst, npy_intp dst_stride, return 0; } +/* + * NOTE: This function is currently unused. It would currently be used for + * builtin dtypes that have an elsize other than 2, 4, 8, or 16 bytes. + * Since unicode and complex swap differently, no such dtype exists. + */ static int -_swap_strided_to_strided(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *NPY_UNUSED(data)) +_swap_strided_to_strided( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(data)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + char *a, *b, c; while (N > 0) { @@ -291,11 +322,16 @@ _swap_strided_to_strided(char *dst, npy_intp dst_stride, } static int -_swap_pair_strided_to_strided(char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp src_itemsize, - NpyAuxData *NPY_UNUSED(data)) +_swap_pair_strided_to_strided( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(data)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_stride = strides[0], dst_stride = strides[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + char *a, *b, c; npy_intp itemsize_half = src_itemsize / 2; @@ -327,17 +363,21 @@ _swap_pair_strided_to_strided(char *dst, npy_intp dst_stride, } static int -_contig_to_contig(char *dst, npy_intp NPY_UNUSED(dst_stride), - char *src, npy_intp NPY_UNUSED(src_stride), - npy_intp N, npy_intp src_itemsize, - NpyAuxData *NPY_UNUSED(data)) +_contig_to_contig( + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *NPY_UNUSED(strides), + NpyAuxData *NPY_UNUSED(data)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; + npy_intp src_itemsize = context->descriptors[0]->elsize; + memmove(dst, src, src_itemsize*N); return 0; } -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * PyArray_GetStridedCopyFn(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize) { @@ -491,7 +531,7 @@ PyArray_GetStridedCopyFn(int aligned, npy_intp src_stride, * #not_pair = 1, 0# */ -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * @function@(int aligned, npy_intp src_stride, npy_intp dst_stride, npy_intp itemsize) { @@ -795,11 +835,16 @@ NPY_NO_EXPORT PyArray_StridedUnaryOp * static NPY_GCC_OPT_3 int @prefix@_cast_@name1@_to_@name2@( - char *dst, npy_intp dst_stride, - char *src, npy_intp src_stride, - npy_intp N, npy_intp NPY_UNUSED(src_itemsize), - NpyAuxData *NPY_UNUSED(data)) + PyArrayMethod_Context *context, char *const *args, + const npy_intp *dimensions, const npy_intp *strides, + NpyAuxData *NPY_UNUSED(data)) { + npy_intp N = dimensions[0]; + char *src = args[0], *dst = args[1]; +#if !@contig@ + npy_intp src_stride = strides[0], dst_stride = strides[1]; +#endif + #if @is_complex1@ _TYPE1 src_value[2]; #elif !@aligned@ @@ -894,7 +939,7 @@ static NPY_GCC_OPT_3 int /**end repeat**/ -NPY_NO_EXPORT PyArray_StridedUnaryOp * +NPY_NO_EXPORT PyArrayMethod_StridedLoop * PyArray_GetStridedNumericCastFn(int aligned, npy_intp src_stride, npy_intp dst_stride, int src_type_num, int dst_type_num) @@ -985,8 +1030,7 @@ PyArray_TransferNDimToStrided(npy_intp ndim, npy_intp const *coords, npy_intp coords_inc, npy_intp const *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, - PyArray_StridedUnaryOp *stransfer, - NpyAuxData *data) + NPY_cast_info *cast_info) { npy_intp i, M, N, coord0, shape0, src_stride0, coord1, shape1, src_stride1; @@ -995,12 +1039,17 @@ PyArray_TransferNDimToStrided(npy_intp ndim, shape0 = shape[0]; src_stride0 = src_strides[0]; N = shape0 - coord0; + + npy_intp strides[2] = {src_stride0, dst_stride}; + + char *args[2] = {src, dst}; if (N >= count) { - return stransfer(dst, dst_stride, src, src_stride0, - count, src_itemsize, data); + return cast_info->func(&cast_info->context, + args, &count, strides, cast_info->auxdata); } - int res = stransfer(dst, dst_stride, src, src_stride0, - N, src_itemsize, data); + int res = cast_info->func(&cast_info->context, + args, &N, strides, cast_info->auxdata); + if (res < 0) { return -1; } @@ -1022,13 +1071,14 @@ PyArray_TransferNDimToStrided(npy_intp ndim, M = (shape1 - coord1 - 1); N = shape0*M; for (i = 0; i < M; ++i) { + args[0] = src; args[1] = dst; if (shape0 >= count) { - return stransfer(dst, dst_stride, src, src_stride0, - count, src_itemsize, data); + return cast_info->func(&cast_info->context, + args, &count, strides, cast_info->auxdata); } else { - res = stransfer(dst, dst_stride, src, src_stride0, - shape0, src_itemsize, data); + res = cast_info->func(&cast_info->context, + args, &shape0, strides, cast_info->auxdata); if (res < 0) { return -1; } @@ -1085,13 +1135,14 @@ PyArray_TransferNDimToStrided(npy_intp ndim, /* A loop for dimensions 0 and 1 */ for (i = 0; i < shape1; ++i) { + args[0] = src; args[1] = dst; if (shape0 >= count) { - return stransfer(dst, dst_stride, src, src_stride0, - count, src_itemsize, data); + return cast_info->func(&cast_info->context, + args, &count, strides, cast_info->auxdata); } else { - res = stransfer(dst, dst_stride, src, src_stride0, - shape0, src_itemsize, data); + res = cast_info->func(&cast_info->context, + args, &shape0, strides, cast_info->auxdata); if (res < 0) { return -1; } @@ -1112,8 +1163,7 @@ PyArray_TransferStridedToNDim(npy_intp ndim, npy_intp const *coords, npy_intp coords_inc, npy_intp const *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, - PyArray_StridedUnaryOp *stransfer, - NpyAuxData *data) + NPY_cast_info *cast_info) { npy_intp i, M, N, coord0, shape0, dst_stride0, coord1, shape1, dst_stride1; @@ -1122,12 +1172,16 @@ PyArray_TransferStridedToNDim(npy_intp ndim, shape0 = shape[0]; dst_stride0 = dst_strides[0]; N = shape0 - coord0; + + npy_intp strides[2] = {src_stride, dst_stride0}; + + char *args[2] = {src, dst}; if (N >= count) { - return stransfer(dst, dst_stride0, src, src_stride, - count, src_itemsize, data); + return cast_info->func(&cast_info->context, + args, &count, strides, cast_info->auxdata); } - int res = stransfer(dst, dst_stride0, src, src_stride, - N, src_itemsize, data); + int res = cast_info->func(&cast_info->context, + args, &N, strides, cast_info->auxdata); if (res < 0) { return -1; } @@ -1149,13 +1203,14 @@ PyArray_TransferStridedToNDim(npy_intp ndim, M = (shape1 - coord1 - 1); N = shape0*M; for (i = 0; i < M; ++i) { + args[0] = src; args[1] = dst; if (shape0 >= count) { - return stransfer(dst, dst_stride0, src, src_stride, - count, src_itemsize, data); + return cast_info->func(&cast_info->context, + args, &count, strides, cast_info->auxdata); } else { - res = stransfer(dst, dst_stride0, src, src_stride, - shape0, src_itemsize, data); + res = cast_info->func(&cast_info->context, + args, &shape0, strides, cast_info->auxdata); if (res < 0) { return -1; } @@ -1212,13 +1267,14 @@ PyArray_TransferStridedToNDim(npy_intp ndim, /* A loop for dimensions 0 and 1 */ for (i = 0; i < shape1; ++i) { + args[0] = src; args[1] = dst; if (shape0 >= count) { - return stransfer(dst, dst_stride0, src, src_stride, - count, src_itemsize, data); + return cast_info->func(&cast_info->context, + args, &count, strides, cast_info->auxdata); } else { - res = stransfer(dst, dst_stride0, src, src_stride, - shape0, src_itemsize, data); + res = cast_info->func(&cast_info->context, + args, &shape0, strides, cast_info->auxdata); if (res < 0) { return -1; } @@ -1240,26 +1296,27 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, npy_intp const *coords, npy_intp coords_inc, npy_intp const *shape, npy_intp shape_inc, npy_intp count, npy_intp src_itemsize, - PyArray_MaskedStridedUnaryOp *stransfer, - NpyAuxData *data) + NPY_cast_info *cast_info) { npy_intp i, M, N, coord0, shape0, dst_stride0, coord1, shape1, dst_stride1; + PyArray_MaskedStridedUnaryOp *stransfer = + (PyArray_MaskedStridedUnaryOp*)cast_info->func; /* Finish off dimension 0 */ coord0 = coords[0]; shape0 = shape[0]; dst_stride0 = dst_strides[0]; N = shape0 - coord0; + + npy_intp strides[2] = {src_stride, dst_stride0}; + + char *args[2] = {src, dst}; if (N >= count) { - return stransfer( - dst, dst_stride0, src, src_stride, - mask, mask_stride, - count, src_itemsize, data); + return stransfer(&cast_info->context, + args, &count, strides, mask, mask_stride, cast_info->auxdata); } - int res = stransfer( - dst, dst_stride0, src, src_stride, - mask, mask_stride, - N, src_itemsize, data); + int res = stransfer(&cast_info->context, + args, &count, strides, mask, mask_stride, cast_info->auxdata); if (res < 0) { return -1; } @@ -1282,17 +1339,16 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, M = (shape1 - coord1 - 1); N = shape0*M; for (i = 0; i < M; ++i) { + args[0] = src; args[1] = dst; if (shape0 >= count) { - return stransfer( - dst, dst_stride0, src, src_stride, - mask, mask_stride, - count, src_itemsize, data); + return stransfer(&cast_info->context, + args, &count, strides, + mask, mask_stride, cast_info->auxdata); } else { - int res = stransfer( - dst, dst_stride0, src, src_stride, - mask, mask_stride, - shape0, src_itemsize, data); + int res = stransfer(&cast_info->context, + args, &shape0, strides, + mask, mask_stride, cast_info->auxdata); if (res < 0) { return -1; } @@ -1350,17 +1406,16 @@ PyArray_TransferMaskedStridedToNDim(npy_intp ndim, /* A loop for dimensions 0 and 1 */ for (i = 0; i < shape1; ++i) { + args[0] = src; args[1] = dst; if (shape0 >= count) { - return stransfer( - dst, dst_stride0, src, src_stride, - mask, mask_stride, - count, src_itemsize, data); + return stransfer(&cast_info->context, + args, &count, strides, mask, + mask_stride, cast_info->auxdata); } else { - res = stransfer( - dst, dst_stride0, src, src_stride, - mask, mask_stride, - shape0, src_itemsize, data); + int res = stransfer(&cast_info->context, + args, &shape0, strides, + mask, mask_stride, cast_info->auxdata); if (res < 0) { return -1; } @@ -1657,22 +1712,15 @@ mapiter_@name@(PyArrayMapIterObject *mit) npy_intp reset_offsets[2] = {0, 0}; /* Use strided transfer functions for the inner loop */ - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; npy_intp fixed_strides[2]; -#if @isget@ - npy_intp src_itemsize = PyArray_ITEMSIZE(array); -#else - npy_intp src_itemsize = PyArray_ITEMSIZE(mit->extra_op); -#endif - /* * Get a dtype transfer function, since there are no * buffers, this is safe. */ NpyIter_GetInnerFixedStrideArray(mit->subspace_iter, fixed_strides); + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction(is_aligned, #if @isget@ fixed_strides[0], fixed_strides[1], @@ -1682,7 +1730,7 @@ mapiter_@name@(PyArrayMapIterObject *mit) PyArray_DESCR(mit->extra_op), PyArray_DESCR(array), #endif 0, - &stransfer, &transferdata, + &cast_info, &needs_api) != NPY_SUCCEED) { return -1; } @@ -1719,7 +1767,7 @@ mapiter_@name@(PyArrayMapIterObject *mit) #if @isget@ && @one_iter@ if (check_and_adjust_index(&indval, fancy_dims[i], iteraxis, _save) < 0 ) { - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } #else @@ -1751,7 +1799,7 @@ mapiter_@name@(PyArrayMapIterObject *mit) &errmsg)) { NPY_END_THREADS; PyErr_SetString(PyExc_ValueError, errmsg); - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } if (is_subiter_trivial != 0) { @@ -1781,7 +1829,7 @@ mapiter_@name@(PyArrayMapIterObject *mit) * not at all... */ if (needs_api && PyErr_Occurred()) { - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } #endif @@ -1789,21 +1837,21 @@ mapiter_@name@(PyArrayMapIterObject *mit) do { #if @isget@ - if (NPY_UNLIKELY(stransfer( - subspace_ptrs[1], subspace_strides[1], - subspace_ptrs[0], subspace_strides[0], - *counter, src_itemsize, transferdata) < 0)) { + if (NPY_UNLIKELY(cast_info.func(&cast_info.context, + subspace_ptrs, counter, subspace_strides, + cast_info.auxdata) < 0)) { NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } #else - if (NPY_UNLIKELY(stransfer( - subspace_ptrs[0], subspace_strides[0], - subspace_ptrs[1], subspace_strides[1], - *counter, src_itemsize, transferdata) < 0)) { + /* The operand order is reveresed here */ + char *args[2] = {subspace_ptrs[1], subspace_ptrs[0]}; + npy_intp strides[2] = {subspace_strides[1], subspace_strides[0]}; + if (NPY_UNLIKELY(cast_info.func(&cast_info.context, + args, counter, strides, cast_info.auxdata) < 0)) { NPY_END_THREADS; - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return -1; } #endif @@ -1815,7 +1863,7 @@ mapiter_@name@(PyArrayMapIterObject *mit) } /**end repeat1**/ - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); } return 0; } diff --git a/numpy/core/src/multiarray/mapping.c b/numpy/core/src/multiarray/mapping.c index cb5c3823dccf..41311b03f331 100644 --- a/numpy/core/src/multiarray/mapping.c +++ b/numpy/core/src/multiarray/mapping.c @@ -63,15 +63,61 @@ array_length(PyArrayObject *self) /* -------------------------------------------------------------- */ + +/* + * Helper for `PyArray_MapIterSwapAxes` (and related), see its documentation. + */ +static void +_get_transpose(int fancy_ndim, int consec, int ndim, int getmap, npy_intp *dims) +{ + /* + * For getting the array the tuple for transpose is + * (n1,...,n1+n2-1,0,...,n1-1,n1+n2,...,n3-1) + * n1 is the number of dimensions of the broadcast index array + * n2 is the number of dimensions skipped at the start + * n3 is the number of dimensions of the result + */ + + /* + * For setting the array the tuple for transpose is + * (n2,...,n1+n2-1,0,...,n2-1,n1+n2,...n3-1) + */ + int n1 = fancy_ndim; + int n2 = consec; /* axes to insert at */ + int n3 = ndim; + + /* use n1 as the boundary if getting but n2 if setting */ + int bnd = getmap ? n1 : n2; + int val = bnd; + int i = 0; + while (val < n1 + n2) { + dims[i++] = val++; + } + val = 0; + while (val < bnd) { + dims[i++] = val++; + } + val = n1 + n2; + while (val < n3) { + dims[i++] = val++; + } +} + + /*NUMPY_API * + * Swap the axes to or from their inserted form. MapIter always puts the + * advanced (array) indices first in the iteration. But if they are + * consecutive, will insert/transpose them back before returning. + * This is stored as `mit->consec != 0` (the place where they are inserted) + * For assignments, the opposite happens: The values to be assigned are + * transposed (getmap=1 instead of getmap=0). `getmap=0` and `getmap=1` + * undo the other operation. */ NPY_NO_EXPORT void PyArray_MapIterSwapAxes(PyArrayMapIterObject *mit, PyArrayObject **ret, int getmap) { PyObject *new; - int n1, n2, n3, val, bnd; - int i; PyArray_Dims permute; npy_intp d[NPY_MAXDIMS]; PyArrayObject *arr; @@ -85,10 +131,10 @@ PyArray_MapIterSwapAxes(PyArrayMapIterObject *mit, PyArrayObject **ret, int getm */ arr = *ret; if (PyArray_NDIM(arr) != mit->nd) { - for (i = 1; i <= PyArray_NDIM(arr); i++) { + for (int i = 1; i <= PyArray_NDIM(arr); i++) { permute.ptr[mit->nd-i] = PyArray_DIMS(arr)[PyArray_NDIM(arr)-i]; } - for (i = 0; i < mit->nd-PyArray_NDIM(arr); i++) { + for (int i = 0; i < mit->nd-PyArray_NDIM(arr); i++) { permute.ptr[i] = 1; } new = PyArray_Newshape(arr, &permute, NPY_ANYORDER); @@ -99,44 +145,8 @@ PyArray_MapIterSwapAxes(PyArrayMapIterObject *mit, PyArrayObject **ret, int getm } } - /* - * Setting and getting need to have different permutations. - * On the get we are permuting the returned object, but on - * setting we are permuting the object-to-be-set. - * The set permutation is the inverse of the get permutation. - */ - - /* - * For getting the array the tuple for transpose is - * (n1,...,n1+n2-1,0,...,n1-1,n1+n2,...,n3-1) - * n1 is the number of dimensions of the broadcast index array - * n2 is the number of dimensions skipped at the start - * n3 is the number of dimensions of the result - */ + _get_transpose(mit->nd_fancy, mit->consec, mit->nd, getmap, permute.ptr); - /* - * For setting the array the tuple for transpose is - * (n2,...,n1+n2-1,0,...,n2-1,n1+n2,...n3-1) - */ - n1 = mit->nd_fancy; - n2 = mit->consec; /* axes to insert at */ - n3 = mit->nd; - - /* use n1 as the boundary if getting but n2 if setting */ - bnd = getmap ? n1 : n2; - val = bnd; - i = 0; - while (val < n1 + n2) { - permute.ptr[i++] = val++; - } - val = 0; - while (val < bnd) { - permute.ptr[i++] = val++; - } - val = n1 + n2; - while (val < n3) { - permute.ptr[i++] = val++; - } new = PyArray_Transpose(*ret, &permute); Py_DECREF(*ret); *ret = (PyArrayObject *)new; @@ -1037,8 +1047,6 @@ array_boolean_subscript(PyArrayObject *self, PyArrayObject *op[2] = {self, bmask}; npy_uint32 flags, op_flags[2]; npy_intp fixed_strides[3]; - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; NpyIter_IterNextFunc *iternext; npy_intp innersize, *innerstrides; @@ -1063,12 +1071,13 @@ array_boolean_subscript(PyArrayObject *self, /* Get a dtype transfer function */ NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction( IsUintAligned(self) && IsAligned(self), fixed_strides[0], itemsize, dtype, dtype, 0, - &stransfer, &transferdata, + &cast_info, &needs_api) != NPY_SUCCEED) { Py_DECREF(ret); NpyIter_Deallocate(iter); @@ -1080,7 +1089,7 @@ array_boolean_subscript(PyArrayObject *self, if (iternext == NULL) { Py_DECREF(ret); NpyIter_Deallocate(iter); - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); return NULL; } @@ -1091,6 +1100,8 @@ array_boolean_subscript(PyArrayObject *self, self_stride = innerstrides[0]; bmask_stride = innerstrides[1]; + npy_intp strides[2] = {self_stride, itemsize}; + int res = 0; do { innersize = *NpyIter_GetInnerLoopSizePtr(iter); @@ -1106,8 +1117,9 @@ array_boolean_subscript(PyArrayObject *self, /* Process unmasked values */ bmask_data = npy_memchr(bmask_data, 0, bmask_stride, innersize, &subloopsize, 0); - res = stransfer(ret_data, itemsize, self_data, self_stride, - subloopsize, itemsize, transferdata); + char *args[2] = {self_data, ret_data}; + res = cast_info.func(&cast_info.context, + args, &subloopsize, strides, cast_info.auxdata); if (res < 0) { break; } @@ -1122,7 +1134,7 @@ array_boolean_subscript(PyArrayObject *self, if (!NpyIter_Deallocate(iter)) { res = -1; } - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); if (res < 0) { /* Should be practically impossible, since there is no cast */ Py_DECREF(ret); @@ -1164,7 +1176,7 @@ NPY_NO_EXPORT int array_assign_boolean_subscript(PyArrayObject *self, PyArrayObject *bmask, PyArrayObject *v, NPY_ORDER order) { - npy_intp size, src_itemsize, v_stride; + npy_intp size, v_stride; char *v_data; int needs_api = 0; npy_intp bmask_size; @@ -1216,7 +1228,6 @@ array_assign_boolean_subscript(PyArrayObject *self, v_stride = 0; } - src_itemsize = PyArray_DESCR(v)->elsize; v_data = PyArray_DATA(v); /* Create an iterator for the data */ @@ -1231,8 +1242,6 @@ array_assign_boolean_subscript(PyArrayObject *self, npy_intp innersize, *innerstrides; char **dataptrs; - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; npy_intp self_stride, bmask_stride, subloopsize; char *self_data; char *bmask_data; @@ -1264,13 +1273,14 @@ array_assign_boolean_subscript(PyArrayObject *self, /* Get a dtype transfer function */ NpyIter_GetInnerFixedStrideArray(iter, fixed_strides); + NPY_cast_info cast_info; if (PyArray_GetDTypeTransferFunction( IsUintAligned(self) && IsAligned(self) && IsUintAligned(v) && IsAligned(v), v_stride, fixed_strides[0], PyArray_DESCR(v), PyArray_DESCR(self), 0, - &stransfer, &transferdata, + &cast_info, &needs_api) != NPY_SUCCEED) { NpyIter_Deallocate(iter); return -1; @@ -1280,6 +1290,8 @@ array_assign_boolean_subscript(PyArrayObject *self, NPY_BEGIN_THREADS_NDITER(iter); } + npy_intp strides[2] = {v_stride, self_stride}; + do { innersize = *NpyIter_GetInnerLoopSizePtr(iter); self_data = dataptrs[0]; @@ -1294,8 +1306,10 @@ array_assign_boolean_subscript(PyArrayObject *self, /* Process unmasked values */ bmask_data = npy_memchr(bmask_data, 0, bmask_stride, innersize, &subloopsize, 0); - res = stransfer(self_data, self_stride, v_data, v_stride, - subloopsize, src_itemsize, transferdata); + + char *args[2] = {v_data, self_data}; + res = cast_info.func(&cast_info.context, + args, &subloopsize, strides, cast_info.auxdata); if (res < 0) { break; } @@ -1309,7 +1323,7 @@ array_assign_boolean_subscript(PyArrayObject *self, NPY_END_THREADS; } - NPY_AUXDATA_FREE(transferdata); + NPY_cast_info_xfree(&cast_info); if (!NpyIter_Deallocate(iter)) { res = -1; } @@ -2328,7 +2342,7 @@ PyArray_MapIterNext(PyArrayMapIterObject *mit) * @param Number of indices * @param The array that is being iterated * - * @return 0 on success -1 on failure + * @return 0 on success -1 on failure (broadcasting or too many fancy indices) */ static int mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, @@ -2369,6 +2383,17 @@ mapiter_fill_info(PyArrayMapIterObject *mit, npy_index_info *indices, } } + /* Before contunuing, ensure that there are not too fancy indices */ + if (indices[i].type & HAS_FANCY) { + if (NPY_UNLIKELY(j >= NPY_MAXDIMS)) { + PyErr_Format(PyExc_IndexError, + "too many advanced (array) indices. This probably " + "means you are indexing with too many booleans. " + "(more than %d found)", NPY_MAXDIMS); + return -1; + } + } + /* (iterating) fancy index, store the iterator */ if (indices[i].type == HAS_FANCY) { mit->fancy_strides[j] = PyArray_STRIDE(arr, curr_dim); @@ -2655,6 +2680,7 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, /* For shape reporting on error */ PyArrayObject *original_extra_op = extra_op; + /* NOTE: MAXARGS is the actual limit (2*NPY_MAXDIMS is index number one) */ PyArrayObject *index_arrays[NPY_MAXDIMS]; PyArray_Descr *intp_descr; PyArray_Descr *dtypes[NPY_MAXDIMS]; /* borrowed references */ @@ -3188,15 +3214,23 @@ PyArray_MapIterNew(npy_index_info *indices , int index_num, int index_type, int extra_ndim = PyArray_NDIM(original_extra_op); npy_intp *extra_dims = PyArray_DIMS(original_extra_op); - PyObject *shape1 = convert_shape_to_string(extra_ndim, extra_dims, " "); + PyObject *shape1 = convert_shape_to_string(extra_ndim, extra_dims, ""); if (shape1 == NULL) { goto finish; } - PyObject *shape2 = convert_shape_to_string(mit->nd, mit->dimensions, ""); - if (shape2 == NULL) + /* Unscramble the iterator shape for reporting when `mit->consec` is used */ + npy_intp transposed[NPY_MAXDIMS]; + _get_transpose(mit->nd_fancy, mit->consec, mit->nd, 1, transposed); + for (i = 0; i < mit->nd; i++) { + transposed[i] = mit->dimensions[transposed[i]]; + } + + PyObject *shape2 = convert_shape_to_string(mit->nd, transposed, ""); + if (shape2 == NULL) { Py_DECREF(shape1); goto finish; + } PyErr_Format(PyExc_ValueError, "shape mismatch: value array of shape %S could not be broadcast " diff --git a/numpy/core/src/multiarray/methods.c b/numpy/core/src/multiarray/methods.c index 9c8bb4135005..251e527a6b96 100644 --- a/numpy/core/src/multiarray/methods.c +++ b/numpy/core/src/multiarray/methods.c @@ -10,6 +10,7 @@ #include "numpy/arrayscalars.h" #include "arrayfunction_override.h" +#include "npy_argparse.h" #include "npy_config.h" #include "npy_pycompat.h" #include "npy_import.h" @@ -103,20 +104,23 @@ forward_ndarray_method(PyArrayObject *self, PyObject *args, PyObject *kwds, static PyObject * -array_take(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_take(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int dimension = NPY_MAXDIMS; PyObject *indices; PyArrayObject *out = NULL; NPY_CLIPMODE mode = NPY_RAISE; - static char *kwlist[] = {"indices", "axis", "out", "mode", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&O&O&:take", kwlist, - &indices, - PyArray_AxisConverter, &dimension, - PyArray_OutputConverter, &out, - PyArray_ClipmodeConverter, &mode)) + if (npy_parse_arguments("take", args, len_args, kwnames, + "indices", NULL, &indices, + "|axis", &PyArray_AxisConverter, &dimension, + "|out", &PyArray_OutputConverter, &out, + "|mode", &PyArray_ClipmodeConverter, &mode, + NULL, NULL, NULL) < 0) { return NULL; + } PyObject *ret = PyArray_TakeFrom(self, indices, dimension, out, mode); @@ -199,14 +203,16 @@ array_reshape(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_squeeze(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_squeeze(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *axis_in = NULL; npy_bool axis_flags[NPY_MAXDIMS]; + NPY_PREPARE_ARGPARSER; - static char *kwlist[] = {"axis", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:squeeze", kwlist, - &axis_in)) { + if (npy_parse_arguments("squeeze", args, len_args, kwnames, + "|axis", NULL, &axis_in, + NULL, NULL, NULL) < 0) { return NULL; } @@ -224,16 +230,18 @@ array_squeeze(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_view(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_view(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *out_dtype = NULL; PyObject *out_type = NULL; PyArray_Descr *dtype = NULL; + NPY_PREPARE_ARGPARSER; - static char *kwlist[] = {"dtype", "type", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO:view", kwlist, - &out_dtype, - &out_type)) { + if (npy_parse_arguments("view", args, len_args, kwnames, + "|dtype", NULL, &out_dtype, + "|type", NULL, &out_type, + NULL, NULL, NULL) < 0) { return NULL; } @@ -271,16 +279,19 @@ array_view(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_argmax(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_argmax(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis = NPY_MAXDIMS; PyArrayObject *out = NULL; - static char *kwlist[] = {"axis", "out", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&O&:argmax", kwlist, - PyArray_AxisConverter, &axis, - PyArray_OutputConverter, &out)) + if (npy_parse_arguments("argmax", args, len_args, kwnames, + "|axis", &PyArray_AxisConverter, &axis, + "|out", &PyArray_OutputConverter, &out, + NULL, NULL, NULL) < 0) { return NULL; + } PyObject *ret = PyArray_ArgMax(self, axis, out); @@ -294,16 +305,19 @@ array_argmax(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_argmin(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_argmin(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis = NPY_MAXDIMS; PyArrayObject *out = NULL; - static char *kwlist[] = {"axis", "out", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&O&:argmin", kwlist, - PyArray_AxisConverter, &axis, - PyArray_OutputConverter, &out)) + if (npy_parse_arguments("argmin", args, len_args, kwnames, + "|axis", &PyArray_AxisConverter, &axis, + "|out", &PyArray_OutputConverter, &out, + NULL, NULL, NULL) < 0) { return NULL; + } PyObject *ret = PyArray_ArgMin(self, axis, out); @@ -804,10 +818,9 @@ array_setscalar(PyArrayObject *self, PyObject *args) static PyObject * -array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_astype(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - static char *kwlist[] = {"dtype", "order", "casting", - "subok", "copy", NULL}; PyArray_Descr *dtype = NULL; /* * TODO: UNSAFE default for compatibility, I think @@ -816,13 +829,15 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) NPY_CASTING casting = NPY_UNSAFE_CASTING; NPY_ORDER order = NPY_KEEPORDER; int forcecopy = 1, subok = 1; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&ii:astype", kwlist, - PyArray_DescrConverter, &dtype, - PyArray_OrderConverter, &order, - PyArray_CastingConverter, &casting, - &subok, - &forcecopy)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("astype", args, len_args, kwnames, + "dtype", &PyArray_DescrConverter, &dtype, + "|order", &PyArray_OrderConverter, &order, + "|casting", &PyArray_CastingConverter, &casting, + "|subok", &PyArray_PythonPyIntFromInt, &subok, + "|copy", &PyArray_PythonPyIntFromInt, &forcecopy, + NULL, NULL, NULL) < 0) { Py_XDECREF(dtype); return NULL; } @@ -852,44 +867,44 @@ array_astype(PyArrayObject *self, PyObject *args, PyObject *kwds) Py_INCREF(self); return (PyObject *)self; } - else if (PyArray_CanCastArrayTo(self, dtype, casting)) { - PyArrayObject *ret; - - /* This steals the reference to dtype, so no DECREF of dtype */ - ret = (PyArrayObject *)PyArray_NewLikeArray( - self, order, dtype, subok); - if (ret == NULL) { - return NULL; - } - /* NumPy 1.20, 2020-10-01 */ - if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && - DEPRECATE_FUTUREWARNING( - "casting an array to a subarray dtype " - "will not using broadcasting in the future, but cast each " - "element to the new dtype and then append the dtype's shape " - "to the new array. You can opt-in to the new behaviour, by " - "additional field to the cast: " - "`arr.astype(np.dtype([('f', dtype)]))['f']`.\n" - "This may lead to a different result or to current failures " - "succeeding. " - "(FutureWarning since NumPy 1.20)") < 0) { - Py_DECREF(ret); - return NULL; - } - - if (PyArray_CopyInto(ret, self) < 0) { - Py_DECREF(ret); - return NULL; - } - - return (PyObject *)ret; - } - else { + if (!PyArray_CanCastArrayTo(self, dtype, casting)) { + PyErr_Clear(); npy_set_invalid_cast_error( PyArray_DESCR(self), dtype, casting, PyArray_NDIM(self) == 0); Py_DECREF(dtype); return NULL; } + + PyArrayObject *ret; + + /* This steals the reference to dtype, so no DECREF of dtype */ + ret = (PyArrayObject *)PyArray_NewLikeArray( + self, order, dtype, subok); + if (ret == NULL) { + return NULL; + } + /* NumPy 1.20, 2020-10-01 */ + if ((PyArray_NDIM(self) != PyArray_NDIM(ret)) && + DEPRECATE_FUTUREWARNING( + "casting an array to a subarray dtype " + "will not use broadcasting in the future, but cast each " + "element to the new dtype and then append the dtype's shape " + "to the new array. You can opt-in to the new behaviour, by " + "additional field to the cast: " + "`arr.astype(np.dtype([('f', dtype)]))['f']`.\n" + "This may lead to a different result or to current failures " + "succeeding. " + "(FutureWarning since NumPy 1.20)") < 0) { + Py_DECREF(ret); + return NULL; + } + + if (PyArray_CopyInto(ret, self) < 0) { + Py_DECREF(ret); + return NULL; + } + + return (PyObject *)ret; } /* default sub-type implementation */ @@ -1019,7 +1034,7 @@ array_getarray(PyArrayObject *self, PyObject *args) } /* - * Check whether any of a set of input and output args have a non-default + * Check whether any of the input and output args have a non-default * __array_ufunc__ method. Return 1 if so, 0 if not, and -1 on error. * * This function primarily exists to help ndarray.__array_ufunc__ determine @@ -1143,13 +1158,15 @@ array_function(PyArrayObject *NPY_UNUSED(self), PyObject *c_args, PyObject *c_kw } static PyObject * -array_copy(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_copy(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { NPY_ORDER order = NPY_CORDER; - static char *kwlist[] = {"order", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&:copy", kwlist, - PyArray_OrderConverter, &order)) { + if (npy_parse_arguments("copy", args, len_args, kwnames, + "|order", PyArray_OrderConverter, &order, + NULL, NULL, NULL) < 0) { return NULL; } @@ -1257,7 +1274,8 @@ array_choose(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_sort(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_sort(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis=-1; int val; @@ -1265,12 +1283,13 @@ array_sort(PyArrayObject *self, PyObject *args, PyObject *kwds) PyObject *order = NULL; PyArray_Descr *saved = NULL; PyArray_Descr *newd; - static char *kwlist[] = {"axis", "kind", "order", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|iO&O:sort", kwlist, - &axis, - PyArray_SortkindConverter, &sortkind, - &order)) { + if (npy_parse_arguments("sort", args, len_args, kwnames, + "|axis", &PyArray_PythonPyIntFromInt, &axis, + "|kind", &PyArray_SortkindConverter, &sortkind, + "|order", NULL, &order, + NULL, NULL, NULL) < 0) { return NULL; } if (order == Py_None) { @@ -1313,7 +1332,8 @@ array_sort(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_partition(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_partition(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis=-1; int val; @@ -1321,16 +1341,16 @@ array_partition(PyArrayObject *self, PyObject *args, PyObject *kwds) PyObject *order = NULL; PyArray_Descr *saved = NULL; PyArray_Descr *newd; - static char *kwlist[] = {"kth", "axis", "kind", "order", NULL}; PyArrayObject * ktharray; PyObject * kthobj; + NPY_PREPARE_ARGPARSER; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|iO&O:partition", kwlist, - &kthobj, - &axis, - PyArray_SelectkindConverter, &sortkind, - &order)) { + if (npy_parse_arguments("partition", args, len_args, kwnames, + "kth", NULL, &kthobj, + "|axis", &PyArray_PythonPyIntFromInt, &axis, + "|kind", &PyArray_SelectkindConverter, &sortkind, + "|order", NULL, &order, + NULL, NULL, NULL) < 0) { return NULL; } @@ -1381,18 +1401,20 @@ array_partition(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_argsort(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_argsort(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis = -1; NPY_SORTKIND sortkind = NPY_QUICKSORT; PyObject *order = NULL, *res; PyArray_Descr *newd, *saved=NULL; - static char *kwlist[] = {"axis", "kind", "order", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&O&O:argsort", kwlist, - PyArray_AxisConverter, &axis, - PyArray_SortkindConverter, &sortkind, - &order)) { + if (npy_parse_arguments("argsort", args, len_args, kwnames, + "|axis", &PyArray_AxisConverter, &axis, + "|kind", &PyArray_SortkindConverter, &sortkind, + "|order", NULL, &order, + NULL, NULL, NULL) < 0) { return NULL; } if (order == Py_None) { @@ -1433,21 +1455,23 @@ array_argsort(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * -array_argpartition(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_argpartition(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis = -1; NPY_SELECTKIND sortkind = NPY_INTROSELECT; PyObject *order = NULL, *res; PyArray_Descr *newd, *saved=NULL; - static char *kwlist[] = {"kth", "axis", "kind", "order", NULL}; PyObject * kthobj; PyArrayObject * ktharray; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&O&O:argpartition", kwlist, - &kthobj, - PyArray_AxisConverter, &axis, - PyArray_SelectkindConverter, &sortkind, - &order)) { + if (npy_parse_arguments("argpartition", args, len_args, kwnames, + "kth", NULL, &kthobj, + "|axis", &PyArray_AxisConverter, &axis, + "|kind", &PyArray_SelectkindConverter, &sortkind, + "|order", NULL, &order, + NULL, NULL, NULL) < 0) { return NULL; } if (order == Py_None) { @@ -1494,17 +1518,20 @@ array_argpartition(PyArrayObject *self, PyObject *args, PyObject *kwds) } static PyObject * -array_searchsorted(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_searchsorted(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - static char *kwlist[] = {"v", "side", "sorter", NULL}; PyObject *keys; PyObject *sorter; NPY_SEARCHSIDE side = NPY_SEARCHLEFT; + NPY_PREPARE_ARGPARSER; sorter = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&O:searchsorted", - kwlist, &keys, - PyArray_SearchsideConverter, &side, &sorter)) { + if (npy_parse_arguments("searchsorted", args, len_args, kwnames, + "v", NULL, &keys, + "|side", &PyArray_SearchsideConverter, &side, + "|sorter", NULL, &sorter, + NULL, NULL, NULL) < 0) { return NULL; } if (sorter == Py_None) { @@ -2285,14 +2312,17 @@ array_cumprod(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * -array_dot(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_dot(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *a = (PyObject *)self, *b, *o = NULL; PyArrayObject *ret; - char* kwlist[] = {"b", "out", NULL }; + NPY_PREPARE_ARGPARSER; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:dot", kwlist, &b, &o)) { + if (npy_parse_arguments("dot", args, len_args, kwnames, + "b", NULL, &b, + "|out", NULL, &o, + NULL, NULL, NULL) < 0) { return NULL; } @@ -2374,20 +2404,22 @@ array_nonzero(PyArrayObject *self, PyObject *args) static PyObject * -array_trace(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_trace(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { int axis1 = 0, axis2 = 1, offset = 0; PyArray_Descr *dtype = NULL; PyArrayObject *out = NULL; int rtype; - static char *kwlist[] = {"offset", "axis1", "axis2", "dtype", "out", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|iiiO&O&:trace", kwlist, - &offset, - &axis1, - &axis2, - PyArray_DescrConverter2, &dtype, - PyArray_OutputConverter, &out)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("trace", args, len_args, kwnames, + "|offset", &PyArray_PythonPyIntFromInt, &offset, + "|axis1", &PyArray_PythonPyIntFromInt, &axis1, + "|axis2", &PyArray_PythonPyIntFromInt, &axis2, + "|dtype", &PyArray_DescrConverter2, &dtype, + "|out", &PyArray_OutputConverter, &out, + NULL, NULL, NULL) < 0) { Py_XDECREF(dtype); return NULL; } @@ -2448,13 +2480,15 @@ array_diagonal(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * -array_flatten(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_flatten(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { NPY_ORDER order = NPY_CORDER; - static char *kwlist[] = {"order", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&:flatten", kwlist, - PyArray_OrderConverter, &order)) { + if (npy_parse_arguments("flatten", args, len_args, kwnames, + "|order", PyArray_OrderConverter, &order, + NULL, NULL, NULL) < 0) { return NULL; } return PyArray_Flatten(self, order); @@ -2462,13 +2496,15 @@ array_flatten(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * -array_ravel(PyArrayObject *self, PyObject *args, PyObject *kwds) +array_ravel(PyArrayObject *self, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { NPY_ORDER order = NPY_CORDER; - static char *kwlist[] = {"order", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O&:ravel", kwlist, - PyArray_OrderConverter, &order)) { + if (npy_parse_arguments("ravel", args, len_args, kwnames, + "|order", PyArray_OrderConverter, &order, + NULL, NULL, NULL) < 0) { return NULL; } return PyArray_Ravel(self, order); @@ -2724,19 +2760,19 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"argmax", (PyCFunction)array_argmax, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"argmin", (PyCFunction)array_argmin, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"argpartition", (PyCFunction)array_argpartition, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"argsort", (PyCFunction)array_argsort, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"astype", (PyCFunction)array_astype, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"byteswap", (PyCFunction)array_byteswap, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2757,7 +2793,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS, NULL}, {"copy", (PyCFunction)array_copy, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"cumprod", (PyCFunction)array_cumprod, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2769,13 +2805,13 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"dot", (PyCFunction)array_dot, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"fill", (PyCFunction)array_fill, METH_VARARGS, NULL}, {"flatten", (PyCFunction)array_flatten, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"getfield", (PyCFunction)array_getfield, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2802,7 +2838,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS, NULL}, {"partition", (PyCFunction)array_partition, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"prod", (PyCFunction)array_prod, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2814,7 +2850,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"ravel", (PyCFunction)array_ravel, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"repeat", (PyCFunction)array_repeat, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2829,7 +2865,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"searchsorted", (PyCFunction)array_searchsorted, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"setfield", (PyCFunction)array_setfield, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2838,10 +2874,10 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"sort", (PyCFunction)array_sort, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"squeeze", (PyCFunction)array_squeeze, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"std", (PyCFunction)array_stddev, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2853,7 +2889,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS, NULL}, {"take", (PyCFunction)array_take, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"tobytes", (PyCFunction)array_tobytes, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -2868,7 +2904,7 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"trace", (PyCFunction)array_trace, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"transpose", (PyCFunction)array_transpose, METH_VARARGS, NULL}, @@ -2877,6 +2913,6 @@ NPY_NO_EXPORT PyMethodDef array_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"view", (PyCFunction)array_view, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {NULL, NULL, 0, NULL} /* sentinel */ }; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index af5949e73b47..f7c3ea093a29 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -26,7 +26,7 @@ #include "numpy/arrayscalars.h" #include "numpy/npy_math.h" - +#include "npy_argparse.h" #include "npy_config.h" #include "npy_pycompat.h" #include "npy_import.h" @@ -448,17 +448,10 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis, /* Get the priority subtype for the array */ PyTypeObject *subtype = PyArray_GetSubType(narrays, arrays); - - if (dtype == NULL) { - /* Get the resulting dtype from combining all the arrays */ - dtype = (PyArray_Descr *)PyArray_ResultType( - narrays, arrays, 0, NULL); - if (dtype == NULL) { - return NULL; - } - } - else { - Py_INCREF(dtype); + PyArray_Descr *descr = PyArray_FindConcatenationDescriptor( + narrays, arrays, (PyObject *)dtype); + if (descr == NULL) { + return NULL; } /* @@ -467,7 +460,7 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis, * resolution rules matching that of the NpyIter. */ PyArray_CreateMultiSortedStridePerm(narrays, arrays, ndim, strideperm); - s = dtype->elsize; + s = descr->elsize; for (idim = ndim-1; idim >= 0; --idim) { int iperm = strideperm[idim]; strides[iperm] = s; @@ -475,17 +468,13 @@ PyArray_ConcatenateArrays(int narrays, PyArrayObject **arrays, int axis, } /* Allocate the array for the result. This steals the 'dtype' reference. */ - ret = (PyArrayObject *)PyArray_NewFromDescr(subtype, - dtype, - ndim, - shape, - strides, - NULL, - 0, - NULL); + ret = (PyArrayObject *)PyArray_NewFromDescr_int( + subtype, descr, ndim, shape, strides, NULL, 0, NULL, + NULL, 0, 1); if (ret == NULL) { return NULL; } + assert(PyArray_DESCR(ret) == descr); } /* @@ -575,32 +564,22 @@ PyArray_ConcatenateFlattenedArrays(int narrays, PyArrayObject **arrays, /* Get the priority subtype for the array */ PyTypeObject *subtype = PyArray_GetSubType(narrays, arrays); - if (dtype == NULL) { - /* Get the resulting dtype from combining all the arrays */ - dtype = (PyArray_Descr *)PyArray_ResultType( - narrays, arrays, 0, NULL); - if (dtype == NULL) { - return NULL; - } - } - else { - Py_INCREF(dtype); + PyArray_Descr *descr = PyArray_FindConcatenationDescriptor( + narrays, arrays, (PyObject *)dtype); + if (descr == NULL) { + return NULL; } - stride = dtype->elsize; + stride = descr->elsize; /* Allocate the array for the result. This steals the 'dtype' reference. */ - ret = (PyArrayObject *)PyArray_NewFromDescr(subtype, - dtype, - 1, - &shape, - &stride, - NULL, - 0, - NULL); + ret = (PyArrayObject *)PyArray_NewFromDescr_int( + subtype, descr, 1, &shape, &stride, NULL, 0, NULL, + NULL, 0, 1); if (ret == NULL) { return NULL; } + assert(PyArray_DESCR(ret) == descr); } /* @@ -1491,7 +1470,9 @@ array_putmask(PyObject *NPY_UNUSED(module), PyObject *args, PyObject *kwds) NPY_NO_EXPORT unsigned char PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) { -#if NPY_USE_NEW_CASTINGIMPL + if (type1 == type2) { + return 1; + } /* * Do not use PyArray_CanCastTypeTo because it supports legacy flexible * dtypes as input. @@ -1503,9 +1484,6 @@ PyArray_EquivTypes(PyArray_Descr *type1, PyArray_Descr *type2) } /* If casting is "no casting" this dtypes are considered equivalent. */ return PyArray_MinCastSafety(safety, NPY_NO_CASTING) == NPY_NO_CASTING; -#else - return PyArray_LegacyEquivTypes(type1, type2); -#endif } @@ -1577,135 +1555,23 @@ _prepend_ones(PyArrayObject *arr, int nd, int ndmin, NPY_ORDER order) ((order) == NPY_CORDER && PyArray_IS_C_CONTIGUOUS(op)) || \ ((order) == NPY_FORTRANORDER && PyArray_IS_F_CONTIGUOUS(op))) -static PyObject * -_array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) +static NPY_INLINE PyObject * +_array_fromobject_generic( + PyObject *op, PyArray_Descr *type, npy_bool copy, NPY_ORDER order, + npy_bool subok, int ndmin) { - PyObject *op; PyArrayObject *oparr = NULL, *ret = NULL; - npy_bool subok = NPY_FALSE; - npy_bool copy = NPY_TRUE; - int ndmin = 0, nd; - PyObject* like; - PyArray_Descr *type = NULL; PyArray_Descr *oldtype = NULL; - NPY_ORDER order = NPY_KEEPORDER; - int flags = 0; - - PyObject* array_function_result = NULL; - - static char *kwd[] = {"object", "dtype", "copy", "order", "subok", - "ndmin", "like", NULL}; - - if (PyTuple_GET_SIZE(args) > 2) { - PyErr_Format(PyExc_TypeError, - "array() takes from 1 to 2 positional arguments but " - "%zd were given", PyTuple_GET_SIZE(args)); - return NULL; - } - - array_function_result = array_implement_c_array_function_creation( - "array", args, kws); - if (array_function_result != Py_NotImplemented) { - return array_function_result; - } - - /* super-fast path for ndarray argument calls */ - if (PyTuple_GET_SIZE(args) == 0) { - goto full_path; - } - op = PyTuple_GET_ITEM(args, 0); - if (PyArray_CheckExact(op)) { - PyObject * dtype_obj = Py_None; - oparr = (PyArrayObject *)op; - /* get dtype which can be positional */ - if (PyTuple_GET_SIZE(args) == 2) { - dtype_obj = PyTuple_GET_ITEM(args, 1); - } - else if (kws) { - dtype_obj = PyDict_GetItemWithError(kws, npy_ma_str_dtype); - if (dtype_obj == NULL && PyErr_Occurred()) { - return NULL; - } - if (dtype_obj == NULL) { - dtype_obj = Py_None; - } - } - if (dtype_obj != Py_None) { - goto full_path; - } - - /* array(ndarray) */ - if (kws == NULL) { - ret = (PyArrayObject *)PyArray_NewCopy(oparr, order); - goto finish; - } - else { - /* fast path for copy=False rest default (np.asarray) */ - PyObject * copy_obj, * order_obj, *ndmin_obj; - copy_obj = PyDict_GetItemWithError(kws, npy_ma_str_copy); - if (copy_obj == NULL && PyErr_Occurred()) { - return NULL; - } - if (copy_obj != Py_False) { - goto full_path; - } - copy = NPY_FALSE; - - /* order does not matter for contiguous 1d arrays */ - if (PyArray_NDIM((PyArrayObject*)op) > 1 || - !PyArray_IS_C_CONTIGUOUS((PyArrayObject*)op)) { - order_obj = PyDict_GetItemWithError(kws, npy_ma_str_order); - if (order_obj == NULL && PyErr_Occurred()) { - return NULL; - } - else if (order_obj != Py_None && order_obj != NULL) { - goto full_path; - } - } - - ndmin_obj = PyDict_GetItemWithError(kws, npy_ma_str_ndmin); - if (ndmin_obj == NULL && PyErr_Occurred()) { - return NULL; - } - else if (ndmin_obj) { - long t = PyLong_AsLong(ndmin_obj); - if (error_converting(t)) { - goto clean_type; - } - else if (t > NPY_MAXDIMS) { - goto full_path; - } - ndmin = t; - } - - /* copy=False with default dtype, order (any is OK) and ndim */ - ret = oparr; - Py_INCREF(ret); - goto finish; - } - } - -full_path: - if (!PyArg_ParseTupleAndKeywords(args, kws, "O|O&O&O&O&i$O:array", kwd, - &op, - PyArray_DescrConverter2, &type, - PyArray_BoolConverter, ©, - PyArray_OrderConverter, &order, - PyArray_BoolConverter, &subok, - &ndmin, - &like)) { - goto clean_type; - } + int nd, flags = 0; if (ndmin > NPY_MAXDIMS) { PyErr_Format(PyExc_ValueError, "ndmin bigger than allowable number of dimensions " "NPY_MAXDIMS (=%d)", NPY_MAXDIMS); - goto clean_type; + return NULL; } /* fast exit if simple call */ - if ((subok && PyArray_Check(op)) || - (!subok && PyArray_CheckExact(op))) { + if (PyArray_CheckExact(op) || (subok && PyArray_Check(op))) { oparr = (PyArrayObject *)op; if (type == NULL) { if (!copy && STRIDING_OK(oparr, order)) { @@ -1760,8 +1626,7 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) ret = (PyArrayObject *)PyArray_CheckFromAny(op, type, 0, 0, flags, NULL); - finish: - Py_XDECREF(type); +finish: if (ret == NULL) { return NULL; } @@ -1775,16 +1640,215 @@ _array_fromobject(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) * steals a reference to ret */ return _prepend_ones(ret, nd, ndmin, order); +} -clean_type: +#undef STRIDING_OK + + +static PyObject * +array_array(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) +{ + PyObject *op; + npy_bool subok = NPY_FALSE; + npy_bool copy = NPY_TRUE; + int ndmin = 0; + PyArray_Descr *type = NULL; + NPY_ORDER order = NPY_KEEPORDER; + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (len_args != 1 || (kwnames != NULL)) { + if (npy_parse_arguments("array", args, len_args, kwnames, + "object", NULL, &op, + "|dtype", &PyArray_DescrConverter2, &type, + "$copy", &PyArray_BoolConverter, ©, + "$order", &PyArray_OrderConverter, &order, + "$subok", &PyArray_BoolConverter, &subok, + "$ndmin", &PyArray_PythonPyIntFromInt, &ndmin, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { + Py_XDECREF(type); + return NULL; + } + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "array", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } + } + } + else { + /* Fast path for symmetry (we copy by default which is slow) */ + op = args[0]; + } + + PyObject *res = _array_fromobject_generic( + op, type, copy, order, subok, ndmin); Py_XDECREF(type); - return NULL; + return res; } static PyObject * -array_copyto(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +array_asarray(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { + PyObject *op; + PyArray_Descr *type = NULL; + NPY_ORDER order = NPY_KEEPORDER; + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (len_args != 1 || (kwnames != NULL)) { + if (npy_parse_arguments("asarray", args, len_args, kwnames, + "a", NULL, &op, + "|dtype", &PyArray_DescrConverter2, &type, + "|order", &PyArray_OrderConverter, &order, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { + Py_XDECREF(type); + return NULL; + } + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "asarray", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } + } + } + else { + op = args[0]; + } + PyObject *res = _array_fromobject_generic( + op, type, NPY_FALSE, order, NPY_FALSE, 0); + Py_XDECREF(type); + return res; +} + +static PyObject * +array_asanyarray(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) +{ + PyObject *op; + PyArray_Descr *type = NULL; + NPY_ORDER order = NPY_KEEPORDER; + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (len_args != 1 || (kwnames != NULL)) { + if (npy_parse_arguments("asanyarray", args, len_args, kwnames, + "a", NULL, &op, + "|dtype", &PyArray_DescrConverter2, &type, + "|order", &PyArray_OrderConverter, &order, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { + Py_XDECREF(type); + return NULL; + } + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "asanyarray", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } + } + } + else { + op = args[0]; + } + + PyObject *res = _array_fromobject_generic( + op, type, NPY_FALSE, order, NPY_TRUE, 0); + Py_XDECREF(type); + return res; +} + + +static PyObject * +array_ascontiguousarray(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) +{ + PyObject *op; + PyArray_Descr *type = NULL; + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (len_args != 1 || (kwnames != NULL)) { + if (npy_parse_arguments("ascontiguousarray", args, len_args, kwnames, + "a", NULL, &op, + "|dtype", &PyArray_DescrConverter2, &type, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { + Py_XDECREF(type); + return NULL; + } + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "ascontiguousarray", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } + } + } + else { + op = args[0]; + } + + PyObject *res = _array_fromobject_generic( + op, type, NPY_FALSE, NPY_CORDER, NPY_FALSE, 1); + Py_XDECREF(type); + return res; +} + + +static PyObject * +array_asfortranarray(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) +{ + PyObject *op; + PyArray_Descr *type = NULL; + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (len_args != 1 || (kwnames != NULL)) { + if (npy_parse_arguments("asfortranarray", args, len_args, kwnames, + "a", NULL, &op, + "|dtype", &PyArray_DescrConverter2, &type, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { + Py_XDECREF(type); + return NULL; + } + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "asfortranarray", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } + } + } + else { + op = args[0]; + } + + PyObject *res = _array_fromobject_generic( + op, type, NPY_FALSE, NPY_FORTRANORDER, NPY_FALSE, 1); + Py_XDECREF(type); + return res; +} + + +static PyObject * +array_copyto(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +{ static char *kwlist[] = {"dst", "src", "casting", "where", NULL}; PyObject *wheremask_in = NULL; PyArrayObject *dst = NULL, *src = NULL, *wheremask = NULL; @@ -1827,30 +1891,34 @@ array_copyto(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) } static PyObject * -array_empty(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +array_empty(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - - static char *kwlist[] = {"shape", "dtype", "order", "like", NULL}; PyArray_Descr *typecode = NULL; PyArray_Dims shape = {NULL, 0}; NPY_ORDER order = NPY_CORDER; - PyObject *like = NULL; npy_bool is_f_order; - PyObject *array_function_result = NULL; PyArrayObject *ret = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&$O:empty", kwlist, - PyArray_IntpConverter, &shape, - PyArray_DescrConverter, &typecode, - PyArray_OrderConverter, &order, - &like)) { + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("empty", args, len_args, kwnames, + "shape", &PyArray_IntpConverter, &shape, + "|dtype", &PyArray_DescrConverter, &typecode, + "|order", &PyArray_OrderConverter, &order, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { goto fail; } - array_function_result = array_implement_c_array_function_creation( - "empty", args, kwds); - if (array_function_result != Py_NotImplemented) { - return array_function_result; + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "empty", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(typecode); + npy_free_cache_dim_obj(shape); + return deferred; + } } switch (order) { @@ -2025,29 +2093,35 @@ array_scalar(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) } static PyObject * -array_zeros(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kwds) +array_zeros(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - static char *kwlist[] = {"shape", "dtype", "order", "like", NULL}; PyArray_Descr *typecode = NULL; PyArray_Dims shape = {NULL, 0}; NPY_ORDER order = NPY_CORDER; - PyObject *like = NULL; npy_bool is_f_order = NPY_FALSE; - PyObject *array_function_result = NULL; PyArrayObject *ret = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O&O&$O:zeros", kwlist, - PyArray_IntpConverter, &shape, - PyArray_DescrConverter, &typecode, - PyArray_OrderConverter, &order, - &like)) { + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("zeros", args, len_args, kwnames, + "shape", &PyArray_IntpConverter, &shape, + "|dtype", &PyArray_DescrConverter, &typecode, + "|order", &PyArray_OrderConverter, &order, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { goto fail; } - array_function_result = array_implement_c_array_function_creation( - "zeros", args, kwds); - if (array_function_result != Py_NotImplemented) { - return array_function_result; + + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "zeros", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(typecode); + npy_free_cache_dim_obj(shape); + return deferred; + } } switch (order) { @@ -2105,7 +2179,6 @@ array_fromstring(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds static char *kwlist[] = {"string", "dtype", "count", "sep", "like", NULL}; PyObject *like = NULL; PyArray_Descr *descr = NULL; - PyObject *array_function_result = NULL; if (!PyArg_ParseTupleAndKeywords(args, keywds, "s#|O&" NPY_SSIZE_T_PYFMT "s$O:fromstring", kwlist, @@ -2113,11 +2186,13 @@ array_fromstring(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds Py_XDECREF(descr); return NULL; } - - array_function_result = array_implement_c_array_function_creation( - "fromstring", args, keywds); - if (array_function_result != Py_NotImplemented) { - return array_function_result; + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "fromstring", like, args, keywds, NULL, 0, NULL); + if (deferred != Py_NotImplemented) { + Py_XDECREF(descr); + return deferred; + } } /* binary mode, condition copied from PyArray_FromString */ @@ -2145,7 +2220,6 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) static char *kwlist[] = {"file", "dtype", "count", "sep", "offset", "like", NULL}; PyObject *like = NULL; PyArray_Descr *type = NULL; - PyObject *array_function_result = NULL; int own; npy_off_t orig_pos = 0, offset = 0; FILE *fp; @@ -2157,14 +2231,18 @@ array_fromfile(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) return NULL; } - array_function_result = array_implement_c_array_function_creation( - "fromfile", args, keywds); - if (array_function_result != Py_NotImplemented) { - return array_function_result; + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "fromfile", like, args, keywds, NULL, 0, NULL); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } } file = NpyPath_PathlikeToFspath(file); if (file == NULL) { + Py_XDECREF(type); return NULL; } @@ -2232,7 +2310,6 @@ array_fromiter(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) static char *kwlist[] = {"iter", "dtype", "count", "like", NULL}; PyObject *like = NULL; PyArray_Descr *descr = NULL; - PyObject *array_function_result = NULL; if (!PyArg_ParseTupleAndKeywords(args, keywds, "OO&|" NPY_SSIZE_T_PYFMT "$O:fromiter", kwlist, @@ -2240,12 +2317,13 @@ array_fromiter(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds) Py_XDECREF(descr); return NULL; } - - array_function_result = array_implement_c_array_function_creation( - "fromiter", args, keywds); - if (array_function_result != Py_NotImplemented) { - Py_DECREF(descr); - return array_function_result; + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "fromiter", like, args, keywds, NULL, 0, NULL); + if (deferred != Py_NotImplemented) { + Py_XDECREF(descr); + return deferred; + } } return PyArray_FromIter(iter, descr, (npy_intp)nin); @@ -2259,7 +2337,6 @@ array_frombuffer(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds static char *kwlist[] = {"buffer", "dtype", "count", "offset", "like", NULL}; PyObject *like = NULL; PyArray_Descr *type = NULL; - PyObject *array_function_result = NULL; if (!PyArg_ParseTupleAndKeywords(args, keywds, "O|O&" NPY_SSIZE_T_PYFMT NPY_SSIZE_T_PYFMT "$O:frombuffer", kwlist, @@ -2268,10 +2345,13 @@ array_frombuffer(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *keywds return NULL; } - array_function_result = array_implement_c_array_function_creation( - "frombuffer", args, keywds); - if (array_function_result != Py_NotImplemented) { - return array_function_result; + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "frombuffer", like, args, keywds, NULL, 0, NULL); + if (deferred != Py_NotImplemented) { + Py_XDECREF(type); + return deferred; + } } if (type == NULL) { @@ -2340,7 +2420,7 @@ array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) { PyObject *v, *a, *o = NULL; PyArrayObject *ret; - char* kwlist[] = {"a", "b", "out", NULL }; + static char* kwlist[] = {"a", "b", "out", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O:matrixproduct", kwlist, &a, &v, &o)) { @@ -2843,56 +2923,80 @@ array_fastCopyAndTranspose(PyObject *NPY_UNUSED(dummy), PyObject *args) } static PyObject * -array_correlate(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) +array_correlate(PyObject *NPY_UNUSED(dummy), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *shape, *a0; int mode = 0; - static char *kwlist[] = {"a", "v", "mode", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|i:correlate", kwlist, - &a0, &shape, &mode)) { + if (npy_parse_arguments("correlate", args, len_args, kwnames, + "a", NULL, &a0, + "v", NULL, &shape, + "|mode", &PyArray_CorrelatemodeConverter, &mode, + NULL, NULL, NULL) < 0) { return NULL; } return PyArray_Correlate(a0, shape, mode); } static PyObject* -array_correlate2(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) +array_correlate2(PyObject *NPY_UNUSED(dummy), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *shape, *a0; int mode = 0; - static char *kwlist[] = {"a", "v", "mode", NULL}; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|i:correlate2", kwlist, - &a0, &shape, &mode)) { + if (npy_parse_arguments("correlate2", args, len_args, kwnames, + "a", NULL, &a0, + "v", NULL, &shape, + "|mode", &PyArray_CorrelatemodeConverter, &mode, + NULL, NULL, NULL) < 0) { return NULL; } return PyArray_Correlate2(a0, shape, mode); } static PyObject * -array_arange(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject *kws) { +array_arange(PyObject *NPY_UNUSED(ignored), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) +{ PyObject *o_start = NULL, *o_stop = NULL, *o_step = NULL, *range=NULL; - PyObject *like = NULL; - PyObject *array_function_result = NULL; - static char *kwd[] = {"start", "stop", "step", "dtype", "like", NULL}; PyArray_Descr *typecode = NULL; - - if (!PyArg_ParseTupleAndKeywords(args, kws, "O|OOO&$O:arange", kwd, - &o_start, - &o_stop, - &o_step, - PyArray_DescrConverter2, &typecode, - &like)) { + PyObject *like = NULL; + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("arange", args, len_args, kwnames, + "|start", NULL, &o_start, + "|stop", NULL, &o_stop, + "|step", NULL, &o_step, + "|dtype", &PyArray_DescrConverter2, &typecode, + "$like", NULL, &like, + NULL, NULL, NULL) < 0) { Py_XDECREF(typecode); return NULL; } + if (like != NULL) { + PyObject *deferred = array_implement_c_array_function_creation( + "arange", like, NULL, NULL, args, len_args, kwnames); + if (deferred != Py_NotImplemented) { + Py_XDECREF(typecode); + return deferred; + } + } - array_function_result = array_implement_c_array_function_creation( - "arange", args, kws); - if (array_function_result != Py_NotImplemented) { - Py_XDECREF(typecode); - return array_function_result; + if (o_stop == NULL) { + if (len_args == 0){ + PyErr_SetString(PyExc_TypeError, + "arange() requires stop to be specified."); + Py_XDECREF(typecode); + return NULL; + } + } + else if (o_start == NULL) { + o_start = o_stop; + o_stop = NULL; } range = PyArray_ArangeObj(o_start, o_stop, o_step, typecode); @@ -3252,7 +3356,7 @@ array_can_cast_safely(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *from_obj = NULL; PyArray_Descr *d1 = NULL; PyArray_Descr *d2 = NULL; - npy_bool ret; + int ret; PyObject *retobj = NULL; NPY_CASTING casting = NPY_SAFE_CASTING; static char *kwlist[] = {"from_", "to", "casting", NULL}; @@ -3382,6 +3486,10 @@ array_result_type(PyObject *NPY_UNUSED(dummy), PyObject *args) if (arr[narr] == NULL) { goto finish; } + if (PyLong_CheckExact(obj) || PyFloat_CheckExact(obj) || + PyComplex_CheckExact(obj)) { + ((PyArrayObject_fields *)arr[narr])->flags |= _NPY_ARRAY_WAS_PYSCALAR; + } ++narr; } else { @@ -3427,6 +3535,42 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) return res; } + +static int +trimmode_converter(PyObject *obj, TrimMode *trim) +{ + if (!PyUnicode_Check(obj) || PyUnicode_GetLength(obj) != 1) { + goto error; + } + const char *trimstr = PyUnicode_AsUTF8AndSize(obj, NULL); + + if (trimstr != NULL) { + if (trimstr[0] == 'k') { + *trim = TrimMode_None; + } + else if (trimstr[0] == '.') { + *trim = TrimMode_Zeros; + } + else if (trimstr[0] == '0') { + *trim = TrimMode_LeaveOneZero; + } + else if (trimstr[0] == '-') { + *trim = TrimMode_DptZeros; + } + else { + goto error; + } + } + return NPY_SUCCEED; + +error: + PyErr_Format(PyExc_TypeError, + "if supplied, trim must be 'k', '.', '0' or '-' found `%100S`", + obj); + return NPY_FAIL; +} + + /* * Prints floating-point scalars using the Dragon4 algorithm, scientific mode. * See docstring of `np.format_float_scientific` for description of arguments. @@ -3434,43 +3578,29 @@ array_datetime_data(PyObject *NPY_UNUSED(dummy), PyObject *args) * precision, which is equivalent to `None`. */ static PyObject * -dragon4_scientific(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) +dragon4_scientific(PyObject *NPY_UNUSED(dummy), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *obj; - static char *kwlist[] = {"x", "precision", "unique", "sign", "trim", - "pad_left", "exp_digits", NULL}; - int precision=-1, pad_left=-1, exp_digits=-1; - char *trimstr=NULL; + int precision=-1, pad_left=-1, exp_digits=-1, min_digits=-1; DigitMode digit_mode; TrimMode trim = TrimMode_None; int sign=0, unique=1; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|iiisii:dragon4_scientific", - kwlist, &obj, &precision, &unique, &sign, &trimstr, &pad_left, - &exp_digits)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("dragon4_scientific", args, len_args, kwnames, + "x", NULL , &obj, + "|precision", &PyArray_PythonPyIntFromInt, &precision, + "|unique", &PyArray_PythonPyIntFromInt, &unique, + "|sign", &PyArray_PythonPyIntFromInt, &sign, + "|trim", &trimmode_converter, &trim, + "|pad_left", &PyArray_PythonPyIntFromInt, &pad_left, + "|exp_digits", &PyArray_PythonPyIntFromInt, &exp_digits, + "|min_digits", &PyArray_PythonPyIntFromInt, &min_digits, + NULL, NULL, NULL) < 0) { return NULL; } - if (trimstr != NULL) { - if (strcmp(trimstr, "k") == 0) { - trim = TrimMode_None; - } - else if (strcmp(trimstr, ".") == 0) { - trim = TrimMode_Zeros; - } - else if (strcmp(trimstr, "0") == 0) { - trim = TrimMode_LeaveOneZero; - } - else if (strcmp(trimstr, "-") == 0) { - trim = TrimMode_DptZeros; - } - else { - PyErr_SetString(PyExc_TypeError, - "if supplied, trim must be 'k', '.', '0' or '-'"); - return NULL; - } - } - digit_mode = unique ? DigitMode_Unique : DigitMode_Exact; if (unique == 0 && precision < 0) { @@ -3479,7 +3609,7 @@ dragon4_scientific(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) return NULL; } - return Dragon4_Scientific(obj, digit_mode, precision, sign, trim, + return Dragon4_Scientific(obj, digit_mode, precision, min_digits, sign, trim, pad_left, exp_digits); } @@ -3490,44 +3620,31 @@ dragon4_scientific(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) * precision, which is equivalent to `None`. */ static PyObject * -dragon4_positional(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) +dragon4_positional(PyObject *NPY_UNUSED(dummy), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { PyObject *obj; - static char *kwlist[] = {"x", "precision", "unique", "fractional", - "sign", "trim", "pad_left", "pad_right", NULL}; - int precision=-1, pad_left=-1, pad_right=-1; - char *trimstr=NULL; + int precision=-1, pad_left=-1, pad_right=-1, min_digits=-1; CutoffMode cutoff_mode; DigitMode digit_mode; TrimMode trim = TrimMode_None; int sign=0, unique=1, fractional=0; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|iiiisii:dragon4_positional", - kwlist, &obj, &precision, &unique, &fractional, &sign, &trimstr, - &pad_left, &pad_right)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("dragon4_positional", args, len_args, kwnames, + "x", NULL , &obj, + "|precision", &PyArray_PythonPyIntFromInt, &precision, + "|unique", &PyArray_PythonPyIntFromInt, &unique, + "|fractional", &PyArray_PythonPyIntFromInt, &fractional, + "|sign", &PyArray_PythonPyIntFromInt, &sign, + "|trim", &trimmode_converter, &trim, + "|pad_left", &PyArray_PythonPyIntFromInt, &pad_left, + "|pad_right", &PyArray_PythonPyIntFromInt, &pad_right, + "|min_digits", &PyArray_PythonPyIntFromInt, &min_digits, + NULL, NULL, NULL) < 0) { return NULL; } - if (trimstr != NULL) { - if (strcmp(trimstr, "k") == 0) { - trim = TrimMode_None; - } - else if (strcmp(trimstr, ".") == 0) { - trim = TrimMode_Zeros; - } - else if (strcmp(trimstr, "0") == 0) { - trim = TrimMode_LeaveOneZero; - } - else if (strcmp(trimstr, "-") == 0) { - trim = TrimMode_DptZeros; - } - else { - PyErr_SetString(PyExc_TypeError, - "if supplied, trim must be 'k', '.', '0' or '-'"); - return NULL; - } - } - digit_mode = unique ? DigitMode_Unique : DigitMode_Exact; cutoff_mode = fractional ? CutoffMode_FractionLength : CutoffMode_TotalLength; @@ -3538,8 +3655,8 @@ dragon4_positional(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) return NULL; } - return Dragon4_Positional(obj, digit_mode, cutoff_mode, precision, sign, - trim, pad_left, pad_right); + return Dragon4_Positional(obj, digit_mode, cutoff_mode, precision, + min_digits, sign, trim, pad_left, pad_right); } static PyObject * @@ -3558,7 +3675,7 @@ format_longfloat(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject *kwds) "not a longfloat"); return NULL; } - return Dragon4_Scientific(obj, DigitMode_Unique, precision, 0, + return Dragon4_Scientific(obj, DigitMode_Unique, precision, -1, 0, TrimMode_LeaveOneZero, -1, -1); } @@ -4053,15 +4170,19 @@ array_may_share_memory(PyObject *NPY_UNUSED(ignored), PyObject *args, PyObject * } static PyObject * -normalize_axis_index(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) +normalize_axis_index(PyObject *NPY_UNUSED(self), + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - static char *kwlist[] = {"axis", "ndim", "msg_prefix", NULL}; int axis; int ndim; PyObject *msg_prefix = Py_None; + NPY_PREPARE_ARGPARSER; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "ii|O:normalize_axis_index", - kwlist, &axis, &ndim, &msg_prefix)) { + if (npy_parse_arguments("normalize_axis_index", args, len_args, kwnames, + "axis", &PyArray_PythonPyIntFromInt, &axis, + "ndim", &PyArray_PythonPyIntFromInt, &ndim, + "|msg_prefix", NULL, &msg_prefix, + NULL, NULL, NULL) < 0) { return NULL; } if (check_and_adjust_axis_msg(&axis, ndim, msg_prefix) < 0) { @@ -4072,6 +4193,42 @@ normalize_axis_index(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) } +static PyObject * +_reload_guard(PyObject *NPY_UNUSED(self)) { + static int initialized = 0; + +#if !defined(PYPY_VERSION) + if (PyThreadState_Get()->interp != PyInterpreterState_Main()) { + if (PyErr_WarnEx(PyExc_UserWarning, + "NumPy was imported from a Python sub-interpreter but " + "NumPy does not properly support sub-interpreters. " + "This will likely work for most users but might cause hard to " + "track down issues or subtle bugs. " + "A common user of the rare sub-interpreter feature is wsgi " + "which also allows single-interpreter mode.\n" + "Improvements in the case of bugs are welcome, but is not " + "on the NumPy roadmap, and full support may require " + "significant effort to achieve.", 2) < 0) { + return NULL; + } + /* No need to give the other warning in a sub-interpreter as well... */ + initialized = 1; + Py_RETURN_NONE; + } +#endif + if (initialized) { + if (PyErr_WarnEx(PyExc_UserWarning, + "The NumPy module was reloaded (imported a second time). " + "This can in some cases result in small but subtle issues " + "and is discouraged.", 2) < 0) { + return NULL; + } + } + initialized = 1; + Py_RETURN_NONE; +} + + static struct PyMethodDef array_module_methods[] = { {"_get_implementing_args", (PyCFunction)array__get_implementing_args, @@ -4095,8 +4252,20 @@ static struct PyMethodDef array_module_methods[] = { (PyCFunction)array_set_typeDict, METH_VARARGS, NULL}, {"array", - (PyCFunction)_array_fromobject, - METH_VARARGS|METH_KEYWORDS, NULL}, + (PyCFunction)array_array, + METH_FASTCALL | METH_KEYWORDS, NULL}, + {"asarray", + (PyCFunction)array_asarray, + METH_FASTCALL | METH_KEYWORDS, NULL}, + {"asanyarray", + (PyCFunction)array_asanyarray, + METH_FASTCALL | METH_KEYWORDS, NULL}, + {"ascontiguousarray", + (PyCFunction)array_ascontiguousarray, + METH_FASTCALL | METH_KEYWORDS, NULL}, + {"asfortranarray", + (PyCFunction)array_asfortranarray, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"copyto", (PyCFunction)array_copyto, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4105,16 +4274,16 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS|METH_KEYWORDS, NULL}, {"arange", (PyCFunction)array_arange, - METH_VARARGS|METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"zeros", (PyCFunction)array_zeros, - METH_VARARGS|METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"count_nonzero", (PyCFunction)array_count_nonzero, METH_VARARGS|METH_KEYWORDS, NULL}, {"empty", (PyCFunction)array_empty, - METH_VARARGS|METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"empty_like", (PyCFunction)array_empty_like, METH_VARARGS|METH_KEYWORDS, NULL}, @@ -4156,10 +4325,10 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS, NULL}, {"correlate", (PyCFunction)array_correlate, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"correlate2", (PyCFunction)array_correlate2, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"frombuffer", (PyCFunction)array_frombuffer, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -4206,10 +4375,10 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"dragon4_positional", (PyCFunction)dragon4_positional, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"dragon4_scientific", (PyCFunction)dragon4_scientific, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"compare_chararrays", (PyCFunction)compare_chararrays, METH_VARARGS | METH_KEYWORDS, NULL}, @@ -4242,7 +4411,7 @@ static struct PyMethodDef array_module_methods[] = { {"unpackbits", (PyCFunction)io_unpack, METH_VARARGS | METH_KEYWORDS, NULL}, {"normalize_axis_index", (PyCFunction)normalize_axis_index, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"set_legacy_print_mode", (PyCFunction)set_legacy_print_mode, METH_VARARGS, NULL}, {"_discover_array_parameters", (PyCFunction)_discover_array_parameters, @@ -4263,6 +4432,9 @@ static struct PyMethodDef array_module_methods[] = { METH_VARARGS, NULL}, {"_set_madvise_hugepage", (PyCFunction)_set_madvise_hugepage, METH_O, NULL}, + {"_reload_guard", (PyCFunction)_reload_guard, + METH_NOARGS, + "Give a warning on reload and big warning in sub-interpreters."}, {NULL, NULL, 0, NULL} /* sentinel */ }; @@ -4525,13 +4697,6 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) { goto err; } - /* - * Before calling PyType_Ready, initialize the tp_hash slot in - * PyArray_Type to work around mingw32 not being able initialize - * static structure slots with functions from the Python C_API. - */ - PyArray_Type.tp_hash = PyObject_HashNotImplemented; - if (PyType_Ready(&PyUFunc_Type) < 0) { goto err; } diff --git a/numpy/core/src/multiarray/nditer_api.c b/numpy/core/src/multiarray/nditer_api.c index 059f2c437b15..a1ca5bff51d4 100644 --- a/numpy/core/src/multiarray/nditer_api.c +++ b/numpy/core/src/multiarray/nditer_api.c @@ -1557,6 +1557,8 @@ NpyIter_DebugPrint(NpyIter *iter) if (itflags&NPY_ITFLAG_BUFFER) { NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter); + NpyIter_TransferInfo *transferinfo = NBF_TRANSFERINFO(bufferdata); + printf("| BufferData:\n"); printf("| BufferSize: %d\n", (int)NBF_BUFFERSIZE(bufferdata)); printf("| Size: %d\n", (int)NBF_SIZE(bufferdata)); @@ -1598,19 +1600,19 @@ NpyIter_DebugPrint(NpyIter *iter) } printf("| ReadTransferFn: "); for (iop = 0; iop < nop; ++iop) - printf("%p ", (void *)NBF_READTRANSFERFN(bufferdata)[iop]); + printf("%p ", (void *)transferinfo[iop].read.func); printf("\n"); printf("| ReadTransferData: "); for (iop = 0; iop < nop; ++iop) - printf("%p ", (void *)NBF_READTRANSFERDATA(bufferdata)[iop]); + printf("%p ", (void *)transferinfo[iop].read.auxdata); printf("\n"); printf("| WriteTransferFn: "); for (iop = 0; iop < nop; ++iop) - printf("%p ", (void *)NBF_WRITETRANSFERFN(bufferdata)[iop]); + printf("%p ", (void *)transferinfo[iop].write.func); printf("\n"); printf("| WriteTransferData: "); for (iop = 0; iop < nop; ++iop) - printf("%p ", (void *)NBF_WRITETRANSFERDATA(bufferdata)[iop]); + printf("%p ", (void *)transferinfo[iop].write.auxdata); printf("\n"); printf("| Buffers: "); for (iop = 0; iop < nop; ++iop) @@ -1758,6 +1760,9 @@ npyiter_allocate_buffers(NpyIter *iter, char **errmsg) } goto fail; } + if (PyDataType_FLAGCHK(op_dtype[iop], NPY_NEEDS_INIT)) { + memset(buffer, '\0', itemsize*buffersize); + } buffers[iop] = buffer; } } @@ -1889,9 +1894,6 @@ npyiter_copy_from_buffers(NpyIter *iter) npy_intp reduce_outerdim = 0; npy_intp *reduce_outerstrides = NULL; - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - npy_intp axisdata_incr = NIT_AXISDATA_SIZEOF(itflags, ndim, nop) / NPY_SIZEOF_INTP; @@ -1909,9 +1911,8 @@ npyiter_copy_from_buffers(NpyIter *iter) transfersize *= NBF_REDUCE_OUTERSIZE(bufferdata); } + NpyIter_TransferInfo *transferinfo = NBF_TRANSFERINFO(bufferdata); for (iop = 0; iop < nop; ++iop) { - stransfer = NBF_WRITETRANSFERFN(bufferdata)[iop]; - transferdata = NBF_WRITETRANSFERDATA(bufferdata)[iop]; buffer = buffers[iop]; /* * Copy the data back to the arrays. If the type has refs, @@ -1920,7 +1921,7 @@ npyiter_copy_from_buffers(NpyIter *iter) * The flag USINGBUFFER is set when the buffer was used, so * only copy back when this flag is on. */ - if ((stransfer != NULL) && + if ((transferinfo[iop].write.func != NULL) && (op_itflags[iop]&(NPY_OP_ITFLAG_WRITE|NPY_OP_ITFLAG_USINGBUFFER)) == (NPY_OP_ITFLAG_WRITE|NPY_OP_ITFLAG_USINGBUFFER)) { npy_intp op_transfersize; @@ -2011,8 +2012,7 @@ npyiter_copy_from_buffers(NpyIter *iter) dst_coords, axisdata_incr, dst_shape, axisdata_incr, op_transfersize, dtypes[iop]->elsize, - (PyArray_MaskedStridedUnaryOp *)stransfer, - transferdata) < 0) { + &transferinfo[iop].write) < 0) { return -1; } } @@ -2024,27 +2024,28 @@ npyiter_copy_from_buffers(NpyIter *iter) dst_coords, axisdata_incr, dst_shape, axisdata_incr, op_transfersize, dtypes[iop]->elsize, - stransfer, - transferdata) < 0) { + &transferinfo[iop].write) < 0) { return -1; } } } /* If there's no copy back, we may have to decrement refs. In - * this case, the transfer function has a 'decsrcref' transfer - * function, so we can use it to do the decrement. + * this case, the transfer is instead a function which clears + * (DECREFs) the single input. * * The flag USINGBUFFER is set when the buffer was used, so * only decrement refs when this flag is on. */ - else if (stransfer != NULL && + else if (transferinfo[iop].write.func != NULL && (op_itflags[iop]&NPY_OP_ITFLAG_USINGBUFFER) != 0) { NPY_IT_DBG_PRINT1("Iterator: Freeing refs and zeroing buffer " "of operand %d\n", (int)iop); /* Decrement refs */ - if (stransfer(NULL, 0, buffer, dtypes[iop]->elsize, - transfersize, dtypes[iop]->elsize, - transferdata) < 0) { + npy_intp buf_stride = dtypes[iop]->elsize; + if (transferinfo[iop].write.func( + &transferinfo[iop].write.context, + &buffer, &transfersize, &buf_stride, + transferinfo[iop].write.auxdata) < 0) { /* Since this should only decrement, it should never error */ assert(0); return -1; @@ -2094,9 +2095,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) npy_intp *reduce_outerstrides = NULL; char **reduce_outerptrs = NULL; - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; - /* * Have to get this flag before npyiter_checkreducesize sets * it for the next iteration. @@ -2207,13 +2205,9 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) is_onestride = 1; } + NpyIter_TransferInfo *transferinfo = NBF_TRANSFERINFO(bufferdata); for (iop = 0; iop < nop; ++iop) { - /* - * If the buffer is write-only, these two are NULL, and the buffer - * pointers will be set up but the read copy won't be done - */ - stransfer = NBF_READTRANSFERFN(bufferdata)[iop]; - transferdata = NBF_READTRANSFERDATA(bufferdata)[iop]; + switch (op_itflags[iop]& (NPY_OP_ITFLAG_BUFNEVER| NPY_OP_ITFLAG_CAST| @@ -2231,8 +2225,8 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) * could be zero, but strides[iop] was initialized * to the first non-trivial stride. */ - stransfer = NULL; /* The flag NPY_OP_ITFLAG_USINGBUFFER can be ignored here */ + assert(!(op_itflags[iop] & NPY_OP_ITFLAG_USINGBUFFER)); break; /* Never need to buffer this operand */ case NPY_OP_ITFLAG_BUFNEVER|NPY_OP_ITFLAG_REDUCE: @@ -2244,8 +2238,8 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) * could be zero, but strides[iop] was initialized * to the first non-trivial stride. */ - stransfer = NULL; /* The flag NPY_OP_ITFLAG_USINGBUFFER can be ignored here */ + assert(!(op_itflags[iop] & NPY_OP_ITFLAG_USINGBUFFER)); break; /* Just a copy */ case 0: @@ -2263,7 +2257,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) if (is_onestride) { ptrs[iop] = ad_ptrs[iop]; strides[iop] = ad_strides[iop]; - stransfer = NULL; /* Signal that the buffer is not being used */ op_itflags[iop] &= (~NPY_OP_ITFLAG_USINGBUFFER); } @@ -2278,7 +2271,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) strides[iop] = ad_strides[iop]; reduce_outerstrides[iop] = NAD_STRIDES(reduce_outeraxisdata)[iop]; - stransfer = NULL; /* Signal that the buffer is not being used */ op_itflags[iop] &= (~NPY_OP_ITFLAG_USINGBUFFER); } @@ -2309,7 +2301,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) NPY_IT_DBG_PRINT1("reduce op %d all one stride\n", (int)iop); ptrs[iop] = ad_ptrs[iop]; reduce_outerstrides[iop] = 0; - stransfer = NULL; /* Signal that the buffer is not being used */ op_itflags[iop] &= (~NPY_OP_ITFLAG_USINGBUFFER); } @@ -2324,7 +2315,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) /* Outer reduce loop advances by one item */ reduce_outerstrides[iop] = NAD_STRIDES(reduce_outeraxisdata)[iop]; - stransfer = NULL; /* Signal that the buffer is not being used */ op_itflags[iop] &= (~NPY_OP_ITFLAG_USINGBUFFER); } @@ -2350,7 +2340,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) ptrs[iop] = ad_ptrs[iop]; strides[iop] = ad_strides[iop]; reduce_outerstrides[iop] = 0; - stransfer = NULL; /* Signal that the buffer is not being used */ op_itflags[iop] &= (~NPY_OP_ITFLAG_USINGBUFFER); } @@ -2365,7 +2354,6 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) /* Outer reduce loop advances by one item */ reduce_outerstrides[iop] = NAD_STRIDES(reduce_outeraxisdata)[iop]; - stransfer = NULL; /* Signal that the buffer is not being used */ op_itflags[iop] &= (~NPY_OP_ITFLAG_USINGBUFFER); } @@ -2443,7 +2431,12 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) break; } - if (stransfer != NULL) { + /* + * If OP_ITFLAG_USINGBUFFER is enabled and the read func is not NULL, + * the buffer needs to be read. + */ + if (op_itflags[iop] & NPY_OP_ITFLAG_USINGBUFFER && + transferinfo[iop].read.func != NULL) { npy_intp src_itemsize; npy_intp op_transfersize; @@ -2454,7 +2447,7 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) src_itemsize = PyArray_DTYPE(operands[iop])->elsize; - /* If stransfer wasn't set to NULL, buffering is required */ + /* If we reach here, buffering is required */ any_buffered = 1; /* @@ -2539,41 +2532,33 @@ npyiter_copy_to_buffers(NpyIter *iter, char **prev_dataptrs) skip_transfer = 1; } - /* If the data type requires zero-inititialization */ - if (PyDataType_FLAGCHK(dtypes[iop], NPY_NEEDS_INIT)) { - NPY_IT_DBG_PRINT("Iterator: Buffer requires init, " - "memsetting to 0\n"); - memset(ptrs[iop], 0, dtypes[iop]->elsize*op_transfersize); - /* Can't skip the transfer in this case */ - skip_transfer = 0; - } - - if (!skip_transfer) { + /* + * Copy data to the buffers if necessary. + * + * We always copy if the operand has references. In that case + * a "write" function must be in use that either copies or clears + * the buffer. + * This write from buffer call does not check for skip-transfer + * so we have to assume the buffer is cleared. For dtypes that + * do not have references, we can assume that the write function + * will leave the source (buffer) unmodified. + */ + if (!skip_transfer || PyDataType_REFCHK(dtypes[iop])) { NPY_IT_DBG_PRINT2("Iterator: Copying operand %d to " "buffer (%d items)\n", (int)iop, (int)op_transfersize); if (PyArray_TransferNDimToStrided( - ndim_transfer, ptrs[iop], dst_stride, - ad_ptrs[iop], src_strides, axisdata_incr, - src_coords, axisdata_incr, - src_shape, axisdata_incr, - op_transfersize, src_itemsize, - stransfer, transferdata) < 0) { + ndim_transfer, ptrs[iop], dst_stride, + ad_ptrs[iop], src_strides, axisdata_incr, + src_coords, axisdata_incr, + src_shape, axisdata_incr, + op_transfersize, src_itemsize, + &transferinfo[iop].read) < 0) { return -1; } } } - else if (ptrs[iop] == buffers[iop]) { - /* If the data type requires zero-inititialization */ - if (PyDataType_FLAGCHK(dtypes[iop], NPY_NEEDS_INIT)) { - NPY_IT_DBG_PRINT1("Iterator: Write-only buffer for " - "operand %d requires init, " - "memsetting to 0\n", (int)iop); - memset(ptrs[iop], 0, dtypes[iop]->elsize*transfersize); - } - } - } /* @@ -2640,6 +2625,7 @@ npyiter_clear_buffers(NpyIter *iter) /* Cleanup any buffers with references */ char **buffers = NBF_BUFFERS(bufferdata); PyArray_Descr **dtypes = NIT_DTYPES(iter); + npyiter_opitflags *op_itflags = NIT_OPITFLAGS(iter); for (int iop = 0; iop < nop; ++iop, ++buffers) { /* * We may want to find a better way to do this, on the other hand, @@ -2648,7 +2634,8 @@ npyiter_clear_buffers(NpyIter *iter) * a well defined state (either NULL or owning the reference). * Only we implement cleanup */ - if (!PyDataType_REFCHK(dtypes[iop])) { + if (!PyDataType_REFCHK(dtypes[iop]) || + !(op_itflags[iop]&NPY_OP_ITFLAG_USINGBUFFER)) { continue; } if (*buffers == 0) { diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index b379a28ac3ae..a0154e4744f6 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -235,8 +235,8 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, NBF_SIZE(bufferdata) = 0; memset(NBF_BUFFERS(bufferdata), 0, nop*NPY_SIZEOF_INTP); memset(NBF_PTRS(bufferdata), 0, nop*NPY_SIZEOF_INTP); - memset(NBF_READTRANSFERDATA(bufferdata), 0, nop*NPY_SIZEOF_INTP); - memset(NBF_WRITETRANSFERDATA(bufferdata), 0, nop*NPY_SIZEOF_INTP); + /* Ensure that the transferdata/auxdata is NULLed */ + memset(NBF_TRANSFERINFO(bufferdata), 0, nop * sizeof(NpyIter_TransferInfo)); } /* Fill in the AXISDATA arrays and set the ITERSIZE field */ @@ -577,13 +577,11 @@ NpyIter_Copy(NpyIter *iter) NpyIter_BufferData *bufferdata; npy_intp buffersize, itemsize; char **buffers; - NpyAuxData **readtransferdata, **writetransferdata; bufferdata = NIT_BUFFERDATA(newiter); buffers = NBF_BUFFERS(bufferdata); - readtransferdata = NBF_READTRANSFERDATA(bufferdata); - writetransferdata = NBF_WRITETRANSFERDATA(bufferdata); buffersize = NBF_BUFFERSIZE(bufferdata); + NpyIter_TransferInfo *transferinfo = NBF_TRANSFERINFO(bufferdata); for (iop = 0; iop < nop; ++iop) { if (buffers[iop] != NULL) { @@ -596,30 +594,33 @@ NpyIter_Copy(NpyIter *iter) if (buffers[iop] == NULL) { out_of_memory = 1; } + else { + if (PyDataType_FLAGCHK(dtypes[iop], NPY_NEEDS_INIT)) { + memset(buffers[iop], '\0', itemsize*buffersize); + } + } } } - if (readtransferdata[iop] != NULL) { + if (transferinfo[iop].read.func != NULL) { if (out_of_memory) { - readtransferdata[iop] = NULL; + transferinfo[iop].read.func = NULL; /* No cleanup */ } else { - readtransferdata[iop] = - NPY_AUXDATA_CLONE(readtransferdata[iop]); - if (readtransferdata[iop] == NULL) { + if (NPY_cast_info_copy(&transferinfo[iop].read, + &transferinfo[iop].read) < 0) { out_of_memory = 1; } } } - if (writetransferdata[iop] != NULL) { + if (transferinfo[iop].write.func != NULL) { if (out_of_memory) { - writetransferdata[iop] = NULL; + transferinfo[iop].write.func = NULL; /* No cleanup */ } else { - writetransferdata[iop] = - NPY_AUXDATA_CLONE(writetransferdata[iop]); - if (writetransferdata[iop] == NULL) { + if (NPY_cast_info_copy(&transferinfo[iop].write, + &transferinfo[iop].write) < 0) { out_of_memory = 1; } } @@ -688,26 +689,18 @@ NpyIter_Deallocate(NpyIter *iter) NpyIter_BufferData *bufferdata = NIT_BUFFERDATA(iter); char **buffers; - NpyAuxData **transferdata; /* buffers */ buffers = NBF_BUFFERS(bufferdata); for (iop = 0; iop < nop; ++iop, ++buffers) { PyArray_free(*buffers); } + + NpyIter_TransferInfo *transferinfo = NBF_TRANSFERINFO(bufferdata); /* read bufferdata */ - transferdata = NBF_READTRANSFERDATA(bufferdata); - for(iop = 0; iop < nop; ++iop, ++transferdata) { - if (*transferdata) { - NPY_AUXDATA_FREE(*transferdata); - } - } - /* write bufferdata */ - transferdata = NBF_WRITETRANSFERDATA(bufferdata); - for(iop = 0; iop < nop; ++iop, ++transferdata) { - if (*transferdata) { - NPY_AUXDATA_FREE(*transferdata); - } + for (iop = 0; iop < nop; ++iop, ++transferinfo) { + NPY_cast_info_xfree(&transferinfo->read); + NPY_cast_info_xfree(&transferinfo->write); } } @@ -3142,13 +3135,8 @@ npyiter_allocate_transfer_functions(NpyIter *iter) PyArrayObject **op = NIT_OPERANDS(iter); PyArray_Descr **op_dtype = NIT_DTYPES(iter); npy_intp *strides = NAD_STRIDES(axisdata), op_stride; - PyArray_StridedUnaryOp **readtransferfn = NBF_READTRANSFERFN(bufferdata), - **writetransferfn = NBF_WRITETRANSFERFN(bufferdata); - NpyAuxData **readtransferdata = NBF_READTRANSFERDATA(bufferdata), - **writetransferdata = NBF_WRITETRANSFERDATA(bufferdata); + NpyIter_TransferInfo *transferinfo = NBF_TRANSFERINFO(bufferdata); - PyArray_StridedUnaryOp *stransfer = NULL; - NpyAuxData *transferdata = NULL; int needs_api = 0; for (iop = 0; iop < nop; ++iop) { @@ -3174,17 +3162,14 @@ npyiter_allocate_transfer_functions(NpyIter *iter) PyArray_DESCR(op[iop]), op_dtype[iop], move_references, - &stransfer, - &transferdata, + &transferinfo[iop].read, &needs_api) != NPY_SUCCEED) { iop -= 1; /* This one cannot be cleaned up yet. */ goto fail; } - readtransferfn[iop] = stransfer; - readtransferdata[iop] = transferdata; } else { - readtransferfn[iop] = NULL; + transferinfo[iop].read.func = NULL; } if (flags & NPY_OP_ITFLAG_WRITE) { int move_references = 1; @@ -3200,38 +3185,33 @@ npyiter_allocate_transfer_functions(NpyIter *iter) * could be inconsistent. */ if (PyArray_GetMaskedDTypeTransferFunction( - (flags & NPY_OP_ITFLAG_ALIGNED) != 0, - op_dtype[iop]->elsize, - op_stride, - (strides[maskop] == mask_dtype->elsize) ? - mask_dtype->elsize : - NPY_MAX_INTP, - op_dtype[iop], - PyArray_DESCR(op[iop]), - mask_dtype, - move_references, - (PyArray_MaskedStridedUnaryOp **)&stransfer, - &transferdata, - &needs_api) != NPY_SUCCEED) { + (flags & NPY_OP_ITFLAG_ALIGNED) != 0, + op_dtype[iop]->elsize, + op_stride, + (strides[maskop] == mask_dtype->elsize) ? + mask_dtype->elsize : NPY_MAX_INTP, + op_dtype[iop], + PyArray_DESCR(op[iop]), + mask_dtype, + move_references, + &transferinfo[iop].write, + &needs_api) != NPY_SUCCEED) { goto fail; } } else { if (PyArray_GetDTypeTransferFunction( - (flags & NPY_OP_ITFLAG_ALIGNED) != 0, - op_dtype[iop]->elsize, - op_stride, - op_dtype[iop], - PyArray_DESCR(op[iop]), - move_references, - &stransfer, - &transferdata, - &needs_api) != NPY_SUCCEED) { + (flags & NPY_OP_ITFLAG_ALIGNED) != 0, + op_dtype[iop]->elsize, + op_stride, + op_dtype[iop], + PyArray_DESCR(op[iop]), + move_references, + &transferinfo[iop].write, + &needs_api) != NPY_SUCCEED) { goto fail; } } - writetransferfn[iop] = stransfer; - writetransferdata[iop] = transferdata; } /* If no write back but there are references make a decref fn */ else if (PyDataType_REFCHK(op_dtype[iop])) { @@ -3241,25 +3221,22 @@ npyiter_allocate_transfer_functions(NpyIter *iter) * src references. */ if (PyArray_GetDTypeTransferFunction( - (flags & NPY_OP_ITFLAG_ALIGNED) != 0, - op_dtype[iop]->elsize, 0, - op_dtype[iop], NULL, - 1, - &stransfer, - &transferdata, - &needs_api) != NPY_SUCCEED) { + (flags & NPY_OP_ITFLAG_ALIGNED) != 0, + op_dtype[iop]->elsize, 0, + op_dtype[iop], NULL, + 1, + &transferinfo[iop].write, + &needs_api) != NPY_SUCCEED) { goto fail; } - writetransferfn[iop] = stransfer; - writetransferdata[iop] = transferdata; } else { - writetransferfn[iop] = NULL; + transferinfo[iop].write.func = NULL; } } else { - readtransferfn[iop] = NULL; - writetransferfn[iop] = NULL; + transferinfo[iop].read.func = NULL; + transferinfo[iop].write.func = NULL; } } @@ -3272,14 +3249,8 @@ npyiter_allocate_transfer_functions(NpyIter *iter) fail: for (i = 0; i < iop+1; ++i) { - if (readtransferdata[iop] != NULL) { - NPY_AUXDATA_FREE(readtransferdata[iop]); - readtransferdata[iop] = NULL; - } - if (writetransferdata[iop] != NULL) { - NPY_AUXDATA_FREE(writetransferdata[iop]); - writetransferdata[iop] = NULL; - } + NPY_cast_info_xfree(&transferinfo[iop].read); + NPY_cast_info_xfree(&transferinfo[iop].write); } return 0; } diff --git a/numpy/core/src/multiarray/nditer_impl.h b/numpy/core/src/multiarray/nditer_impl.h index 378d6f7117b3..a5a9177e5f97 100644 --- a/numpy/core/src/multiarray/nditer_impl.h +++ b/numpy/core/src/multiarray/nditer_impl.h @@ -21,6 +21,7 @@ #include "convert_datatype.h" #include "lowlevel_strided_loops.h" +#include "dtype_transfer.h" /********** ITERATOR CONSTRUCTION TIMING **************/ #define NPY_IT_CONSTRUCTION_TIMING 0 @@ -148,8 +149,9 @@ struct NpyIter_InternalOnly { char iter_flexdata; }; -typedef struct NpyIter_AD NpyIter_AxisData; -typedef struct NpyIter_BD NpyIter_BufferData; +typedef struct NpyIter_AxisData_tag NpyIter_AxisData; +typedef struct NpyIter_TransferInfo_tag NpyIter_TransferInfo; +typedef struct NpyIter_BufferData_tag NpyIter_BufferData; typedef npy_int16 npyiter_opitflags; @@ -167,7 +169,8 @@ typedef npy_int16 npyiter_opitflags; #define NIT_OPITFLAGS_SIZEOF(itflags, ndim, nop) \ (NPY_INTP_ALIGNED(sizeof(npyiter_opitflags) * nop)) #define NIT_BUFFERDATA_SIZEOF(itflags, ndim, nop) \ - ((itflags&NPY_ITFLAG_BUFFER) ? ((NPY_SIZEOF_INTP)*(6 + 9*nop)) : 0) + ((itflags&NPY_ITFLAG_BUFFER) ? ( \ + (NPY_SIZEOF_INTP)*(6 + 5*nop) + sizeof(NpyIter_TransferInfo) * nop) : 0) /* Byte offsets of the iterator members starting from iter->iter_flexdata */ #define NIT_PERM_OFFSET() \ @@ -229,11 +232,20 @@ typedef npy_int16 npyiter_opitflags; &(iter)->iter_flexdata + NIT_AXISDATA_OFFSET(itflags, ndim, nop))) /* Internal-only BUFFERDATA MEMBER ACCESS */ -struct NpyIter_BD { + +struct NpyIter_TransferInfo_tag { + NPY_cast_info read; + NPY_cast_info write; + /* Probably unnecessary, but make sure what follows is intp aligned: */ + npy_intp _unused_ensure_alignment[]; +}; + +struct NpyIter_BufferData_tag { npy_intp buffersize, size, bufiterend, reduce_pos, reduce_outersize, reduce_outerdim; npy_intp bd_flexdata; }; + #define NBF_BUFFERSIZE(bufferdata) ((bufferdata)->buffersize) #define NBF_SIZE(bufferdata) ((bufferdata)->size) #define NBF_BUFITEREND(bufferdata) ((bufferdata)->bufiterend) @@ -248,19 +260,13 @@ struct NpyIter_BD { (&(bufferdata)->bd_flexdata + 2*(nop))) #define NBF_REDUCE_OUTERPTRS(bufferdata) ((char **) \ (&(bufferdata)->bd_flexdata + 3*(nop))) -#define NBF_READTRANSFERFN(bufferdata) ((PyArray_StridedUnaryOp **) \ +#define NBF_BUFFERS(bufferdata) ((char **) \ (&(bufferdata)->bd_flexdata + 4*(nop))) -#define NBF_READTRANSFERDATA(bufferdata) ((NpyAuxData **) \ +#define NBF_TRANSFERINFO(bufferdata) ((NpyIter_TransferInfo *) \ (&(bufferdata)->bd_flexdata + 5*(nop))) -#define NBF_WRITETRANSFERFN(bufferdata) ((PyArray_StridedUnaryOp **) \ - (&(bufferdata)->bd_flexdata + 6*(nop))) -#define NBF_WRITETRANSFERDATA(bufferdata) ((NpyAuxData **) \ - (&(bufferdata)->bd_flexdata + 7*(nop))) -#define NBF_BUFFERS(bufferdata) ((char **) \ - (&(bufferdata)->bd_flexdata + 8*(nop))) /* Internal-only AXISDATA MEMBER ACCESS. */ -struct NpyIter_AD { +struct NpyIter_AxisData_tag { npy_intp shape, index; npy_intp ad_flexdata; }; diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 8839d1be717d..7698ae43d07e 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -1134,6 +1134,9 @@ static void npyiter_dealloc(NewNpyArrayIterObject *self) { if (self->iter) { + /* Store error, so that WriteUnraisable cannot clear an existing one */ + PyObject *exc, *val, *tb; + PyErr_Fetch(&exc, &val, &tb); if (npyiter_has_writeback(self->iter)) { if (PyErr_WarnEx(PyExc_RuntimeWarning, "Temporary data has not been written back to one of the " @@ -1152,10 +1155,13 @@ npyiter_dealloc(NewNpyArrayIterObject *self) } } } - NpyIter_Deallocate(self->iter); + if (!NpyIter_Deallocate(self->iter)) { + PyErr_WriteUnraisable(Py_None); + } self->iter = NULL; Py_XDECREF(self->nested_child); self->nested_child = NULL; + PyErr_Restore(exc, val, tb); } Py_TYPE(self)->tp_free((PyObject*)self); } @@ -1507,8 +1513,7 @@ npyiter_next(NewNpyArrayIterObject *self) static PyObject *npyiter_shape_get(NewNpyArrayIterObject *self) { - PyObject *ret; - npy_intp idim, ndim, shape[NPY_MAXDIMS]; + npy_intp ndim, shape[NPY_MAXDIMS]; if (self->iter == NULL || self->finished) { PyErr_SetString(PyExc_ValueError, @@ -1518,14 +1523,7 @@ static PyObject *npyiter_shape_get(NewNpyArrayIterObject *self) if (NpyIter_GetShape(self->iter, shape) == NPY_SUCCEED) { ndim = NpyIter_GetNDim(self->iter); - ret = PyTuple_New(ndim); - if (ret != NULL) { - for (idim = 0; idim < ndim; ++idim) { - PyTuple_SET_ITEM(ret, idim, - PyLong_FromLong(shape[idim])); - } - return ret; - } + return PyArray_IntTupleFromIntp(ndim, shape); } return NULL; @@ -1533,8 +1531,7 @@ static PyObject *npyiter_shape_get(NewNpyArrayIterObject *self) static PyObject *npyiter_multi_index_get(NewNpyArrayIterObject *self) { - PyObject *ret; - npy_intp idim, ndim, multi_index[NPY_MAXDIMS]; + npy_intp ndim, multi_index[NPY_MAXDIMS]; if (self->iter == NULL || self->finished) { PyErr_SetString(PyExc_ValueError, @@ -1545,15 +1542,7 @@ static PyObject *npyiter_multi_index_get(NewNpyArrayIterObject *self) if (self->get_multi_index != NULL) { ndim = NpyIter_GetNDim(self->iter); self->get_multi_index(self->iter, multi_index); - ret = PyTuple_New(ndim); - if (ret == NULL) { - return NULL; - } - for (idim = 0; idim < ndim; ++idim) { - PyTuple_SET_ITEM(ret, idim, - PyLong_FromLong(multi_index[idim])); - } - return ret; + return PyArray_IntTupleFromIntp(ndim, multi_index); } else { if (!NpyIter_HasMultiIndex(self->iter)) { @@ -2320,7 +2309,7 @@ npyiter_close(NewNpyArrayIterObject *self) self->iter = NULL; Py_XDECREF(self->nested_child); self->nested_child = NULL; - if (ret < 0) { + if (ret != NPY_SUCCEED) { return NULL; } Py_RETURN_NONE; @@ -2363,7 +2352,7 @@ static PyMethodDef npyiter_methods[] = { {"__exit__", (PyCFunction)npyiter_exit, METH_VARARGS, NULL}, {"close", (PyCFunction)npyiter_close, - METH_VARARGS, NULL}, + METH_NOARGS, NULL}, {NULL, NULL, 0, NULL}, }; diff --git a/numpy/core/src/multiarray/number.c b/numpy/core/src/multiarray/number.c index a629dfe97fde..a62776748775 100644 --- a/numpy/core/src/multiarray/number.c +++ b/numpy/core/src/multiarray/number.c @@ -16,6 +16,9 @@ #include "binop_override.h" #include "ufunc_override.h" +#include "abstractdtypes.h" +#include "common_dtype.h" +#include "convert_datatype.h" /************************************************************************* **************** Implement Number Protocol **************************** @@ -227,10 +230,7 @@ PyArray_GenericReduceFunction(PyArrayObject *m1, PyObject *op, int axis, { PyObject *args, *ret = NULL, *meth; PyObject *kwds; - if (op == NULL) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + args = Py_BuildValue("(Oi)", m1, axis); kwds = _get_keywords(rtype, out); meth = PyObject_GetAttrString(op, "reduce"); @@ -250,10 +250,7 @@ PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, { PyObject *args, *ret = NULL, *meth; PyObject *kwds; - if (op == NULL) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + args = Py_BuildValue("(Oi)", m1, axis); kwds = _get_keywords(rtype, out); meth = PyObject_GetAttrString(op, "accumulate"); @@ -268,30 +265,14 @@ PyArray_GenericAccumulateFunction(PyArrayObject *m1, PyObject *op, int axis, NPY_NO_EXPORT PyObject * -PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op) +PyArray_GenericBinaryFunction(PyObject *m1, PyObject *m2, PyObject *op) { - /* - * I suspect that the next few lines are buggy and cause NotImplemented to - * be returned at weird times... but if we raise an error here, then - * *everything* breaks. (Like, 'arange(10) + 1' and just - * 'repr(arange(10))' both blow up with an error here.) Not sure what's - * going on with that, but I'll leave it alone for now. - njs, 2015-06-21 - */ - if (op == NULL) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - return PyObject_CallFunctionObjArgs(op, m1, m2, NULL); } NPY_NO_EXPORT PyObject * PyArray_GenericUnaryFunction(PyArrayObject *m1, PyObject *op) { - if (op == NULL) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } return PyObject_CallFunctionObjArgs(op, m1, NULL); } @@ -299,25 +280,17 @@ static PyObject * PyArray_GenericInplaceBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op) { - if (op == NULL) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } return PyObject_CallFunctionObjArgs(op, m1, m2, m1, NULL); } static PyObject * PyArray_GenericInplaceUnaryFunction(PyArrayObject *m1, PyObject *op) { - if (op == NULL) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } return PyObject_CallFunctionObjArgs(op, m1, m1, NULL); } static PyObject * -array_add(PyArrayObject *m1, PyObject *m2) +array_add(PyObject *m1, PyObject *m2) { PyObject *res; @@ -329,7 +302,7 @@ array_add(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_subtract(PyArrayObject *m1, PyObject *m2) +array_subtract(PyObject *m1, PyObject *m2) { PyObject *res; @@ -341,7 +314,7 @@ array_subtract(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_multiply(PyArrayObject *m1, PyObject *m2) +array_multiply(PyObject *m1, PyObject *m2) { PyObject *res; @@ -353,14 +326,14 @@ array_multiply(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_remainder(PyArrayObject *m1, PyObject *m2) +array_remainder(PyObject *m1, PyObject *m2) { BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_remainder, array_remainder); return PyArray_GenericBinaryFunction(m1, m2, n_ops.remainder); } static PyObject * -array_divmod(PyArrayObject *m1, PyObject *m2) +array_divmod(PyObject *m1, PyObject *m2) { BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_divmod, array_divmod); return PyArray_GenericBinaryFunction(m1, m2, n_ops.divmod); @@ -368,7 +341,7 @@ array_divmod(PyArrayObject *m1, PyObject *m2) /* Need this to be version dependent on account of the slot check */ static PyObject * -array_matrix_multiply(PyArrayObject *m1, PyObject *m2) +array_matrix_multiply(PyObject *m1, PyObject *m2) { BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_matrix_multiply, array_matrix_multiply); return PyArray_GenericBinaryFunction(m1, m2, n_ops.matmul); @@ -472,15 +445,16 @@ is_scalar_with_conversion(PyObject *o2, double* out_exponent) * the result is in value (can be NULL if an error occurred) */ static int -fast_scalar_power(PyArrayObject *a1, PyObject *o2, int inplace, +fast_scalar_power(PyObject *o1, PyObject *o2, int inplace, PyObject **value) { double exponent; NPY_SCALARKIND kind; /* NPY_NOSCALAR is not scalar */ - if (PyArray_Check(a1) && - !PyArray_ISOBJECT(a1) && + if (PyArray_Check(o1) && + !PyArray_ISOBJECT((PyArrayObject *)o1) && ((kind=is_scalar_with_conversion(o2, &exponent))>0)) { + PyArrayObject *a1 = (PyArrayObject *)o1; PyObject *fastop = NULL; if (PyArray_ISFLOAT(a1) || PyArray_ISCOMPLEX(a1)) { if (exponent == 1.0) { @@ -544,7 +518,7 @@ fast_scalar_power(PyArrayObject *a1, PyObject *o2, int inplace, } static PyObject * -array_power(PyArrayObject *a1, PyObject *o2, PyObject *modulo) +array_power(PyObject *a1, PyObject *o2, PyObject *modulo) { PyObject *value = NULL; @@ -635,7 +609,7 @@ array_invert(PyArrayObject *m1) } static PyObject * -array_left_shift(PyArrayObject *m1, PyObject *m2) +array_left_shift(PyObject *m1, PyObject *m2) { PyObject *res; @@ -647,7 +621,7 @@ array_left_shift(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_right_shift(PyArrayObject *m1, PyObject *m2) +array_right_shift(PyObject *m1, PyObject *m2) { PyObject *res; @@ -659,7 +633,7 @@ array_right_shift(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_bitwise_and(PyArrayObject *m1, PyObject *m2) +array_bitwise_and(PyObject *m1, PyObject *m2) { PyObject *res; @@ -671,7 +645,7 @@ array_bitwise_and(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_bitwise_or(PyArrayObject *m1, PyObject *m2) +array_bitwise_or(PyObject *m1, PyObject *m2) { PyObject *res; @@ -683,7 +657,7 @@ array_bitwise_or(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_bitwise_xor(PyArrayObject *m1, PyObject *m2) +array_bitwise_xor(PyObject *m1, PyObject *m2) { PyObject *res; @@ -734,7 +708,7 @@ array_inplace_power(PyArrayObject *a1, PyObject *o2, PyObject *NPY_UNUSED(modulo INPLACE_GIVE_UP_IF_NEEDED( a1, o2, nb_inplace_power, array_inplace_power); - if (fast_scalar_power(a1, o2, 1, &value) != 0) { + if (fast_scalar_power((PyObject *)a1, o2, 1, &value) != 0) { value = PyArray_GenericInplaceBinaryFunction(a1, o2, n_ops.power); } return value; @@ -781,7 +755,7 @@ array_inplace_bitwise_xor(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_floor_divide(PyArrayObject *m1, PyObject *m2) +array_floor_divide(PyObject *m1, PyObject *m2) { PyObject *res; @@ -793,13 +767,14 @@ array_floor_divide(PyArrayObject *m1, PyObject *m2) } static PyObject * -array_true_divide(PyArrayObject *m1, PyObject *m2) +array_true_divide(PyObject *m1, PyObject *m2) { PyObject *res; + PyArrayObject *a1 = (PyArrayObject *)m1; BINOP_GIVE_UP_IF_NEEDED(m1, m2, nb_true_divide, array_true_divide); if (PyArray_CheckExact(m1) && - (PyArray_ISFLOAT(m1) || PyArray_ISCOMPLEX(m1)) && + (PyArray_ISFLOAT(a1) || PyArray_ISCOMPLEX(a1)) && try_binary_elide(m1, m2, &array_inplace_true_divide, &res, 0)) { return res; } @@ -930,22 +905,22 @@ array_index(PyArrayObject *v) NPY_NO_EXPORT PyNumberMethods array_as_number = { - .nb_add = (binaryfunc)array_add, - .nb_subtract = (binaryfunc)array_subtract, - .nb_multiply = (binaryfunc)array_multiply, - .nb_remainder = (binaryfunc)array_remainder, - .nb_divmod = (binaryfunc)array_divmod, + .nb_add = array_add, + .nb_subtract = array_subtract, + .nb_multiply = array_multiply, + .nb_remainder = array_remainder, + .nb_divmod = array_divmod, .nb_power = (ternaryfunc)array_power, .nb_negative = (unaryfunc)array_negative, .nb_positive = (unaryfunc)array_positive, .nb_absolute = (unaryfunc)array_absolute, .nb_bool = (inquiry)_array_nonzero, .nb_invert = (unaryfunc)array_invert, - .nb_lshift = (binaryfunc)array_left_shift, - .nb_rshift = (binaryfunc)array_right_shift, - .nb_and = (binaryfunc)array_bitwise_and, - .nb_xor = (binaryfunc)array_bitwise_xor, - .nb_or = (binaryfunc)array_bitwise_or, + .nb_lshift = array_left_shift, + .nb_rshift = array_right_shift, + .nb_and = array_bitwise_and, + .nb_xor = array_bitwise_xor, + .nb_or = array_bitwise_or, .nb_int = (unaryfunc)array_int, .nb_float = (unaryfunc)array_float, @@ -962,11 +937,11 @@ NPY_NO_EXPORT PyNumberMethods array_as_number = { .nb_inplace_xor = (binaryfunc)array_inplace_bitwise_xor, .nb_inplace_or = (binaryfunc)array_inplace_bitwise_or, - .nb_floor_divide = (binaryfunc)array_floor_divide, - .nb_true_divide = (binaryfunc)array_true_divide, + .nb_floor_divide = array_floor_divide, + .nb_true_divide = array_true_divide, .nb_inplace_floor_divide = (binaryfunc)array_inplace_floor_divide, .nb_inplace_true_divide = (binaryfunc)array_inplace_true_divide, - .nb_matrix_multiply = (binaryfunc)array_matrix_multiply, + .nb_matrix_multiply = array_matrix_multiply, .nb_inplace_matrix_multiply = (binaryfunc)array_inplace_matrix_multiply, }; diff --git a/numpy/core/src/multiarray/number.h b/numpy/core/src/multiarray/number.h index 643241b3d02f..4f426f964ca7 100644 --- a/numpy/core/src/multiarray/number.h +++ b/numpy/core/src/multiarray/number.h @@ -56,7 +56,7 @@ NPY_NO_EXPORT PyObject * _PyArray_GetNumericOps(void); NPY_NO_EXPORT PyObject * -PyArray_GenericBinaryFunction(PyArrayObject *m1, PyObject *m2, PyObject *op); +PyArray_GenericBinaryFunction(PyObject *m1, PyObject *m2, PyObject *op); NPY_NO_EXPORT PyObject * PyArray_GenericUnaryFunction(PyArrayObject *m1, PyObject *op); diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index d018fccbbd22..9930f7791d6e 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -331,13 +331,13 @@ format_@name@(@type@ val, npy_bool scientific, { if (scientific) { return Dragon4_Scientific_@Name@(&val, - DigitMode_Unique, precision, + DigitMode_Unique, precision, -1, sign, trim, pad_left, exp_digits); } else { return Dragon4_Positional_@Name@(&val, DigitMode_Unique, CutoffMode_TotalLength, precision, - sign, trim, pad_left, pad_right); + -1, sign, trim, pad_left, pad_right); } } @@ -2395,6 +2395,10 @@ gentype_arrtype_getbuffer(PyObject *self, Py_buffer *view, int flags) self); return -1; } + if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) { + PyErr_SetString(PyExc_BufferError, "scalar buffer is readonly"); + return -1; + } PyArray_Descr *descr = PyArray_DescrFromScalar(self); if (descr == NULL) { return -1; @@ -2413,6 +2417,7 @@ gentype_arrtype_getbuffer(PyObject *self, Py_buffer *view, int flags) view->shape = NULL; view->strides = NULL; view->suboffsets = NULL; + view->readonly = 1; /* assume general (user) scalars are readonly. */ Py_INCREF(self); view->obj = self; view->buf = scalar_value(self, descr); @@ -2444,6 +2449,7 @@ static int @name@_getbuffer(PyObject *self, Py_buffer *view, int flags) { if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) { + PyErr_SetString(PyExc_BufferError, "scalar buffer is readonly"); return -1; } Py@Name@ScalarObject *scalar = (Py@Name@ScalarObject *)self; @@ -2456,6 +2462,7 @@ static int view->shape = NULL; view->strides = NULL; view->suboffsets = NULL; + view->readonly = 1; Py_INCREF(self); view->obj = self; view->buf = &(scalar->obval); @@ -2482,6 +2489,7 @@ static int unicode_getbuffer(PyObject *self, Py_buffer *view, int flags) { if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) { + PyErr_SetString(PyExc_BufferError, "scalar buffer is readonly"); return -1; } PyUnicodeScalarObject *scalar = (PyUnicodeScalarObject *)self; @@ -2493,6 +2501,7 @@ unicode_getbuffer(PyObject *self, Py_buffer *view, int flags) view->shape = NULL; view->strides = NULL; view->suboffsets = NULL; + view->readonly = 1; Py_INCREF(self); view->obj = self; @@ -2522,7 +2531,7 @@ unicode_getbuffer(PyObject *self, Py_buffer *view, int flags) view->format = scalar->buffer_fmt; } else { - scalar->buffer_fmt = PyObject_Malloc(22); + scalar->buffer_fmt = PyMem_Malloc(22); if (scalar->buffer_fmt == NULL) { Py_SETREF(view->obj, NULL); return -1; @@ -2549,6 +2558,7 @@ static int @name@_getbuffer(PyObject *self, Py_buffer *view, int flags) { if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) { + PyErr_SetString(PyExc_BufferError, "scalar buffer is readonly"); return -1; } Py@Name@ScalarObject *scalar = (Py@Name@ScalarObject *)self; @@ -2560,6 +2570,7 @@ static int view->shape = &length; view->strides = NULL; view->suboffsets = NULL; + view->readonly = 1; Py_INCREF(self); view->obj = self; @@ -2651,6 +2662,7 @@ unicode_arrtype_dealloc(PyObject *v) { /* note: may be null if it was never requested */ PyMem_Free(PyArrayScalar_VAL(v, Unicode)); + PyMem_Free(((PyUnicodeScalarObject *)v)->buffer_fmt); /* delegate to the base class */ PyUnicode_Type.tp_dealloc(v); } @@ -2699,7 +2711,7 @@ static PyObject * /* TODO: include type name in error message, which is not @name@ */ PyObject *obj = NULL; - char *kwnames[] = {"", NULL}; /* positional-only */ + static char *kwnames[] = {"", NULL}; /* positional-only */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwnames, &obj)) { return NULL; } @@ -2787,7 +2799,7 @@ static PyObject * object_arrtype_new(PyTypeObject *NPY_UNUSED(type), PyObject *args, PyObject *kwds) { PyObject *obj = Py_None; - char *kwnames[] = {"", NULL}; /* positional-only */ + static char *kwnames[] = {"", NULL}; /* positional-only */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:object_", kwnames, &obj)) { return NULL; } @@ -2813,7 +2825,7 @@ static PyObject * PyObject *obj = NULL, *meta_obj = NULL; Py@Name@ScalarObject *ret; - char *kwnames[] = {"", "", NULL}; /* positional-only */ + static char *kwnames[] = {"", "", NULL}; /* positional-only */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OO", kwnames, &obj, &meta_obj)) { return NULL; } @@ -2872,7 +2884,7 @@ bool_arrtype_new(PyTypeObject *NPY_UNUSED(type), PyObject *args, PyObject *kwds) PyObject *obj = NULL; PyArrayObject *arr; - char *kwnames[] = {"", NULL}; /* positional-only */ + static char *kwnames[] = {"", NULL}; /* positional-only */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O:bool_", kwnames, &obj)) { return NULL; } @@ -2983,7 +2995,7 @@ void_arrtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *obj, *arr; PyObject *new = NULL; - char *kwnames[] = {"", NULL}; /* positional-only */ + static char *kwnames[] = {"", NULL}; /* positional-only */ if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:void", kwnames, &obj)) { return NULL; } @@ -3160,7 +3172,7 @@ static npy_hash_t static npy_hash_t @lname@_arrtype_hash(PyObject *obj) { - return _Py_HashDouble((double) PyArrayScalar_VAL(obj, @name@)); + return Npy_HashDouble(obj, (double)PyArrayScalar_VAL(obj, @name@)); } /* borrowed from complex_hash */ @@ -3168,14 +3180,14 @@ static npy_hash_t c@lname@_arrtype_hash(PyObject *obj) { npy_hash_t hashreal, hashimag, combined; - hashreal = _Py_HashDouble((double) - PyArrayScalar_VAL(obj, C@name@).real); + hashreal = Npy_HashDouble( + obj, (double)PyArrayScalar_VAL(obj, C@name@).real); if (hashreal == -1) { return -1; } - hashimag = _Py_HashDouble((double) - PyArrayScalar_VAL(obj, C@name@).imag); + hashimag = Npy_HashDouble( + obj, (double)PyArrayScalar_VAL(obj, C@name@).imag); if (hashimag == -1) { return -1; } @@ -3190,7 +3202,8 @@ c@lname@_arrtype_hash(PyObject *obj) static npy_hash_t half_arrtype_hash(PyObject *obj) { - return _Py_HashDouble(npy_half_to_double(PyArrayScalar_VAL(obj, Half))); + return Npy_HashDouble( + obj, npy_half_to_double(PyArrayScalar_VAL(obj, Half))); } static npy_hash_t diff --git a/numpy/core/src/multiarray/temp_elide.c b/numpy/core/src/multiarray/temp_elide.c index b19dee418821..2b4621744427 100644 --- a/numpy/core/src/multiarray/temp_elide.c +++ b/numpy/core/src/multiarray/temp_elide.c @@ -274,13 +274,14 @@ check_callers(int * cannot) * "cannot" is set to true if it cannot be done even with swapped arguments */ static int -can_elide_temp(PyArrayObject * alhs, PyObject * orhs, int * cannot) +can_elide_temp(PyObject *olhs, PyObject *orhs, int *cannot) { /* * to be a candidate the array needs to have reference count 1, be an exact * array of a basic type, own its data and size larger than threshold */ - if (Py_REFCNT(alhs) != 1 || !PyArray_CheckExact(alhs) || + PyArrayObject *alhs = (PyArrayObject *)olhs; + if (Py_REFCNT(olhs) != 1 || !PyArray_CheckExact(olhs) || !PyArray_ISNUMBER(alhs) || !PyArray_CHKFLAGS(alhs, NPY_ARRAY_OWNDATA) || !PyArray_ISWRITEABLE(alhs) || @@ -328,22 +329,22 @@ can_elide_temp(PyArrayObject * alhs, PyObject * orhs, int * cannot) * try eliding a binary op, if commutative is true also try swapped arguments */ NPY_NO_EXPORT int -try_binary_elide(PyArrayObject * m1, PyObject * m2, +try_binary_elide(PyObject * m1, PyObject * m2, PyObject * (inplace_op)(PyArrayObject * m1, PyObject * m2), PyObject ** res, int commutative) { /* set when no elision can be done independent of argument order */ int cannot = 0; if (can_elide_temp(m1, m2, &cannot)) { - *res = inplace_op(m1, m2); + *res = inplace_op((PyArrayObject *)m1, m2); #if NPY_ELIDE_DEBUG != 0 puts("elided temporary in binary op"); #endif return 1; } else if (commutative && !cannot) { - if (can_elide_temp((PyArrayObject *)m2, (PyObject *)m1, &cannot)) { - *res = inplace_op((PyArrayObject *)m2, (PyObject *)m1); + if (can_elide_temp(m2, m1, &cannot)) { + *res = inplace_op((PyArrayObject *)m2, m1); #if NPY_ELIDE_DEBUG != 0 puts("elided temporary in commutative binary op"); #endif diff --git a/numpy/core/src/multiarray/temp_elide.h b/numpy/core/src/multiarray/temp_elide.h index d073adf28ec6..206bb025381e 100644 --- a/numpy/core/src/multiarray/temp_elide.h +++ b/numpy/core/src/multiarray/temp_elide.h @@ -8,7 +8,7 @@ NPY_NO_EXPORT int can_elide_temp_unary(PyArrayObject * m1); NPY_NO_EXPORT int -try_binary_elide(PyArrayObject * m1, PyObject * m2, +try_binary_elide(PyObject * m1, PyObject * m2, PyObject * (inplace_op)(PyArrayObject * m1, PyObject * m2), PyObject ** res, int commutative); diff --git a/numpy/core/src/multiarray/usertypes.c b/numpy/core/src/multiarray/usertypes.c index 3eaf99196875..15d46800c471 100644 --- a/numpy/core/src/multiarray/usertypes.c +++ b/numpy/core/src/multiarray/usertypes.c @@ -235,7 +235,7 @@ PyArray_RegisterDataType(PyArray_Descr *descr) !PyDict_CheckExact(descr->fields)) { PyErr_Format(PyExc_ValueError, "Failed to register dtype for %S: Legacy user dtypes " - "using `NPY_ITEM_IS_POINTER` or `NPY_ITEM_REFCOUNT` are" + "using `NPY_ITEM_IS_POINTER` or `NPY_ITEM_REFCOUNT` are " "unsupported. It is possible to create such a dtype only " "if it is a structured dtype with names and fields " "hardcoded at registration time.\n" @@ -538,7 +538,7 @@ PyArray_AddLegacyWrapping_CastingImpl( if (from == to) { spec.flags = NPY_METH_REQUIRES_PYAPI | NPY_METH_SUPPORTS_UNALIGNED; PyType_Slot slots[] = { - {NPY_METH_get_loop, NULL}, + {NPY_METH_get_loop, &legacy_cast_get_strided_loop}, {NPY_METH_resolve_descriptors, &legacy_same_dtype_resolve_descriptors}, {0, NULL}}; spec.slots = slots; @@ -547,7 +547,7 @@ PyArray_AddLegacyWrapping_CastingImpl( else { spec.flags = NPY_METH_REQUIRES_PYAPI; PyType_Slot slots[] = { - {NPY_METH_get_loop, NULL}, + {NPY_METH_get_loop, &legacy_cast_get_strided_loop}, {NPY_METH_resolve_descriptors, &simple_cast_resolve_descriptors}, {0, NULL}}; spec.slots = slots; diff --git a/numpy/core/src/npysort/timsort.c.src b/numpy/core/src/npysort/timsort.c.src index 3fdd46f61c51..5298f5a1d057 100644 --- a/numpy/core/src/npysort/timsort.c.src +++ b/numpy/core/src/npysort/timsort.c.src @@ -507,9 +507,7 @@ timsort_@suff@(void *start, npy_intp num, void *NPY_UNUSED(varr)) ret = 0; cleanup: - if (buffer.pw != NULL) { - free(buffer.pw); - } + free(buffer.pw); return ret; } diff --git a/numpy/core/src/umath/_umath_tests.c.src b/numpy/core/src/umath/_umath_tests.c.src index 750fbeb92a7b..2e79d377e3d4 100644 --- a/numpy/core/src/umath/_umath_tests.c.src +++ b/numpy/core/src/umath/_umath_tests.c.src @@ -532,7 +532,7 @@ UMath_Tests_test_signature(PyObject *NPY_UNUSED(dummy), PyObject *args) } if (f->core_dim_ixs != NULL) { core_dim_ixs = PyTuple_New(core_num_ixs); - if (core_num_dims == NULL) { + if (core_dim_ixs == NULL) { goto fail; } for (i = 0; i < core_num_ixs; i++) { @@ -614,12 +614,14 @@ UMath_Tests_test_dispatch(PyObject *NPY_UNUSED(dummy), PyObject *NPY_UNUSED(dumm if (item == NULL || PyDict_SetItemString(dict, "@str@", item) < 0) { goto err; } + Py_DECREF(item); /**end repeat**/ item = PyList_New(0); if (item == NULL || PyDict_SetItemString(dict, "all", item) < 0) { goto err; } NPY_CPU_DISPATCH_CALL_ALL(_umath_tests_dispatch_attach, (item)); + Py_SETREF(item, NULL); if (PyErr_Occurred()) { goto err; } diff --git a/numpy/core/src/umath/_umath_tests.dispatch.c b/numpy/core/src/umath/_umath_tests.dispatch.c index d86a54411367..85f3650106ea 100644 --- a/numpy/core/src/umath/_umath_tests.dispatch.c +++ b/numpy/core/src/umath/_umath_tests.dispatch.c @@ -29,5 +29,6 @@ void NPY_CPU_DISPATCH_CURFX(_umath_tests_dispatch_attach)(PyObject *list) PyObject *item = PyUnicode_FromString(NPY_TOSTRING(NPY_CPU_DISPATCH_CURFX(func))); if (item) { PyList_Append(list, item); + Py_DECREF(item); } } diff --git a/numpy/core/src/umath/fast_loop_macros.h b/numpy/core/src/umath/fast_loop_macros.h index 74bf016436dd..4a36c9721879 100644 --- a/numpy/core/src/umath/fast_loop_macros.h +++ b/numpy/core/src/umath/fast_loop_macros.h @@ -10,6 +10,21 @@ #ifndef _NPY_UMATH_FAST_LOOP_MACROS_H_ #define _NPY_UMATH_FAST_LOOP_MACROS_H_ +/* + * MAX_STEP_SIZE is used to determine if we need to use SIMD version of the ufunc. + * Very large step size can be as slow as processing it using scalar. The + * value of 2097152 ( = 2MB) was chosen using 2 considerations: + * 1) Typical linux kernel page size is 4Kb, but sometimes it could also be 2MB + * which is == 2097152 Bytes. For a step size as large as this, surely all + * the loads/stores of gather/scatter instructions falls on 16 different pages + * which one would think would slow down gather/scatter instructions. + * 2) It additionally satisfies MAX_STEP_SIZE*16/esize < NPY_MAX_INT32 which + * allows us to use i32 version of gather/scatter (as opposed to the i64 version) + * without problems (step larger than NPY_MAX_INT32*esize/16 would require use of + * i64gather/scatter). esize = element size = 4/8 bytes for float/double. + */ +#define MAX_STEP_SIZE 2097152 + static NPY_INLINE npy_uintp abs_ptrdiff(char *a, char *b) { @@ -46,14 +61,20 @@ abs_ptrdiff(char *a, char *b) npy_intp i;\ for(i = 0; i < n; i++, ip1 += is1, op1 += os1, op2 += os2) -/** (ip1, ip2) -> (op1) */ -#define BINARY_LOOP\ +#define BINARY_DEFS\ char *ip1 = args[0], *ip2 = args[1], *op1 = args[2];\ npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];\ npy_intp n = dimensions[0];\ npy_intp i;\ + +#define BINARY_LOOP_SLIDING\ for(i = 0; i < n; i++, ip1 += is1, ip2 += is2, op1 += os1) +/** (ip1, ip2) -> (op1) */ +#define BINARY_LOOP\ + BINARY_DEFS\ + BINARY_LOOP_SLIDING + /** (ip1, ip2) -> (op1, op2) */ #define BINARY_LOOP_TWO_OUT\ char *ip1 = args[0], *ip2 = args[1], *op1 = args[2], *op2 = args[3];\ @@ -155,10 +176,7 @@ abs_ptrdiff(char *a, char *b) #define IVDEP_LOOP #endif #define BASE_BINARY_LOOP_INP(tin, tout, op) \ - char *ip1 = args[0], *ip2 = args[1], *op1 = args[2];\ - npy_intp is1 = steps[0], is2 = steps[1], os1 = steps[2];\ - npy_intp n = dimensions[0];\ - npy_intp i;\ + BINARY_DEFS\ IVDEP_LOOP \ for(i = 0; i < n; i++, ip1 += is1, ip2 += is2, op1 += os1) { \ const tin in1 = *(tin *)ip1; \ @@ -234,5 +252,117 @@ abs_ptrdiff(char *a, char *b) TYPE io1 = *(TYPE *)iop1; \ BINARY_REDUCE_LOOP_INNER +#define IS_BINARY_STRIDE_ONE(esize, vsize) \ + ((steps[0] == esize) && \ + (steps[1] == esize) && \ + (steps[2] == esize) && \ + (abs_ptrdiff(args[2], args[0]) >= vsize) && \ + (abs_ptrdiff(args[2], args[1]) >= vsize)) + +/* + * stride is equal to element size and input and destination are equal or + * don't overlap within one register. The check of the steps against + * esize also quarantees that steps are >= 0. + */ +#define IS_BLOCKABLE_UNARY(esize, vsize) \ + (steps[0] == (esize) && steps[0] == steps[1] && \ + (npy_is_aligned(args[0], esize) && npy_is_aligned(args[1], esize)) && \ + ((abs_ptrdiff(args[1], args[0]) >= (vsize)) || \ + ((abs_ptrdiff(args[1], args[0]) == 0)))) + +/* + * Avoid using SIMD for very large step sizes for several reasons: + * 1) Supporting large step sizes requires use of i64gather/scatter_ps instructions, + * in which case we need two i64gather instructions and an additional vinsertf32x8 + * instruction to load a single zmm register (since one i64gather instruction + * loads into a ymm register). This is not ideal for performance. + * 2) Gather and scatter instructions can be slow when the loads/stores + * cross page boundaries. + * + * We instead rely on i32gather/scatter_ps instructions which use a 32-bit index + * element. The index needs to be < INT_MAX to avoid overflow. MAX_STEP_SIZE + * ensures this. The condition also requires that the input and output arrays + * should have no overlap in memory. + */ +#define IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP \ + ((labs(steps[0]) < MAX_STEP_SIZE) && \ + (labs(steps[1]) < MAX_STEP_SIZE) && \ + (labs(steps[2]) < MAX_STEP_SIZE) && \ + (nomemoverlap(args[0], steps[0] * dimensions[0], args[2], steps[2] * dimensions[0])) && \ + (nomemoverlap(args[1], steps[1] * dimensions[0], args[2], steps[2] * dimensions[0]))) + +#define IS_UNARY_TWO_OUT_SMALL_STEPS_AND_NOMEMOVERLAP \ + ((labs(steps[0]) < MAX_STEP_SIZE) && \ + (labs(steps[1]) < MAX_STEP_SIZE) && \ + (labs(steps[2]) < MAX_STEP_SIZE) && \ + (nomemoverlap(args[0], steps[0] * dimensions[0], args[2], steps[2] * dimensions[0])) && \ + (nomemoverlap(args[0], steps[0] * dimensions[0], args[1], steps[1] * dimensions[0]))) + +/* + * 1) Output should be contiguous, can handle strided input data + * 2) Input step should be smaller than MAX_STEP_SIZE for performance + * 3) Input and output arrays should have no overlap in memory + */ +#define IS_OUTPUT_BLOCKABLE_UNARY(esizein, esizeout, vsize) \ + ((steps[0] & (esizein-1)) == 0 && \ + steps[1] == (esizeout) && llabs(steps[0]) < MAX_STEP_SIZE && \ + (nomemoverlap(args[1], steps[1] * dimensions[0], args[0], steps[0] * dimensions[0]))) + +#define IS_BLOCKABLE_REDUCE(esize, vsize) \ + (steps[1] == (esize) && abs_ptrdiff(args[1], args[0]) >= (vsize) && \ + npy_is_aligned(args[1], (esize)) && \ + npy_is_aligned(args[0], (esize))) + +#define IS_BLOCKABLE_BINARY(esize, vsize) \ + (steps[0] == steps[1] && steps[1] == steps[2] && steps[2] == (esize) && \ + npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[1], (esize)) && \ + npy_is_aligned(args[0], (esize)) && \ + (abs_ptrdiff(args[2], args[0]) >= (vsize) || \ + abs_ptrdiff(args[2], args[0]) == 0) && \ + (abs_ptrdiff(args[2], args[1]) >= (vsize) || \ + abs_ptrdiff(args[2], args[1]) >= 0)) + +#define IS_BLOCKABLE_BINARY_SCALAR1(esize, vsize) \ + (steps[0] == 0 && steps[1] == steps[2] && steps[2] == (esize) && \ + npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[1], (esize)) && \ + ((abs_ptrdiff(args[2], args[1]) >= (vsize)) || \ + (abs_ptrdiff(args[2], args[1]) == 0)) && \ + abs_ptrdiff(args[2], args[0]) >= (esize)) + +#define IS_BLOCKABLE_BINARY_SCALAR2(esize, vsize) \ + (steps[1] == 0 && steps[0] == steps[2] && steps[2] == (esize) && \ + npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[0], (esize)) && \ + ((abs_ptrdiff(args[2], args[0]) >= (vsize)) || \ + (abs_ptrdiff(args[2], args[0]) == 0)) && \ + abs_ptrdiff(args[2], args[1]) >= (esize)) + +#undef abs_ptrdiff + +#define IS_BLOCKABLE_BINARY_BOOL(esize, vsize) \ + (steps[0] == (esize) && steps[0] == steps[1] && steps[2] == (1) && \ + npy_is_aligned(args[1], (esize)) && \ + npy_is_aligned(args[0], (esize))) + +#define IS_BLOCKABLE_BINARY_SCALAR1_BOOL(esize, vsize) \ + (steps[0] == 0 && steps[1] == (esize) && steps[2] == (1) && \ + npy_is_aligned(args[1], (esize))) + +#define IS_BLOCKABLE_BINARY_SCALAR2_BOOL(esize, vsize) \ + (steps[0] == (esize) && steps[1] == 0 && steps[2] == (1) && \ + npy_is_aligned(args[0], (esize))) + +/* align var to alignment */ +#define LOOP_BLOCK_ALIGN_VAR(var, type, alignment)\ + npy_intp i, peel = npy_aligned_block_offset(var, sizeof(type),\ + alignment, n);\ + for(i = 0; i < peel; i++) + +#define LOOP_BLOCKED(type, vsize)\ + for(; i < npy_blocked_end(peel, sizeof(type), vsize, n);\ + i += (vsize / sizeof(type))) + +#define LOOP_BLOCKED_END\ + for (; i < n; i++) + #endif /* _NPY_UMATH_FAST_LOOP_MACROS_H_ */ diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index c2e06a4fd922..683bd0178bf0 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -20,6 +20,9 @@ #include /* for memchr */ +/* Use Libdivide for faster division */ +#include "numpy/libdivide/libdivide.h" + /* * cutoff blocksize for pairwise summation * decreasing it decreases errors slightly as more pairs are summed but @@ -570,7 +573,7 @@ NPY_NO_EXPORT void /**begin repeat1 * #isa = , _avx2# * #ISA = , AVX2# - * #CHK = 1, HAVE_ATTRIBUTE_TARGET_AVX2# + * #CHK = 1, defined(HAVE_ATTRIBUTE_TARGET_AVX2)# * #ATTR = , NPY_GCC_TARGET_AVX2# */ @@ -840,32 +843,6 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 void UNARY_LOOP_FAST(@type@, @type@, *out = in > 0 ? 1 : (in < 0 ? -1 : 0)); } -NPY_NO_EXPORT void -@TYPE@_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) -{ - BINARY_LOOP { - const @type@ in1 = *(@type@ *)ip1; - const @type@ in2 = *(@type@ *)ip2; - /* - * FIXME: On x86 at least, dividing the smallest representable integer - * by -1 causes a SIFGPE (division overflow). We treat this case here - * (to avoid a SIGFPE crash at python level), but a good solution would - * be to treat integer division problems separately from FPU exceptions - * (i.e. a different approach than npy_set_floatstatus_divbyzero()). - */ - if (in2 == 0 || (in1 == NPY_MIN_@TYPE@ && in2 == -1)) { - npy_set_floatstatus_divbyzero(); - *((@type@ *)op1) = 0; - } - else if (((in1 > 0) != (in2 > 0)) && (in1 % in2 != 0)) { - *((@type@ *)op1) = in1/in2 - 1; - } - else { - *((@type@ *)op1) = in1/in2; - } - } -} - NPY_NO_EXPORT void @TYPE@_remainder(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -951,22 +928,6 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 void UNARY_LOOP_FAST(@type@, @type@, *out = in > 0 ? 1 : 0); } -NPY_NO_EXPORT void -@TYPE@_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) -{ - BINARY_LOOP { - const @type@ in1 = *(@type@ *)ip1; - const @type@ in2 = *(@type@ *)ip2; - if (in2 == 0) { - npy_set_floatstatus_divbyzero(); - *((@type@ *)op1) = 0; - } - else { - *((@type@ *)op1)= in1/in2; - } - } -} - NPY_NO_EXPORT void @TYPE@_remainder(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -1352,14 +1313,48 @@ TIMEDELTA_dm_m_multiply(char **args, npy_intp const *dimensions, npy_intp const NPY_NO_EXPORT void TIMEDELTA_mq_m_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { - BINARY_LOOP { - const npy_timedelta in1 = *(npy_timedelta *)ip1; + /* NOTE: This code is similar to array floor divide */ + BINARY_DEFS + + /* When the divisor is a constant, use libdivide for faster division */ + if (steps[1] == 0) { + /* In case of empty array, just return */ + if (n == 0) { + return; + } + const npy_int64 in2 = *(npy_int64 *)ip2; - if (in1 == NPY_DATETIME_NAT || in2 == 0) { - *((npy_timedelta *)op1) = NPY_DATETIME_NAT; + + /* If divisor is 0, we need not compute anything */ + if (in2 == 0) { + npy_set_floatstatus_divbyzero(); + BINARY_LOOP_SLIDING { + *((npy_timedelta *)op1) = NPY_DATETIME_NAT; + } } else { - *((npy_timedelta *)op1) = in1 / in2; + struct libdivide_s64_t fast_d = libdivide_s64_gen(in2); + BINARY_LOOP_SLIDING { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + if (in1 == NPY_DATETIME_NAT) { + *((npy_timedelta *)op1) = NPY_DATETIME_NAT; + } + else { + *((npy_timedelta *)op1) = libdivide_s64_do(in1, &fast_d);; + } + } + } + } + else { + BINARY_LOOP_SLIDING { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + const npy_int64 in2 = *(npy_int64 *)ip2; + if (in1 == NPY_DATETIME_NAT || in2 == 0) { + *((npy_timedelta *)op1) = NPY_DATETIME_NAT; + } + else { + *((npy_timedelta *)op1) = in1 / in2; + } } } } @@ -1431,23 +1426,69 @@ TIMEDELTA_mm_m_remainder(char **args, npy_intp const *dimensions, npy_intp const NPY_NO_EXPORT void TIMEDELTA_mm_q_floor_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { - BINARY_LOOP { - const npy_timedelta in1 = *(npy_timedelta *)ip1; - const npy_timedelta in2 = *(npy_timedelta *)ip2; - if (in1 == NPY_DATETIME_NAT || in2 == NPY_DATETIME_NAT) { - npy_set_floatstatus_invalid(); - *((npy_int64 *)op1) = 0; + /* NOTE: This code is similar to array floor divide */ + BINARY_DEFS + + /* When the divisor is a constant, use libdivide for faster division */ + if (steps[1] == 0) { + /* In case of empty array, just return */ + if (n == 0) { + return; } - else if (in2 == 0) { + + const npy_timedelta in2 = *(npy_timedelta *)ip2; + + /* If divisor is 0 or NAT, we need not compute anything */ + if (in2 == 0) { npy_set_floatstatus_divbyzero(); - *((npy_int64 *)op1) = 0; + BINARY_LOOP_SLIDING { + *((npy_int64 *)op1) = 0; + } + } + else if (in2 == NPY_DATETIME_NAT) { + npy_set_floatstatus_invalid(); + BINARY_LOOP_SLIDING { + *((npy_int64 *)op1) = 0; + } } else { - if (((in1 > 0) != (in2 > 0)) && (in1 % in2 != 0)) { - *((npy_int64 *)op1) = in1/in2 - 1; + struct libdivide_s64_t fast_d = libdivide_s64_gen(in2); + BINARY_LOOP_SLIDING { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + if (in1 == NPY_DATETIME_NAT) { + npy_set_floatstatus_invalid(); + *((npy_int64 *)op1) = 0; + } + else { + *((npy_int64 *)op1) = libdivide_s64_do(in1, &fast_d); + + /* Negative quotients needs to be rounded down */ + if (((in1 > 0) != (in2 > 0)) && (*((npy_int64 *)op1) * in2 != in1)) { + *((npy_int64 *)op1) = *((npy_int64 *)op1) - 1; + } + } + } + } + } + else { + BINARY_LOOP_SLIDING { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + const npy_timedelta in2 = *(npy_timedelta *)ip2; + if (in1 == NPY_DATETIME_NAT || in2 == NPY_DATETIME_NAT) { + npy_set_floatstatus_invalid(); + *((npy_int64 *)op1) = 0; + } + else if (in2 == 0) { + npy_set_floatstatus_divbyzero(); + *((npy_int64 *)op1) = 0; } else { *((npy_int64 *)op1) = in1/in2; + + /* Negative quotients needs to be rounded down */ + if (((in1 > 0) != (in2 > 0)) && (*((npy_int64 *)op1) * in2 != in1)) { + *((npy_int64 *)op1) = *((npy_int64 *)op1) - 1; + } } } } @@ -1514,39 +1555,6 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 void /**end repeat1**/ /**end repeat**/ -/**begin repeat - * #func = sin, cos, exp, log# - * #scalarf = npy_sinf, npy_cosf, npy_expf, npy_logf# - */ - -NPY_NO_EXPORT NPY_GCC_OPT_3 void -FLOAT_@func@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - UNARY_LOOP { - const npy_float in1 = *(npy_float *)ip1; - *(npy_float *)op1 = @scalarf@(in1); - } -} - -/**end repeat**/ - -NPY_NO_EXPORT NPY_GCC_OPT_3 void -DOUBLE_exp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - UNARY_LOOP { - const npy_double in1 = *(npy_double *)ip1; - *(npy_double *)op1 = npy_exp(in1); - } -} -NPY_NO_EXPORT NPY_GCC_OPT_3 void -DOUBLE_log(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - UNARY_LOOP { - const npy_double in1 = *(npy_double *)ip1; - *(npy_double *)op1 = npy_log(in1); - } -} - /**begin repeat * #isa = avx512f, fma# * #ISA = AVX512F, FMA# @@ -1576,162 +1584,9 @@ NPY_NO_EXPORT NPY_GCC_OPT_3 void } /**end repeat2**/ -/**end repeat1**/ - -/**begin repeat1 - * #func = exp, log# - * #scalarf = npy_expf, npy_logf# - */ - -NPY_NO_EXPORT NPY_GCC_OPT_3 void -FLOAT_@func@_@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - if (!run_unary_@isa@_@func@_FLOAT(args, dimensions, steps)) { - UNARY_LOOP { - /* - * We use the AVX function to compute exp/log for scalar elements as well. - * This is needed to ensure the output of strided and non-strided - * cases match. SIMD code handles strided input cases, but not - * strided output. - */ -#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS - @ISA@_@func@_FLOAT((npy_float *)op1, (npy_float *)ip1, 1, steps[0]); -#else - const npy_float in1 = *(npy_float *)ip1; - *(npy_float *)op1 = @scalarf@(in1); -#endif - } - } -} - -/**end repeat1**/ - -/**begin repeat1 - * #func = cos, sin# - * #enum = npy_compute_cos, npy_compute_sin# - * #scalarf = npy_cosf, npy_sinf# - */ - -NPY_NO_EXPORT NPY_GCC_OPT_3 void -FLOAT_@func@_@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - if (!run_unary_@isa@_sincos_FLOAT(args, dimensions, steps, @enum@)) { - UNARY_LOOP { -#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS - @ISA@_sincos_FLOAT((npy_float *)op1, (npy_float *)ip1, 1, steps[0], @enum@); -#else - const npy_float in1 = *(npy_float *)ip1; - *(npy_float *)op1 = @scalarf@(in1); -#endif - } - } -} - /**end repeat1**/ /**end repeat**/ -NPY_NO_EXPORT NPY_GCC_OPT_3 void -DOUBLE_exp_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - if (!run_unary_avx512f_exp_DOUBLE(args, dimensions, steps)) { - UNARY_LOOP { - const npy_double in1 = *(npy_double *)ip1; - *(npy_double *)op1 = npy_exp(in1); - } - } -} - -NPY_NO_EXPORT NPY_GCC_OPT_3 void -DOUBLE_log_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) -{ - if (!run_unary_avx512f_log_DOUBLE(args, dimensions, steps)) { - UNARY_LOOP { - const npy_double in1 = *(npy_double *)ip1; - *(npy_double *)op1 = npy_log(in1); - } - } -} - -/**begin repeat - * Float types - * #type = npy_float, npy_double, npy_longdouble, npy_float# - * #dtype = npy_float, npy_double, npy_longdouble, npy_half# - * #TYPE = FLOAT, DOUBLE, LONGDOUBLE, HALF# - * #c = f, , l, # - * #C = F, , L, # - * #trf = , , , npy_half_to_float# - */ - -/* - * Pairwise summation, rounding error O(lg n) instead of O(n). - * The recursion depth is O(lg n) as well. - * when updating also update similar complex floats summation - */ -static @type@ -pairwise_sum_@TYPE@(char *a, npy_intp n, npy_intp stride) -{ - if (n < 8) { - npy_intp i; - @type@ res = 0.; - - for (i = 0; i < n; i++) { - res += @trf@(*((@dtype@*)(a + i * stride))); - } - return res; - } - else if (n <= PW_BLOCKSIZE) { - npy_intp i; - @type@ r[8], res; - - /* - * sum a block with 8 accumulators - * 8 times unroll reduces blocksize to 16 and allows vectorization with - * avx without changing summation ordering - */ - r[0] = @trf@(*((@dtype@ *)(a + 0 * stride))); - r[1] = @trf@(*((@dtype@ *)(a + 1 * stride))); - r[2] = @trf@(*((@dtype@ *)(a + 2 * stride))); - r[3] = @trf@(*((@dtype@ *)(a + 3 * stride))); - r[4] = @trf@(*((@dtype@ *)(a + 4 * stride))); - r[5] = @trf@(*((@dtype@ *)(a + 5 * stride))); - r[6] = @trf@(*((@dtype@ *)(a + 6 * stride))); - r[7] = @trf@(*((@dtype@ *)(a + 7 * stride))); - - for (i = 8; i < n - (n % 8); i += 8) { - /* small blocksizes seems to mess with hardware prefetch */ - NPY_PREFETCH(a + (i + 512/(npy_intp)sizeof(@dtype@))*stride, 0, 3); - r[0] += @trf@(*((@dtype@ *)(a + (i + 0) * stride))); - r[1] += @trf@(*((@dtype@ *)(a + (i + 1) * stride))); - r[2] += @trf@(*((@dtype@ *)(a + (i + 2) * stride))); - r[3] += @trf@(*((@dtype@ *)(a + (i + 3) * stride))); - r[4] += @trf@(*((@dtype@ *)(a + (i + 4) * stride))); - r[5] += @trf@(*((@dtype@ *)(a + (i + 5) * stride))); - r[6] += @trf@(*((@dtype@ *)(a + (i + 6) * stride))); - r[7] += @trf@(*((@dtype@ *)(a + (i + 7) * stride))); - } - - /* accumulate now to avoid stack spills for single peel loop */ - res = ((r[0] + r[1]) + (r[2] + r[3])) + - ((r[4] + r[5]) + (r[6] + r[7])); - - /* do non multiple of 8 rest */ - for (; i < n; i++) { - res += @trf@(*((@dtype@ *)(a + i * stride))); - } - return res; - } - else { - /* divide by two but avoid non-multiples of unroll factor */ - npy_intp n2 = n / 2; - - n2 -= n2 % 8; - return pairwise_sum_@TYPE@(a, n2, stride) + - pairwise_sum_@TYPE@(a + n2 * stride, n - n2, stride); - } -} - -/**end repeat**/ - /**begin repeat * Float types * #type = npy_float, npy_double, npy_longdouble# @@ -1739,39 +1594,6 @@ pairwise_sum_@TYPE@(char *a, npy_intp n, npy_intp stride) * #c = f, , l# * #C = F, , L# */ - -/**begin repeat1 - * Arithmetic - * # kind = add, subtract, multiply, divide# - * # OP = +, -, *, /# - * # PW = 1, 0, 0, 0# - */ -NPY_NO_EXPORT void -@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) -{ - if (IS_BINARY_REDUCE) { -#if @PW@ - @type@ * iop1 = (@type@ *)args[0]; - npy_intp n = dimensions[0]; - - *iop1 @OP@= pairwise_sum_@TYPE@(args[1], n, steps[1]); -#else - BINARY_REDUCE_LOOP(@type@) { - io1 @OP@= *(@type@ *)ip2; - } - *((@type@ *)iop1) = io1; -#endif - } - else if (!run_binary_simd_@kind@_@TYPE@(args, dimensions, steps)) { - BINARY_LOOP { - const @type@ in1 = *(@type@ *)ip1; - const @type@ in2 = *(@type@ *)ip2; - *((@type@ *)op1) = in1 @OP@ in2; - } - } -} -/**end repeat1**/ - /**begin repeat1 * #kind = equal, not_equal, less, less_equal, greater, greater_equal, * logical_and, logical_or# @@ -2036,41 +1858,6 @@ NPY_NO_EXPORT void } } -NPY_NO_EXPORT void -@TYPE@_frexp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) -{ - UNARY_LOOP_TWO_OUT { - const @type@ in1 = *(@type@ *)ip1; - *((@type@ *)op1) = npy_frexp@c@(in1, (int *)op2); - } -} - -NPY_NO_EXPORT void -@TYPE@_frexp_avx512_skx(char **args, npy_intp const *dimensions, npy_intp const *steps, void *func) -{ - if (!run_unary_two_out_avx512_skx_frexp_@TYPE@(args, dimensions, steps)) { - @TYPE@_frexp(args, dimensions, steps, func); - } -} - -NPY_NO_EXPORT void -@TYPE@_ldexp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) -{ - BINARY_LOOP { - const @type@ in1 = *(@type@ *)ip1; - const int in2 = *(int *)ip2; - *((@type@ *)op1) = npy_ldexp@c@(in1, in2); - } -} - -NPY_NO_EXPORT void -@TYPE@_ldexp_avx512_skx(char **args, const npy_intp *dimensions, const npy_intp *steps, void *func) -{ - if (!run_binary_avx512_skx_ldexp_@TYPE@(args, dimensions, steps)) { - @TYPE@_ldexp(args, dimensions, steps, func); - } -} - NPY_NO_EXPORT void @TYPE@_ldexp_long(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) { @@ -2101,8 +1888,6 @@ NPY_NO_EXPORT void } } -#define @TYPE@_true_divide @TYPE@_divide - /**end repeat**/ /* @@ -2111,6 +1896,38 @@ NPY_NO_EXPORT void ***************************************************************************** */ +/**begin repeat + * Arithmetic + * # kind = add, subtract, multiply, divide# + * # OP = +, -, *, /# + * # PW = 1, 0, 0, 0# + */ +NPY_NO_EXPORT void +LONGDOUBLE_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + if (IS_BINARY_REDUCE) { +#if @PW@ + npy_longdouble * iop1 = (npy_longdouble *)args[0]; + npy_intp n = dimensions[0]; + + *iop1 @OP@= LONGDOUBLE_pairwise_sum(args[1], n, steps[1]); +#else + BINARY_REDUCE_LOOP(npy_longdouble) { + io1 @OP@= *(npy_longdouble *)ip2; + } + *((npy_longdouble *)iop1) = io1; +#endif + } + else { + BINARY_LOOP { + const npy_longdouble in1 = *(npy_longdouble *)ip1; + const npy_longdouble in2 = *(npy_longdouble *)ip2; + *((npy_longdouble *)op1) = in1 @OP@ in2; + } + } +} +/**end repeat**/ + NPY_NO_EXPORT void LONGDOUBLE_reciprocal(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) { @@ -2141,6 +1958,25 @@ LONGDOUBLE_square(char **args, npy_intp const *dimensions, npy_intp const *steps } } +NPY_NO_EXPORT void +LONGDOUBLE_frexp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + UNARY_LOOP_TWO_OUT { + const npy_longdouble in1 = *(npy_longdouble *)ip1; + *((npy_longdouble *)op1) = npy_frexpl(in1, (int *)op2); + } +} + +NPY_NO_EXPORT void +LONGDOUBLE_ldexp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + BINARY_LOOP { + const npy_longdouble in1 = *(npy_longdouble *)ip1; + const int in2 = *(int *)ip2; + *((npy_longdouble *)op1) = npy_ldexpl(in1, in2); + } +} + /* ***************************************************************************** ** HALF-FLOAT LOOPS ** @@ -2163,7 +1999,7 @@ HALF_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void #if @PW@ npy_intp n = dimensions[0]; - io1 @OP@= pairwise_sum_HALF(args[1], n, steps[1]); + io1 @OP@= HALF_pairwise_sum(args[1], n, steps[1]); #else BINARY_REDUCE_LOOP_INNER { io1 @OP@= npy_half_to_float(*(npy_half *)ip2); @@ -2471,9 +2307,6 @@ HALF_ldexp_long(char **args, npy_intp const *dimensions, npy_intp const *steps, } } -#define HALF_true_divide HALF_divide - - /* ***************************************************************************** ** COMPLEX LOOPS ** @@ -2500,80 +2333,8 @@ HALF_ldexp_long(char **args, npy_intp const *dimensions, npy_intp const *steps, * #SIMD = 1, 1, 0# */ -/* similar to pairwise sum of real floats */ -static void -pairwise_sum_@TYPE@(@ftype@ *rr, @ftype@ * ri, char * a, npy_intp n, - npy_intp stride) -{ - assert(n % 2 == 0); - if (n < 8) { - npy_intp i; - - *rr = 0.; - *ri = 0.; - for (i = 0; i < n; i += 2) { - *rr += *((@ftype@ *)(a + i * stride + 0)); - *ri += *((@ftype@ *)(a + i * stride + sizeof(@ftype@))); - } - return; - } - else if (n <= PW_BLOCKSIZE) { - npy_intp i; - @ftype@ r[8]; - - /* - * sum a block with 8 accumulators - * 8 times unroll reduces blocksize to 16 and allows vectorization with - * avx without changing summation ordering - */ - r[0] = *((@ftype@ *)(a + 0 * stride)); - r[1] = *((@ftype@ *)(a + 0 * stride + sizeof(@ftype@))); - r[2] = *((@ftype@ *)(a + 2 * stride)); - r[3] = *((@ftype@ *)(a + 2 * stride + sizeof(@ftype@))); - r[4] = *((@ftype@ *)(a + 4 * stride)); - r[5] = *((@ftype@ *)(a + 4 * stride + sizeof(@ftype@))); - r[6] = *((@ftype@ *)(a + 6 * stride)); - r[7] = *((@ftype@ *)(a + 6 * stride + sizeof(@ftype@))); - - for (i = 8; i < n - (n % 8); i += 8) { - /* small blocksizes seems to mess with hardware prefetch */ - NPY_PREFETCH(a + (i + 512/(npy_intp)sizeof(@ftype@))*stride, 0, 3); - r[0] += *((@ftype@ *)(a + (i + 0) * stride)); - r[1] += *((@ftype@ *)(a + (i + 0) * stride + sizeof(@ftype@))); - r[2] += *((@ftype@ *)(a + (i + 2) * stride)); - r[3] += *((@ftype@ *)(a + (i + 2) * stride + sizeof(@ftype@))); - r[4] += *((@ftype@ *)(a + (i + 4) * stride)); - r[5] += *((@ftype@ *)(a + (i + 4) * stride + sizeof(@ftype@))); - r[6] += *((@ftype@ *)(a + (i + 6) * stride)); - r[7] += *((@ftype@ *)(a + (i + 6) * stride + sizeof(@ftype@))); - } - - /* accumulate now to avoid stack spills for single peel loop */ - *rr = ((r[0] + r[2]) + (r[4] + r[6])); - *ri = ((r[1] + r[3]) + (r[5] + r[7])); - - /* do non multiple of 8 rest */ - for (; i < n; i+=2) { - *rr += *((@ftype@ *)(a + i * stride + 0)); - *ri += *((@ftype@ *)(a + i * stride + sizeof(@ftype@))); - } - return; - } - else { - /* divide by two but avoid non-multiples of unroll factor */ - @ftype@ rr1, ri1, rr2, ri2; - npy_intp n2 = n / 2; - - n2 -= n2 % 8; - pairwise_sum_@TYPE@(&rr1, &ri1, a, n2, stride); - pairwise_sum_@TYPE@(&rr2, &ri2, a + n2 * stride, n - n2, stride); - *rr = rr1 + rr2; - *ri = ri1 + ri2; - return; - } -} - - +#if !@SIMD@ +// CFLOAT & CDOUBLE defined by 'loops_arithm_fp.dispatch.c.src' /**begin repeat1 * arithmetic * #kind = add, subtract# @@ -2590,7 +2351,7 @@ NPY_NO_EXPORT void @ftype@ * oi = ((@ftype@ *)args[0]) + 1; @ftype@ rr, ri; - pairwise_sum_@TYPE@(&rr, &ri, args[1], n * 2, steps[1] / 2); + @TYPE@_pairwise_sum(&rr, &ri, args[1], n * 2, steps[1] / 2); *or @OP@= rr; *oi @OP@= ri; return; @@ -2620,6 +2381,7 @@ NPY_NO_EXPORT void ((@ftype@ *)op1)[1] = in1r*in2i + in1i*in2r; } } +#endif // !SIMD NPY_NO_EXPORT void @TYPE@_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) @@ -2653,31 +2415,6 @@ NPY_NO_EXPORT void } } -#if @SIMD@ -NPY_NO_EXPORT void -@TYPE@_add_avx512f(char **args, const npy_intp *dimensions, const npy_intp *steps, void *func) -{ - if (IS_BINARY_REDUCE) { - @TYPE@_add(args, dimensions, steps, func); - } - else if (!run_binary_avx512f_add_@TYPE@(args, dimensions, steps)) { - @TYPE@_add(args, dimensions, steps, func); - } -} - -/**begin repeat1 - * arithmetic - * #kind = subtract, multiply# - */ -NPY_NO_EXPORT void -@TYPE@_@kind@_avx512f(char **args, const npy_intp *dimensions, const npy_intp *steps, void *func) -{ - if (!run_binary_avx512f_@kind@_@TYPE@(args, dimensions, steps)) { - @TYPE@_@kind@(args, dimensions, steps, func); - } -} -/**end repeat1**/ -#endif NPY_NO_EXPORT void @TYPE@_floor_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) @@ -2687,6 +2424,21 @@ NPY_NO_EXPORT void const @ftype@ in1i = ((@ftype@ *)ip1)[1]; const @ftype@ in2r = ((@ftype@ *)ip2)[0]; const @ftype@ in2i = ((@ftype@ *)ip2)[1]; +#if defined(__APPLE__) && defined(__aarch64__) + // On macos-arm64 without this block of code, + // when branch prediction goes wrong, the floating point exception + // register does not get cleared and an exception for the + // wrong branch is thrown. + if (in2i == 0) { + ((@ftype@ *)op1)[0] = npy_floor@c@(in1r/in2r); + ((@ftype@ *)op1)[1] = 0; + } + else if (in2r == 0) { + ((@ftype@ *)op1)[0] = npy_floor@c@(in1i/in2i); + ((@ftype@ *)op1)[1] = 0; + } + else +#endif if (npy_fabs@c@(in2r) >= npy_fabs@c@(in2i)) { const @ftype@ rat = in2i/in2r; ((@ftype@ *)op1)[0] = npy_floor@c@((in1r + in1i*rat)/(in2r + in2i*rat)); @@ -2923,8 +2675,6 @@ NPY_NO_EXPORT void } /**end repeat1**/ -#define @TYPE@_true_divide @TYPE@_divide - /**end repeat**/ #undef CGE diff --git a/numpy/core/src/umath/loops.h.src b/numpy/core/src/umath/loops.h.src index a0b68d168b19..bb07e047c372 100644 --- a/numpy/core/src/umath/loops.h.src +++ b/numpy/core/src/umath/loops.h.src @@ -53,6 +53,18 @@ BOOL_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void ***************************************************************************** */ +#ifndef NPY_DISABLE_OPTIMIZATION + #include "loops_arithmetic.dispatch.h" +#endif + +/**begin repeat + * #TYPE = UBYTE, USHORT, UINT, ULONG, ULONGLONG, + BYTE, SHORT, INT, LONG, LONGLONG# + */ + NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_divide, + (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func))) +/**end repeat**/ + /**begin repeat * #TYPE = BYTE, SHORT, INT, LONG, LONGLONG# */ @@ -140,9 +152,6 @@ NPY_NO_EXPORT void NPY_NO_EXPORT void @S@@TYPE@_sign(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); -NPY_NO_EXPORT void -@S@@TYPE@_divide(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - NPY_NO_EXPORT void @S@@TYPE@_remainder(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); @@ -185,6 +194,22 @@ NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_@kind@, /**end repeat1**/ /**end repeat**/ +#ifndef NPY_DISABLE_OPTIMIZATION + #include "loops_arithm_fp.dispatch.h" +#endif +/**begin repeat + * #TYPE = FLOAT, DOUBLE# + */ +/**begin repeat1 + * Arithmetic + * # kind = add, subtract, multiply, divide# + * # OP = +, -, *, /# + */ +NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_@kind@, + (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func))) +/**end repeat1**/ +/**end repeat**/ + /**begin repeat * #TYPE = FLOAT, DOUBLE# */ @@ -197,31 +222,29 @@ NPY_NO_EXPORT void /**end repeat1**/ /**end repeat**/ -NPY_NO_EXPORT void -DOUBLE_exp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - -NPY_NO_EXPORT void -DOUBLE_exp_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - -NPY_NO_EXPORT void -DOUBLE_log(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - -NPY_NO_EXPORT void -DOUBLE_log_avx512f(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - +#ifndef NPY_DISABLE_OPTIMIZATION + #include "loops_trigonometric.dispatch.h" +#endif /**begin repeat - * #func = sin, cos, exp, log# + * #func = sin, cos# */ -NPY_NO_EXPORT void -FLOAT_@func@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void FLOAT_@func@, ( + char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func) +)) +/**end repeat**/ +#ifndef NPY_DISABLE_OPTIMIZATION + #include "loops_exponent_log.dispatch.h" +#endif +/**begin repeat + * #TYPE = FLOAT, DOUBLE# + */ /**begin repeat1 - * #isa = avx512f, fma# + * # kind = exp, log, frexp, ldexp# */ - -NPY_NO_EXPORT void -FLOAT_@func@_@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - +NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_@kind@, ( + char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func) +)) /**end repeat1**/ /**end repeat**/ @@ -345,20 +368,11 @@ NPY_NO_EXPORT void NPY_NO_EXPORT void @TYPE@_frexp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); -NPY_NO_EXPORT void -@TYPE@_frexp_avx512_skx(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - NPY_NO_EXPORT void @TYPE@_ldexp(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); -NPY_NO_EXPORT void -@TYPE@_ldexp_avx512_skx(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - NPY_NO_EXPORT void @TYPE@_ldexp_long(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - -#define @TYPE@_true_divide @TYPE@_divide - /**end repeat**/ @@ -367,6 +381,19 @@ NPY_NO_EXPORT void ** COMPLEX LOOPS ** ***************************************************************************** */ +#ifndef NPY_DISABLE_OPTIMIZATION + #include "loops_arithm_fp.dispatch.h" +#endif +/**begin repeat + * #TYPE = CFLOAT, CDOUBLE# + */ +/**begin repeat1 + * #kind = add, subtract, multiply# + */ +NPY_CPU_DISPATCH_DECLARE(NPY_NO_EXPORT void @TYPE@_@kind@, + (char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data))) +/**end repeat1**/ +/**end repeat**/ #define CGE(xr,xi,yr,yi) (xr > yr || (xr == yr && xi >= yi)); #define CLE(xr,xi,yr,yi) (xr < yr || (xr == yr && xi <= yi)); @@ -380,26 +407,14 @@ NPY_NO_EXPORT void * #TYPE = FLOAT, DOUBLE, LONGDOUBLE# * #c = f, , l# * #C = F, , L# - * #IFSIMD = 1, 1, 0# */ /**begin repeat1 - * #isa = , _avx512f# - */ - -/**begin repeat2 * arithmetic - * #kind = add, subtract# - * #OP = +, -# + * #kind = add, subtract, multiply# */ - -NPY_NO_EXPORT void -C@TYPE@_@kind@@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); - -/**end repeat2**/ - NPY_NO_EXPORT void -C@TYPE@_multiply@isa@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); +C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); /**end repeat1**/ NPY_NO_EXPORT void @@ -480,7 +495,6 @@ C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, v NPY_NO_EXPORT void C@TYPE@_@kind@(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); /**end repeat1**/ -#define C@TYPE@_true_divide C@TYPE@_divide /**end repeat**/ @@ -593,10 +607,6 @@ NPY_NO_EXPORT void TIMEDELTA_mm_qm_divmod(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)); /* Special case equivalents to above functions */ - -#define TIMEDELTA_mq_m_true_divide TIMEDELTA_mq_m_divide -#define TIMEDELTA_md_m_true_divide TIMEDELTA_md_m_divide -#define TIMEDELTA_mm_d_true_divide TIMEDELTA_mm_d_divide #define TIMEDELTA_mq_m_floor_divide TIMEDELTA_mq_m_divide #define TIMEDELTA_md_m_floor_divide TIMEDELTA_md_m_divide /* #define TIMEDELTA_mm_d_floor_divide TIMEDELTA_mm_d_divide */ diff --git a/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src b/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src new file mode 100644 index 000000000000..d8c8fdc9e41e --- /dev/null +++ b/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src @@ -0,0 +1,777 @@ +/*@targets + ** $maxopt baseline + ** sse2 avx2 avx512f + **/ +#define _UMATHMODULE +#define _MULTIARRAYMODULE +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include "simd/simd.h" +#include "loops_utils.h" +#include "loops.h" +#include "lowlevel_strided_loops.h" +// Provides the various *_LOOP macros +#include "fast_loop_macros.h" + +// TODO: replace raw SIMD with NPYV +//############################################################################### +//## Real Single/Double precision +//############################################################################### +/******************************************************************************** + ** Defining the SIMD kernels + ********************************************************************************/ +#ifdef NPY_HAVE_SSE2 +/**begin repeat + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + * #scalarf = npy_sqrtf, npy_sqrt# + * #c = f, # + * #vtype = __m128, __m128d# + * #vtype256 = __m256, __m256d# + * #vtype512 = __m512, __m512d# + * #vpre = _mm, _mm# + * #vpre256 = _mm256, _mm256# + * #vpre512 = _mm512, _mm512# + * #vsuf = ps, pd# + * #vsufs = ss, sd# + * #nan = NPY_NANF, NPY_NAN# + * #double = 0, 1# + * #cast = _mm_castps_si128, _mm_castpd_si128# + */ +/**begin repeat1 +* Arithmetic +* # kind = add, subtract, multiply, divide# +* # OP = +, -, *, /# +* # VOP = add, sub, mul, div# +*/ +static void +sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) +{ +#ifdef NPY_HAVE_AVX512F + const npy_intp vector_size_bytes = 64; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[i] @OP@ ip2[i]; + /* lots of specializations, to squeeze out max performance */ + if (npy_is_aligned(&ip1[i], vector_size_bytes) && npy_is_aligned(&ip2[i], vector_size_bytes)) { + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); + @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + } + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); + @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); + @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + else { + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); + @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + } +#elif defined NPY_HAVE_AVX2 + const npy_intp vector_size_bytes = 32; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[i] @OP@ ip2[i]; + /* lots of specializations, to squeeze out max performance */ + if (npy_is_aligned(&ip1[i], vector_size_bytes) && + npy_is_aligned(&ip2[i], vector_size_bytes)) { + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); + @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + } + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); + @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); + @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + else { + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); + @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + } +#else + const npy_intp vector_size_bytes = 16; + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[i] @OP@ ip2[i]; + /* lots of specializations, to squeeze out max performance */ + if (npy_is_aligned(&ip1[i], vector_size_bytes) && + npy_is_aligned(&ip2[i], vector_size_bytes)) { + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); + @vpre@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); + @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } + } + else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); + @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } + else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); + @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } + else { + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); + @vpre@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); + @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } + } +#endif + LOOP_BLOCKED_END { + op[i] = ip1[i] @OP@ ip2[i]; + } +} + +static void +sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) +{ +#ifdef NPY_HAVE_AVX512F + const npy_intp vector_size_bytes = 64; + const @vtype512@ a = @vpre512@_set1_@vsuf@(ip1[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[0] @OP@ ip2[i]; + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + + +#elif defined NPY_HAVE_AVX2 + const npy_intp vector_size_bytes = 32; + const @vtype256@ a = @vpre256@_set1_@vsuf@(ip1[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[0] @OP@ ip2[i]; + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } +#else + const npy_intp vector_size_bytes = 16; + const @vtype@ a = @vpre@_set1_@vsuf@(ip1[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[0] @OP@ ip2[i]; + if (npy_is_aligned(&ip2[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } +#endif + LOOP_BLOCKED_END { + op[i] = ip1[0] @OP@ ip2[i]; + } +} + +static void +sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) +{ +#ifdef NPY_HAVE_AVX512F + const npy_intp vector_size_bytes = 64; + const @vtype512@ b = @vpre512@_set1_@vsuf@(ip2[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[i] @OP@ ip2[0]; + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); + @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); + @vpre512@_store_@vsuf@(&op[i], c); + } + } + +#elif defined NPY_HAVE_AVX2 + const npy_intp vector_size_bytes = 32; + const @vtype256@ b = @vpre256@_set1_@vsuf@(ip2[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[i] @OP@ ip2[0]; + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); + @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); + @vpre256@_store_@vsuf@(&op[i], c); + } + } +#else + const npy_intp vector_size_bytes = 16; + const @vtype@ b = @vpre@_set1_@vsuf@(ip2[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) + op[i] = ip1[i] @OP@ ip2[0]; + if (npy_is_aligned(&ip1[i], vector_size_bytes)) { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, vector_size_bytes) { + @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); + @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); + @vpre@_store_@vsuf@(&op[i], c); + } + } +#endif + LOOP_BLOCKED_END { + op[i] = ip1[i] @OP@ ip2[0]; + } +} + +/**end repeat1**/ +/**end repeat**/ + +#else // NPY_HAVE_SSE2 + +/**begin repeat + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + * #sfx = f32, f64# + * #CHK = , _F64# + */ +#if NPY_SIMD@CHK@ +/**begin repeat1 +* Arithmetic +* # kind = add, subtract, multiply, divide# +* # OP = +, -, *, /# +* # VOP = add, sub, mul, div# +*/ + +static void +simd_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) +{ + LOOP_BLOCK_ALIGN_VAR(op, @type@, NPY_SIMD_WIDTH) { + op[i] = ip1[i] @OP@ ip2[i]; + } + /* lots of specializations, to squeeze out max performance */ + if (ip1 == ip2) { + LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { + npyv_@sfx@ a = npyv_load_@sfx@(&ip1[i]); + npyv_@sfx@ c = npyv_@VOP@_@sfx@(a, a); + npyv_store_@sfx@(&op[i], c); + } + } + else { + LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { + npyv_@sfx@ a = npyv_load_@sfx@(&ip1[i]); + npyv_@sfx@ b = npyv_load_@sfx@(&ip2[i]); + npyv_@sfx@ c = npyv_@VOP@_@sfx@(a, b); + npyv_store_@sfx@(&op[i], c); + } + } + LOOP_BLOCKED_END { + op[i] = ip1[i] @OP@ ip2[i]; + } +} + +static void +simd_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) +{ + const npyv_@sfx@ v1 = npyv_setall_@sfx@(ip1[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, NPY_SIMD_WIDTH) { + op[i] = ip1[0] @OP@ ip2[i]; + } + LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { + npyv_@sfx@ v2 = npyv_load_@sfx@(&ip2[i]); + npyv_@sfx@ v3 = npyv_@VOP@_@sfx@(v1, v2); + npyv_store_@sfx@(&op[i], v3); + } + LOOP_BLOCKED_END { + op[i] = ip1[0] @OP@ ip2[i]; + } +} + +static void +simd_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) +{ + const npyv_@sfx@ v2 = npyv_setall_@sfx@(ip2[0]); + LOOP_BLOCK_ALIGN_VAR(op, @type@, NPY_SIMD_WIDTH) { + op[i] = ip1[i] @OP@ ip2[0]; + } + LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { + npyv_@sfx@ v1 = npyv_load_@sfx@(&ip1[i]); + npyv_@sfx@ v3 = npyv_@VOP@_@sfx@(v1, v2); + npyv_store_@sfx@(&op[i], v3); + } + LOOP_BLOCKED_END { + op[i] = ip1[i] @OP@ ip2[0]; + } +} +/**end repeat1**/ +#endif /* NPY_SIMD@CHK@ */ +/**end repeat**/ +#endif // NPY_HAVE_SSE2 + +/**begin repeat + * Float types + * #type = npy_float, npy_double, npy_longdouble# + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE# + * #vector = 1, 1, 0# + * #VECTOR = NPY_SIMD, NPY_SIMD_F64, 0 # + */ +/**begin repeat1 + * Arithmetic + * # kind = add, subtract, multiply, divide# + */ +static NPY_INLINE int +run_binary_simd_@kind@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) +{ +#if @vector@ && defined NPY_HAVE_SSE2 + @type@ * ip1 = (@type@ *)args[0]; + @type@ * ip2 = (@type@ *)args[1]; + @type@ * op = (@type@ *)args[2]; + npy_intp n = dimensions[0]; +#if defined NPY_HAVE_AVX512F + const npy_uintp vector_size_bytes = 64; +#elif defined NPY_HAVE_AVX2 + const npy_uintp vector_size_bytes = 32; +#else + const npy_uintp vector_size_bytes = 32; +#endif + /* argument one scalar */ + if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), vector_size_bytes)) { + sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); + return 1; + } + /* argument two scalar */ + else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), vector_size_bytes)) { + sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); + return 1; + } + else if (IS_BLOCKABLE_BINARY(sizeof(@type@), vector_size_bytes)) { + sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n); + return 1; + } +#elif @VECTOR@ + @type@ * ip1 = (@type@ *)args[0]; + @type@ * ip2 = (@type@ *)args[1]; + @type@ * op = (@type@ *)args[2]; + npy_intp n = dimensions[0]; + /* argument one scalar */ + if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), NPY_SIMD_WIDTH)) { + simd_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); + return 1; + } + /* argument two scalar */ + else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), NPY_SIMD_WIDTH)) { + simd_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); + return 1; + } + else if (IS_BLOCKABLE_BINARY(sizeof(@type@), NPY_SIMD_WIDTH)) { + simd_binary_@kind@_@TYPE@(op, ip1, ip2, n); + return 1; + } +#endif + return 0; +} +/**end repeat1**/ +/**end repeat**/ + +/******************************************************************************** + ** Defining ufunc inner functions + ********************************************************************************/ +/**begin repeat + * Float types + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + * #c = f, # + * #C = F, # + */ +/**begin repeat1 + * Arithmetic + * # kind = add, subtract, multiply, divide# + * # OP = +, -, *, /# + * # PW = 1, 0, 0, 0# + */ +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + if (IS_BINARY_REDUCE) { +#if @PW@ + @type@ * iop1 = (@type@ *)args[0]; + npy_intp n = dimensions[0]; + + *iop1 @OP@= @TYPE@_pairwise_sum(args[1], n, steps[1]); +#else + BINARY_REDUCE_LOOP(@type@) { + io1 @OP@= *(@type@ *)ip2; + } + *((@type@ *)iop1) = io1; +#endif + } + else if (!run_binary_simd_@kind@_@TYPE@(args, dimensions, steps)) { + BINARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + const @type@ in2 = *(@type@ *)ip2; + *((@type@ *)op1) = in1 @OP@ in2; + } + } +} +/**end repeat1**/ +/**end repeat**/ + +//############################################################################### +//## Complex Single/Double precision +//############################################################################### +/******************************************************************************** + ** Defining the SIMD kernels + ********************************************************************************/ +#if !defined(_MSC_VER) && defined(NPY_HAVE_AVX512F) + /** + * For somehow MSVC commit aggressive optimization lead + * to raises 'RuntimeWarning: invalid value encountered in multiply' + * + * the issue mainly caused by '_mm512_maskz_loadu_ps', we need to + * investigate about it while moving to NPYV. + */ + #define AVX512F_NOMSVC +#endif + +#ifdef AVX512F_NOMSVC +static NPY_INLINE __mmask16 +avx512_get_full_load_mask_ps(void) +{ + return 0xFFFF; +} + +static NPY_INLINE __mmask8 +avx512_get_full_load_mask_pd(void) +{ + return 0xFF; +} +static NPY_INLINE __m512 +avx512_masked_load_ps(__mmask16 mask, npy_float* addr) +{ + return _mm512_maskz_loadu_ps(mask, (__m512 *)addr); +} + +static NPY_INLINE __m512d +avx512_masked_load_pd(__mmask8 mask, npy_double* addr) +{ + return _mm512_maskz_loadu_pd(mask, (__m512d *)addr); +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 +avx512_get_partial_load_mask_ps(const npy_int num_elem, const npy_int total_elem) +{ + return (0x0001 << num_elem) - 0x0001; +} + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask8 +avx512_get_partial_load_mask_pd(const npy_int num_elem, const npy_int total_elem) +{ + return (0x01 << num_elem) - 0x01; +} +/**begin repeat + * #vsub = ps, pd# + * #type= npy_float, npy_double# + * #epi_vsub = epi32, epi64# + * #vtype = __m512, __m512d# + * #mask = __mmask16, __mmask8# + * #and_const = 0x7fffffff, 0x7fffffffffffffffLL# + * #neg_mask = 0x80000000, 0x8000000000000000# + * #perm_ = 0xb1, 0x55# + * #cmpx_img_mask = 0xAAAA, 0xAA# + * #cmpx_re_mask = 0x5555, 0x55# + * #INF = NPY_INFINITYF, NPY_INFINITY# + * #NAN = NPY_NANF, NPY_NAN# + */ +static @vtype@ +avx512_hadd_@vsub@(const @vtype@ x) +{ + return _mm512_add_@vsub@(x, _mm512_permute_@vsub@(x, @perm_@)); +} + +static @vtype@ +avx512_hsub_@vsub@(const @vtype@ x) +{ + return _mm512_sub_@vsub@(x, _mm512_permute_@vsub@(x, @perm_@)); +} +static NPY_INLINE @vtype@ +avx512_cmul_@vsub@(@vtype@ x1, @vtype@ x2) +{ + // x1 = r1, i1 + // x2 = r2, i2 + @vtype@ x3 = _mm512_permute_@vsub@(x2, @perm_@); // i2, r2 + @vtype@ x12 = _mm512_mul_@vsub@(x1, x2); // r1*r2, i1*i2 + @vtype@ x13 = _mm512_mul_@vsub@(x1, x3); // r1*i2, r2*i1 + @vtype@ outreal = avx512_hsub_@vsub@(x12); // r1*r2 - i1*i2, r1*r2 - i1*i2 + @vtype@ outimg = avx512_hadd_@vsub@(x13); // r1*i2 + i1*r2, r1*i2 + i1*r2 + return _mm512_mask_blend_@vsub@(@cmpx_img_mask@, outreal, outimg); +} +/**end repeat**/ +#endif + +/**begin repeat + * #TYPE = CFLOAT, CDOUBLE# + * #type = npy_float, npy_double# + * #num_lanes = 16, 8# + * #vsuffix = ps, pd# + * #epi_vsub = epi32, epi64# + * #mask = __mmask16, __mmask8# + * #vtype = __m512, __m512d# + * #scale = 4, 8# + * #vindextype = __m512i, __m256i# + * #vindexload = _mm512_loadu_si512, _mm256_loadu_si256# + * #storemask = 0xFF, 0xF# + * #IS_FLOAT = 1, 0# + */ +/**begin repeat1 + * #func = add, subtract, multiply# + * #vectorf = _mm512_add, _mm512_sub, avx512_cmul# + */ +#if defined AVX512F_NOMSVC +static NPY_INLINE void +AVX512F_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) +{ + const npy_intp array_size = dimensions[0]; + npy_intp num_remaining_elements = 2*array_size; + @type@* ip1 = (@type@*) args[0]; + @type@* ip2 = (@type@*) args[1]; + @type@* op = (@type@*) args[2]; + + @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + } + @vtype@ x1, x2; + x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); + x2 = avx512_masked_load_@vsuffix@(load_mask, ip2); + + @vtype@ out = @vectorf@_@vsuffix@(x1, x2); + + _mm512_mask_storeu_@vsuffix@(op, load_mask, out); + + ip1 += @num_lanes@; + ip2 += @num_lanes@; + op += @num_lanes@; + num_remaining_elements -= @num_lanes@; + } +} +#endif // AVX512F_NOMSVC +/**end repeat1**/ +/**end repeat**/ + +/**begin repeat + * #TYPE = CFLOAT, CDOUBLE# + * #type= npy_float, npy_double# + * #esize = 8, 16# + */ +/**begin repeat1 + * #func = add, subtract, multiply# + */ +static NPY_INLINE int +run_binary_avx512f_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) +{ +#if defined AVX512F_NOMSVC + if (IS_BINARY_STRIDE_ONE(@esize@, 64)) { + AVX512F_@func@_@TYPE@(args, dimensions, steps); + return 1; + } + else + return 0; +#endif + return 0; +} +/**end repeat1**/ +/**end repeat**/ + +/******************************************************************************** + ** Defining ufunc inner functions + ********************************************************************************/ +/**begin repeat + * complex types + * #TYPE = CFLOAT, CDOUBLE# + * #ftype = npy_float, npy_double# + * #c = f, # + * #C = F, # + */ +/**begin repeat1 + * arithmetic + * #kind = add, subtract# + * #OP = +, -# + * #PW = 1, 0# + */ +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + // Parenthesis around @PW@ tells clang dead code is intentional + if (IS_BINARY_REDUCE && (@PW@)) { + npy_intp n = dimensions[0]; + @ftype@ * or = ((@ftype@ *)args[0]); + @ftype@ * oi = ((@ftype@ *)args[0]) + 1; + @ftype@ rr, ri; + + @TYPE@_pairwise_sum(&rr, &ri, args[1], n * 2, steps[1] / 2); + *or @OP@= rr; + *oi @OP@= ri; + return; + } + if (!run_binary_avx512f_@kind@_@TYPE@(args, dimensions, steps)) { + BINARY_LOOP { + const @ftype@ in1r = ((@ftype@ *)ip1)[0]; + const @ftype@ in1i = ((@ftype@ *)ip1)[1]; + const @ftype@ in2r = ((@ftype@ *)ip2)[0]; + const @ftype@ in2i = ((@ftype@ *)ip2)[1]; + ((@ftype@ *)op1)[0] = in1r @OP@ in2r; + ((@ftype@ *)op1)[1] = in1i @OP@ in2i; + } + } +} +/**end repeat1**/ + +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_multiply) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + if (!run_binary_avx512f_multiply_@TYPE@(args, dimensions, steps)) { + BINARY_LOOP { + const @ftype@ in1r = ((@ftype@ *)ip1)[0]; + const @ftype@ in1i = ((@ftype@ *)ip1)[1]; + const @ftype@ in2r = ((@ftype@ *)ip2)[0]; + const @ftype@ in2i = ((@ftype@ *)ip2)[1]; + ((@ftype@ *)op1)[0] = in1r*in2r - in1i*in2i; + ((@ftype@ *)op1)[1] = in1r*in2i + in1i*in2r; + } + } +} +/**end repeat**/ diff --git a/numpy/core/src/umath/loops_arithmetic.dispatch.c.src b/numpy/core/src/umath/loops_arithmetic.dispatch.c.src new file mode 100644 index 000000000000..19e05f2b57b0 --- /dev/null +++ b/numpy/core/src/umath/loops_arithmetic.dispatch.c.src @@ -0,0 +1,262 @@ +/*@targets + ** $maxopt baseline + ** sse2 sse41 avx2 avx512f avx512_skx + ** vsx2 + ** neon + **/ +#define _UMATHMODULE +#define _MULTIARRAYMODULE +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include "simd/simd.h" +#include "loops_utils.h" +#include "loops.h" +#include "lowlevel_strided_loops.h" +// Provides the various *_LOOP macros +#include "fast_loop_macros.h" + +//############################################################################### +//## Division +//############################################################################### +/******************************************************************************** + ** Defining the SIMD kernels + * + * Floor division of signed is based on T. Granlund and P. L. Montgomery + * “Division by invariant integers using multiplication(see [Figure 6.1] + * http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.1.2556)" + * For details on TRUNC division see simd/intdiv.h for more clarification + *********************************************************************************** + ** Figure 6.1: Signed division by run–time invariant divisor, rounded towards -INF + *********************************************************************************** + * For q = FLOOR(a/d), all sword: + * sword −dsign = SRL(d, N − 1); + * uword −nsign = (n < −dsign); + * uword −qsign = EOR(−nsign, −dsign); + * q = TRUNC((n − (−dsign ) + (−nsign))/d) − (−qsign); + ********************************************************************************/ + +#if NPY_SIMD +/**begin repeat + * Signed types + * #sfx = s8, s16, s32, s64# + * #len = 8, 16, 32, 64# + */ +static NPY_INLINE void +simd_divide_by_scalar_contig_@sfx@(char **args, npy_intp len) +{ + npyv_lanetype_@sfx@ *src = (npyv_lanetype_@sfx@ *) args[0]; + npyv_lanetype_@sfx@ scalar = *(npyv_lanetype_@sfx@ *) args[1]; + npyv_lanetype_@sfx@ *dst = (npyv_lanetype_@sfx@ *) args[2]; + const int vstep = npyv_nlanes_@sfx@; + const npyv_@sfx@x3 divisor = npyv_divisor_@sfx@(scalar); + + if (scalar == -1) { + npyv_b@len@ noverflow = npyv_cvt_b@len@_@sfx@(npyv_setall_@sfx@(-1)); + npyv_@sfx@ vzero = npyv_zero_@sfx@(); + for (; len >= vstep; len -= vstep, src += vstep, dst += vstep) { + npyv_@sfx@ a = npyv_load_@sfx@(src); + npyv_b@len@ gt_min = npyv_cmpgt_@sfx@(a, npyv_setall_@sfx@(NPY_MIN_INT@len@)); + noverflow = npyv_and_b@len@(noverflow, gt_min); + npyv_@sfx@ neg = npyv_ifsub_@sfx@(gt_min, vzero, a, vzero); + npyv_store_@sfx@(dst, neg); + } + + int raise_err = npyv_tobits_b@len@(npyv_not_b@len@(noverflow)) != 0; + for (; len > 0; --len, ++src, ++dst) { + npyv_lanetype_@sfx@ a = *src; + if (a == NPY_MIN_INT@len@) { + raise_err = 1; + *dst = 0; + } else { + *dst = -a; + } + } + if (raise_err) { + npy_set_floatstatus_divbyzero(); + } + } else { + for (; len >= vstep; len -= vstep, src += vstep, dst += vstep) { + npyv_@sfx@ nsign_d = npyv_setall_@sfx@(scalar < 0); + npyv_@sfx@ a = npyv_load_@sfx@(src); + npyv_@sfx@ nsign_a = npyv_cvt_@sfx@_b@len@(npyv_cmplt_@sfx@(a, nsign_d)); + nsign_a = npyv_and_@sfx@(nsign_a, npyv_setall_@sfx@(1)); + npyv_@sfx@ diff_sign = npyv_sub_@sfx@(nsign_a, nsign_d); + npyv_@sfx@ to_ninf = npyv_xor_@sfx@(nsign_a, nsign_d); + npyv_@sfx@ trunc = npyv_divc_@sfx@(npyv_add_@sfx@(a, diff_sign), divisor); + npyv_@sfx@ floor = npyv_sub_@sfx@(trunc, to_ninf); + npyv_store_@sfx@(dst, floor); + } + + for (; len > 0; --len, ++src, ++dst) { + const npyv_lanetype_@sfx@ a = *src; + npyv_lanetype_@sfx@ r = a / scalar; + // Negative quotients needs to be rounded down + if (((a > 0) != (scalar > 0)) && ((r * scalar) != a)) { + r--; + } + *dst = r; + } + } + npyv_cleanup(); +} +/**end repeat**/ + +/**begin repeat + * Unsigned types + * #sfx = u8, u16, u32, u64# + * #len = 8, 16, 32, 64# + */ +static NPY_INLINE void +simd_divide_by_scalar_contig_@sfx@(char **args, npy_intp len) +{ + npyv_lanetype_@sfx@ *src = (npyv_lanetype_@sfx@ *) args[0]; + npyv_lanetype_@sfx@ scalar = *(npyv_lanetype_@sfx@ *) args[1]; + npyv_lanetype_@sfx@ *dst = (npyv_lanetype_@sfx@ *) args[2]; + const int vstep = npyv_nlanes_@sfx@; + const npyv_@sfx@x3 divisor = npyv_divisor_@sfx@(scalar); + + for (; len >= vstep; len -= vstep, src += vstep, dst += vstep) { + npyv_@sfx@ a = npyv_load_@sfx@(src); + npyv_@sfx@ c = npyv_divc_@sfx@(a, divisor); + npyv_store_@sfx@(dst, c); + } + + for (; len > 0; --len, ++src, ++dst) { + const npyv_lanetype_@sfx@ a = *src; + *dst = a / scalar; + } + npyv_cleanup(); +} +/**end repeat**/ +#endif + +/******************************************************************************** + ** Defining ufunc inner functions + ********************************************************************************/ + +/**begin repeat + * Signed types + * #type = npy_byte, npy_short, npy_int, npy_long, npy_longlong# + * #TYPE = BYTE, SHORT, INT, LONG, LONGLONG# + */ +#undef TO_SIMD_SFX +#if 0 +/**begin repeat1 + * #len = 8, 16, 32, 64# + */ +#elif NPY_BITSOF_@TYPE@ == @len@ + #define TO_SIMD_SFX(X) X##_s@len@ +/**end repeat1**/ +#endif + +#if NPY_BITSOF_@TYPE@ == 64 && !defined(NPY_HAVE_VSX4) && (defined(NPY_HAVE_VSX) || defined(NPY_HAVE_NEON)) + #undef TO_SIMD_SFX +#endif + +NPY_FINLINE @type@ floor_div_@TYPE@(const @type@ n, const @type@ d) +{ + /* + * FIXME: On x86 at least, dividing the smallest representable integer + * by -1 causes a SIFGPE (division overflow). We treat this case here + * (to avoid a SIGFPE crash at python level), but a good solution would + * be to treat integer division problems separately from FPU exceptions + * (i.e. a different approach than npy_set_floatstatus_divbyzero()). + */ + if (NPY_UNLIKELY(d == 0 || (n == NPY_MIN_@TYPE@ && d == -1))) { + npy_set_floatstatus_divbyzero(); + return 0; + } + @type@ r = n / d; + // Negative quotients needs to be rounded down + if (((n > 0) != (d > 0)) && ((r * d) != n)) { + r--; + } + return r; +} + +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_divide) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + if (IS_BINARY_REDUCE) { + BINARY_REDUCE_LOOP(@type@) { + io1 = floor_div_@TYPE@(io1, *(@type@*)ip2); + } + *((@type@ *)iop1) = io1; + } +#if NPY_SIMD && defined(TO_SIMD_SFX) + // for contiguous block of memory, divisor is a scalar and not 0 + else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), NPY_SIMD_WIDTH) && + (*(@type@ *)args[1]) != 0) { + TO_SIMD_SFX(simd_divide_by_scalar_contig)(args, dimensions[0]); + } +#endif + else { + BINARY_LOOP { + *((@type@ *)op1) = floor_div_@TYPE@(*(@type@*)ip1, *(@type@*)ip2); + } + } +} +/**end repeat**/ + +/**begin repeat + * Unsigned types + * #type = npy_ubyte, npy_ushort, npy_uint, npy_ulong, npy_ulonglong# + * #TYPE = UBYTE, USHORT, UINT, ULONG, ULONGLONG# + * #STYPE = BYTE, SHORT, INT, LONG, LONGLONG# + */ +#undef TO_SIMD_SFX +#if 0 +/**begin repeat1 + * #len = 8, 16, 32, 64# + */ +#elif NPY_BITSOF_@STYPE@ == @len@ + #define TO_SIMD_SFX(X) X##_u@len@ +/**end repeat1**/ +#endif +/* + * For 64-bit division on Armv7, Aarch64, and IBM/Power, NPYV fall-backs to the scalar division + * because emulating multiply-high on these architectures is going to be expensive comparing + * to the native scalar dividers. + * Therefore it's better to disable NPYV in this special case to avoid any unnecessary shuffles. + * Power10(VSX4) is an exception here since it has native support for integer vector division, + * note neither infrastructure nor NPYV has supported VSX4 yet. + */ +#if NPY_BITSOF_@STYPE@ == 64 && !defined(NPY_HAVE_VSX4) && (defined(NPY_HAVE_VSX) || defined(NPY_HAVE_NEON)) + #undef TO_SIMD_SFX +#endif +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_divide) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ + if (IS_BINARY_REDUCE) { + BINARY_REDUCE_LOOP(@type@) { + const @type@ d = *(@type@ *)ip2; + if (NPY_UNLIKELY(d == 0)) { + npy_set_floatstatus_divbyzero(); + io1 = 0; + } else { + io1 /= d; + } + } + *((@type@ *)iop1) = io1; + } +#if NPY_SIMD && defined(TO_SIMD_SFX) + // for contiguous block of memory, divisor is a scalar and not 0 + else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), NPY_SIMD_WIDTH) && + (*(@type@ *)args[1]) != 0) { + TO_SIMD_SFX(simd_divide_by_scalar_contig)(args, dimensions[0]); + } +#endif + else { + BINARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + const @type@ in2 = *(@type@ *)ip2; + if (NPY_UNLIKELY(in2 == 0)) { + npy_set_floatstatus_divbyzero(); + *((@type@ *)op1) = 0; + } else{ + *((@type@ *)op1) = in1 / in2; + } + } + } +} +/**end repeat**/ diff --git a/numpy/core/src/umath/loops_exponent_log.dispatch.c.src b/numpy/core/src/umath/loops_exponent_log.dispatch.c.src new file mode 100644 index 000000000000..41e0bf37b6f8 --- /dev/null +++ b/numpy/core/src/umath/loops_exponent_log.dispatch.c.src @@ -0,0 +1,1305 @@ +/*@targets + ** $maxopt baseline + ** (avx2 fma3) avx512f avx512_skx + **/ + +#define _UMATHMODULE +#define _MULTIARRAYMODULE +#define NPY_NO_DEPRECATED_API NPY_API_VERSION + +#include + +#include "numpy/npy_math.h" +#include "simd/simd.h" +#include "loops_utils.h" +#include "loops.h" +#include "lowlevel_strided_loops.h" +// Provides the various *_LOOP macros +#include "fast_loop_macros.h" +#include "npy_simd_data.h" + +// TODO: tweak & replace raw SIMD with NPYV + +/******************************************************************************** + ** bunch of helper functions used in ISA_exp/log_FLOAT + ********************************************************************************/ +#if !defined(_MSC_VER) && defined(NPY_HAVE_AVX512F) + /** + * For somehow MSVC commit aggressive optimization lead + * to raises 'RuntimeWarning: RuntimeWarning: overflow encountered in exp' + * + * the issue mainly caused by '_mm512_maskz_loadu_ps', we need to + * investigate about it while moving to NPYV. + */ + #define SIMD_AVX512F +#elif defined(NPY_HAVE_AVX2) && defined(NPY_HAVE_FMA3) + #define SIMD_AVX2_FMA3 +#endif +#if !defined(_MSC_VER) && defined(NPY_HAVE_AVX512_SKX) + #define SIMD_AVX512_SKX +#endif +#if defined(SIMD_AVX512F) && !(defined(__clang__) && (__clang_major__ < 10 || \ + (__clang_major__ == 10 && __clang_minor__ < 1))) + #define SIMD_AVX512F_NOCLANG_BUG +#endif + +#ifdef SIMD_AVX2_FMA3 + +static NPY_INLINE __m256 +fma_get_full_load_mask_ps(void) +{ + return _mm256_set1_ps(-1.0); +} + +static NPY_INLINE __m256i +fma_get_full_load_mask_pd(void) +{ + return _mm256_castpd_si256(_mm256_set1_pd(-1.0)); +} + +static NPY_INLINE __m256 +fma_get_partial_load_mask_ps(const npy_int num_elem, const npy_int num_lanes) +{ + float maskint[16] = {-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0,-1.0, + 1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0}; + float* addr = maskint + num_lanes - num_elem; + return _mm256_loadu_ps(addr); +} + +static NPY_INLINE __m256i +fma_get_partial_load_mask_pd(const npy_int num_elem, const npy_int num_lanes) +{ + npy_int maskint[16] = {-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1}; + npy_int* addr = maskint + 2*num_lanes - 2*num_elem; + return _mm256_loadu_si256((__m256i*) addr); +} + +static NPY_INLINE __m256 +fma_masked_gather_ps(__m256 src, + npy_float* addr, + __m256i vindex, + __m256 mask) +{ + return _mm256_mask_i32gather_ps(src, addr, vindex, mask, 4); +} + +static NPY_INLINE __m256d +fma_masked_gather_pd(__m256d src, + npy_double* addr, + __m128i vindex, + __m256d mask) +{ + return _mm256_mask_i32gather_pd(src, addr, vindex, mask, 8); +} + +static NPY_INLINE __m256 +fma_masked_load_ps(__m256 mask, npy_float* addr) +{ + return _mm256_maskload_ps(addr, _mm256_cvtps_epi32(mask)); +} + +static NPY_INLINE __m256d +fma_masked_load_pd(__m256i mask, npy_double* addr) +{ + return _mm256_maskload_pd(addr, mask); +} + +static NPY_INLINE __m256 +fma_set_masked_lanes_ps(__m256 x, __m256 val, __m256 mask) +{ + return _mm256_blendv_ps(x, val, mask); +} + +static NPY_INLINE __m256d +fma_set_masked_lanes_pd(__m256d x, __m256d val, __m256d mask) +{ + return _mm256_blendv_pd(x, val, mask); +} + +static NPY_INLINE __m256 +fma_blend(__m256 x, __m256 y, __m256 ymask) +{ + return _mm256_blendv_ps(x, y, ymask); +} + +static NPY_INLINE __m256 +fma_invert_mask_ps(__m256 ymask) +{ + return _mm256_andnot_ps(ymask, _mm256_set1_ps(-1.0)); +} + +static NPY_INLINE __m256i +fma_invert_mask_pd(__m256i ymask) +{ + return _mm256_andnot_si256(ymask, _mm256_set1_epi32(0xFFFFFFFF)); +} + +static NPY_INLINE __m256 +fma_get_exponent(__m256 x) +{ + /* + * Special handling of denormals: + * 1) Multiply denormal elements with 2**100 (0x71800000) + * 2) Get the 8 bits of unbiased exponent + * 3) Subtract 100 from exponent of denormals + */ + + __m256 two_power_100 = _mm256_castsi256_ps(_mm256_set1_epi32(0x71800000)); + __m256 denormal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_LT_OQ); + __m256 normal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_GE_OQ); + + /* + * It is necessary for temp1 to be volatile, a bug in clang optimizes it out which leads + * to an overflow warning in some cases. See https://github.com/numpy/numpy/issues/18005 + */ + volatile __m256 temp1 = _mm256_blendv_ps(x, _mm256_set1_ps(0.0f), normal_mask); + __m256 temp = _mm256_mul_ps(temp1, two_power_100); + x = _mm256_blendv_ps(x, temp, denormal_mask); + + __m256 exp = _mm256_cvtepi32_ps( + _mm256_sub_epi32( + _mm256_srli_epi32( + _mm256_castps_si256(x), 23),_mm256_set1_epi32(0x7E))); + + __m256 denorm_exp = _mm256_sub_ps(exp, _mm256_set1_ps(100.0f)); + return _mm256_blendv_ps(exp, denorm_exp, denormal_mask); +} + +static NPY_INLINE __m256 +fma_get_mantissa(__m256 x) +{ + /* + * Special handling of denormals: + * 1) Multiply denormal elements with 2**100 (0x71800000) + * 2) Get the 23 bits of mantissa + * 3) Mantissa for denormals is not affected by the multiplication + */ + + __m256 two_power_100 = _mm256_castsi256_ps(_mm256_set1_epi32(0x71800000)); + __m256 denormal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_LT_OQ); + __m256 normal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_GE_OQ); + + /* + * It is necessary for temp1 to be volatile, a bug in clang optimizes it out which leads + * to an overflow warning in some cases. See https://github.com/numpy/numpy/issues/18005 + */ + volatile __m256 temp1 = _mm256_blendv_ps(x, _mm256_set1_ps(0.0f), normal_mask); + __m256 temp = _mm256_mul_ps(temp1, two_power_100); + x = _mm256_blendv_ps(x, temp, denormal_mask); + + __m256i mantissa_bits = _mm256_set1_epi32(0x7fffff); + __m256i exp_126_bits = _mm256_set1_epi32(126 << 23); + return _mm256_castsi256_ps( + _mm256_or_si256( + _mm256_and_si256( + _mm256_castps_si256(x), mantissa_bits), exp_126_bits)); +} + +static NPY_INLINE __m256 +fma_scalef_ps(__m256 poly, __m256 quadrant) +{ + /* + * Handle denormals (which occur when quadrant <= -125): + * 1) This function computes poly*(2^quad) by adding the exponent of + poly to quad + * 2) When quad <= -125, the output is a denormal and the above logic + breaks down + * 3) To handle such cases, we split quadrant: -125 + (quadrant + 125) + * 4) poly*(2^-125) is computed the usual way + * 5) 2^(quad-125) can be computed by: 2 << abs(quad-125) + * 6) The final div operation generates the denormal + */ + __m256 minquadrant = _mm256_set1_ps(-125.0f); + __m256 denormal_mask = _mm256_cmp_ps(quadrant, minquadrant, _CMP_LE_OQ); + if (_mm256_movemask_ps(denormal_mask) != 0x0000) { + __m256 quad_diff = _mm256_sub_ps(quadrant, minquadrant); + quad_diff = _mm256_sub_ps(_mm256_setzero_ps(), quad_diff); + quad_diff = _mm256_blendv_ps(_mm256_setzero_ps(), quad_diff, denormal_mask); + __m256i two_power_diff = _mm256_sllv_epi32( + _mm256_set1_epi32(1), _mm256_cvtps_epi32(quad_diff)); + quadrant = _mm256_max_ps(quadrant, minquadrant); //keep quadrant >= -126 + __m256i exponent = _mm256_slli_epi32(_mm256_cvtps_epi32(quadrant), 23); + poly = _mm256_castsi256_ps( + _mm256_add_epi32( + _mm256_castps_si256(poly), exponent)); + __m256 denorm_poly = _mm256_div_ps(poly, _mm256_cvtepi32_ps(two_power_diff)); + return _mm256_blendv_ps(poly, denorm_poly, denormal_mask); + } + else { + __m256i exponent = _mm256_slli_epi32(_mm256_cvtps_epi32(quadrant), 23); + poly = _mm256_castsi256_ps( + _mm256_add_epi32( + _mm256_castps_si256(poly), exponent)); + return poly; + } +} + +#endif // SIMD_AVX2_FMA3 + +#ifdef SIMD_AVX512F + +static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 +avx512_get_full_load_mask_ps(void) +{ + return 0xFFFF; +} + +static NPY_INLINE __mmask8 +avx512_get_full_load_mask_pd(void) +{ + return 0xFF; +} + +static NPY_INLINE __mmask16 +avx512_get_partial_load_mask_ps(const npy_int num_elem, const npy_int total_elem) +{ + return (0x0001 << num_elem) - 0x0001; +} + +static NPY_INLINE __mmask8 +avx512_get_partial_load_mask_pd(const npy_int num_elem, const npy_int total_elem) +{ + return (0x01 << num_elem) - 0x01; +} + +static NPY_INLINE __m512 +avx512_masked_gather_ps(__m512 src, + npy_float* addr, + __m512i vindex, + __mmask16 kmask) +{ + return _mm512_mask_i32gather_ps(src, kmask, vindex, addr, 4); +} + +static NPY_INLINE __m512d +avx512_masked_gather_pd(__m512d src, + npy_double* addr, + __m256i vindex, + __mmask8 kmask) +{ + return _mm512_mask_i32gather_pd(src, kmask, vindex, addr, 8); +} + +static NPY_INLINE __m512 +avx512_masked_load_ps(__mmask16 mask, npy_float* addr) +{ + return _mm512_maskz_loadu_ps(mask, (__m512 *)addr); +} + +static NPY_INLINE __m512d +avx512_masked_load_pd(__mmask8 mask, npy_double* addr) +{ + return _mm512_maskz_loadu_pd(mask, (__m512d *)addr); +} + +static NPY_INLINE __m512 +avx512_set_masked_lanes_ps(__m512 x, __m512 val, __mmask16 mask) +{ + return _mm512_mask_blend_ps(mask, x, val); +} + +static NPY_INLINE __m512d +avx512_set_masked_lanes_pd(__m512d x, __m512d val, __mmask8 mask) +{ + return _mm512_mask_blend_pd(mask, x, val); +} + +static NPY_INLINE __m512 +avx512_blend(__m512 x, __m512 y, __mmask16 ymask) +{ + return _mm512_mask_mov_ps(x, ymask, y); +} + +static NPY_INLINE __mmask16 +avx512_invert_mask_ps(__mmask16 ymask) +{ + return _mm512_knot(ymask); +} + +static NPY_INLINE __mmask8 +avx512_invert_mask_pd(__mmask8 ymask) +{ + return _mm512_knot(ymask); +} + +static NPY_INLINE __m512 +avx512_get_exponent(__m512 x) +{ + return _mm512_add_ps(_mm512_getexp_ps(x), _mm512_set1_ps(1.0f)); +} + +static NPY_INLINE __m512 +avx512_get_mantissa(__m512 x) +{ + return _mm512_getmant_ps(x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src); +} + +static NPY_INLINE __m512 +avx512_scalef_ps(__m512 poly, __m512 quadrant) +{ + return _mm512_scalef_ps(poly, quadrant); +} + +static NPY_INLINE __m512d +avx512_permute_x4var_pd(__m512d t0, + __m512d t1, + __m512d t2, + __m512d t3, + __m512i index) +{ + __mmask8 lut_mask = _mm512_cmp_epi64_mask( + _mm512_and_epi64(_mm512_set1_epi64(0x10ULL), index), + _mm512_set1_epi64(0), _MM_CMPINT_GT); + __m512d res1 = _mm512_permutex2var_pd(t0, index, t1); + __m512d res2 = _mm512_permutex2var_pd(t2, index, t3); + return _mm512_mask_blend_pd(lut_mask, res1, res2); +} + +static NPY_INLINE __m512d +avx512_permute_x8var_pd(__m512d t0, __m512d t1, __m512d t2, __m512d t3, + __m512d t4, __m512d t5, __m512d t6, __m512d t7, + __m512i index) +{ + __mmask8 lut_mask = _mm512_cmp_epi64_mask( + _mm512_and_epi64(_mm512_set1_epi64(0x20ULL), index), + _mm512_set1_epi64(0), _MM_CMPINT_GT); + __m512d res1 = avx512_permute_x4var_pd(t0, t1, t2, t3, index); + __m512d res2 = avx512_permute_x4var_pd(t4, t5, t6, t7, index); + return _mm512_mask_blend_pd(lut_mask, res1, res2); +} + +#endif // SIMD_AVX512F + +/******************************************************************************** + ** Defining the SIMD kernels + ********************************************************************************/ +/**begin repeat + * #ISA = FMA, AVX512F# + * #isa = fma, avx512# + * #vtype = __m256, __m512# + * #vsize = 256, 512# + * #BYTES = 32, 64# + * #NUM_LANES = 8, 16# + * #mask = __m256, __mmask16# + * #vsub = , _mask# + * #or_masks =_mm256_or_ps, _mm512_kor# + * #and_masks =_mm256_and_ps, _mm512_kand# + * #xor_masks =_mm256_xor_ps, _mm512_kxor# + * #fmadd = _mm256_fmadd_ps, _mm512_fmadd_ps# + * #mask_to_int = _mm256_movemask_ps, # + * #full_mask= 0xFF, 0xFFFF# + * #masked_store = _mm256_maskstore_ps, _mm512_mask_storeu_ps# + * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #CHK = SIMD_AVX2_FMA3, SIMD_AVX512F# + */ +#ifdef @CHK@ +/* + * Vectorized Cody-Waite range reduction technique + * Performs the reduction step x* = x - y*C in three steps: + * 1) x* = x - y*c1 + * 2) x* = x - y*c2 + * 3) x* = x - y*c3 + * c1, c2 are exact floating points, c3 = C - c1 - c2 simulates higher precision + */ +static NPY_INLINE @vtype@ +simd_range_reduction(@vtype@ x, @vtype@ y, @vtype@ c1, @vtype@ c2, @vtype@ c3) +{ + @vtype@ reduced_x = @fmadd@(y, c1, x); + reduced_x = @fmadd@(y, c2, reduced_x); + reduced_x = @fmadd@(y, c3, reduced_x); + return reduced_x; +} +/* + * Vectorized implementation of exp using AVX2 and AVX512: + * 1) if x >= xmax; return INF (overflow) + * 2) if x <= xmin; return 0.0f (underflow) + * 3) Range reduction (using Coyd-Waite): + * a) y = x - k*ln(2); k = rint(x/ln(2)); y \in [0, ln(2)] + * 4) Compute exp(y) = P/Q, ratio of 2 polynomials P and Q + * b) P = 5th order and Q = 2nd order polynomials obtained from Remez's + * algorithm (mini-max polynomial approximation) + * 5) Compute exp(x) = exp(y) * 2^k + * 6) Max ULP error measured across all 32-bit FP's = 2.52 (x = 0xc2781e37) + * 7) Max relative error measured across all 32-bit FP's= 2.1264E-07 (for the + * same x = 0xc2781e37) + */ +static void +simd_exp_FLOAT(npy_float * op, + npy_float * ip, + const npy_intp array_size, + const npy_intp steps) +{ + const npy_intp stride = steps/(npy_intp)sizeof(npy_float); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); + npy_float xmax = 88.72283935546875f; + npy_float xmin = -103.97208404541015625f; + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + npy_int32 indexarr[16]; + for (npy_int32 ii = 0; ii < 16; ii++) { + indexarr[ii] = ii*stride; + } + + /* Load up frequently used constants */ + @vtype@ codyw_c1 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_LOGE_2_HIGHf); + @vtype@ codyw_c2 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_LOGE_2_LOWf); + @vtype@ exp_p0 = _mm@vsize@_set1_ps(NPY_COEFF_P0_EXPf); + @vtype@ exp_p1 = _mm@vsize@_set1_ps(NPY_COEFF_P1_EXPf); + @vtype@ exp_p2 = _mm@vsize@_set1_ps(NPY_COEFF_P2_EXPf); + @vtype@ exp_p3 = _mm@vsize@_set1_ps(NPY_COEFF_P3_EXPf); + @vtype@ exp_p4 = _mm@vsize@_set1_ps(NPY_COEFF_P4_EXPf); + @vtype@ exp_p5 = _mm@vsize@_set1_ps(NPY_COEFF_P5_EXPf); + @vtype@ exp_q0 = _mm@vsize@_set1_ps(NPY_COEFF_Q0_EXPf); + @vtype@ exp_q1 = _mm@vsize@_set1_ps(NPY_COEFF_Q1_EXPf); + @vtype@ exp_q2 = _mm@vsize@_set1_ps(NPY_COEFF_Q2_EXPf); + @vtype@ cvt_magic = _mm@vsize@_set1_ps(NPY_RINT_CVT_MAGICf); + @vtype@ log2e = _mm@vsize@_set1_ps(NPY_LOG2Ef); + @vtype@ inf = _mm@vsize@_set1_ps(NPY_INFINITYF); + @vtype@ zeros_f = _mm@vsize@_set1_ps(0.0f); + @vtype@ poly, num_poly, denom_poly, quadrant; + @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); + + @mask@ xmax_mask, xmin_mask, nan_mask, inf_mask; + @mask@ overflow_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); + @mask@ underflow_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); + @mask@ load_mask = @isa@_get_full_load_mask_ps(); + npy_intp num_remaining_elements = array_size; + + while (num_remaining_elements > 0) { + + if (num_remaining_elements < num_lanes) { + load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, + num_lanes); + } + + @vtype@ x; + if (stride == 1) { + x = @isa@_masked_load_ps(load_mask, ip); + } + else { + x = @isa@_masked_gather_ps(zeros_f, ip, vindex, load_mask); + } + + nan_mask = _mm@vsize@_cmp_ps@vsub@(x, x, _CMP_NEQ_UQ); + x = @isa@_set_masked_lanes_ps(x, zeros_f, nan_mask); + + xmax_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(xmax), _CMP_GE_OQ); + xmin_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(xmin), _CMP_LE_OQ); + inf_mask = _mm@vsize@_cmp_ps@vsub@(x, inf, _CMP_EQ_OQ); + overflow_mask = @or_masks@(overflow_mask, + @xor_masks@(xmax_mask, inf_mask)); + underflow_mask = @or_masks@(underflow_mask, xmin_mask); + + x = @isa@_set_masked_lanes_ps(x, zeros_f, @or_masks@( + @or_masks@(nan_mask, xmin_mask), xmax_mask)); + + quadrant = _mm@vsize@_mul_ps(x, log2e); + + /* round to nearest */ + quadrant = _mm@vsize@_add_ps(quadrant, cvt_magic); + quadrant = _mm@vsize@_sub_ps(quadrant, cvt_magic); + + /* Cody-Waite's range reduction algorithm */ + x = simd_range_reduction(x, quadrant, codyw_c1, codyw_c2, zeros_f); + + num_poly = @fmadd@(exp_p5, x, exp_p4); + num_poly = @fmadd@(num_poly, x, exp_p3); + num_poly = @fmadd@(num_poly, x, exp_p2); + num_poly = @fmadd@(num_poly, x, exp_p1); + num_poly = @fmadd@(num_poly, x, exp_p0); + denom_poly = @fmadd@(exp_q2, x, exp_q1); + denom_poly = @fmadd@(denom_poly, x, exp_q0); + poly = _mm@vsize@_div_ps(num_poly, denom_poly); + + /* + * compute val = poly * 2^quadrant; which is same as adding the + * exponent of quadrant to the exponent of poly. quadrant is an int, + * so extracting exponent is simply extracting 8 bits. + */ + poly = @isa@_scalef_ps(poly, quadrant); + + /* + * elem > xmax; return inf + * elem < xmin; return 0.0f + * elem = +/- nan, return nan + */ + poly = @isa@_set_masked_lanes_ps(poly, _mm@vsize@_set1_ps(NPY_NANF), nan_mask); + poly = @isa@_set_masked_lanes_ps(poly, inf, xmax_mask); + poly = @isa@_set_masked_lanes_ps(poly, zeros_f, xmin_mask); + + @masked_store@(op, @cvtps_epi32@(load_mask), poly); + + ip += num_lanes*stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } + + if (@mask_to_int@(overflow_mask)) { + npy_set_floatstatus_overflow(); + } + + if (@mask_to_int@(underflow_mask)) { + npy_set_floatstatus_underflow(); + } +} + +/* + * Vectorized implementation of log using AVX2 and AVX512 + * 1) if x < 0.0f; return -NAN (invalid input) + * 2) Range reduction: y = x/2^k; + * a) y = normalized mantissa, k is the exponent (0.5 <= y < 1) + * 3) Compute log(y) = P/Q, ratio of 2 polynomials P and Q + * b) P = 5th order and Q = 5th order polynomials obtained from Remez's + * algorithm (mini-max polynomial approximation) + * 5) Compute log(x) = log(y) + k*ln(2) + * 6) Max ULP error measured across all 32-bit FP's = 3.83 (x = 0x3f486945) + * 7) Max relative error measured across all 32-bit FP's = 2.359E-07 (for same + * x = 0x3f486945) + */ + +static void +simd_log_FLOAT(npy_float * op, + npy_float * ip, + const npy_intp array_size, + const npy_intp steps) +{ + const npy_intp stride = steps/(npy_intp)sizeof(npy_float); + const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_OUTPUT_BLOCKABLE_UNARY + */ + npy_int32 indexarr[16]; + for (npy_int32 ii = 0; ii < 16; ii++) { + indexarr[ii] = ii*stride; + } + + /* Load up frequently used constants */ + @vtype@ log_p0 = _mm@vsize@_set1_ps(NPY_COEFF_P0_LOGf); + @vtype@ log_p1 = _mm@vsize@_set1_ps(NPY_COEFF_P1_LOGf); + @vtype@ log_p2 = _mm@vsize@_set1_ps(NPY_COEFF_P2_LOGf); + @vtype@ log_p3 = _mm@vsize@_set1_ps(NPY_COEFF_P3_LOGf); + @vtype@ log_p4 = _mm@vsize@_set1_ps(NPY_COEFF_P4_LOGf); + @vtype@ log_p5 = _mm@vsize@_set1_ps(NPY_COEFF_P5_LOGf); + @vtype@ log_q0 = _mm@vsize@_set1_ps(NPY_COEFF_Q0_LOGf); + @vtype@ log_q1 = _mm@vsize@_set1_ps(NPY_COEFF_Q1_LOGf); + @vtype@ log_q2 = _mm@vsize@_set1_ps(NPY_COEFF_Q2_LOGf); + @vtype@ log_q3 = _mm@vsize@_set1_ps(NPY_COEFF_Q3_LOGf); + @vtype@ log_q4 = _mm@vsize@_set1_ps(NPY_COEFF_Q4_LOGf); + @vtype@ log_q5 = _mm@vsize@_set1_ps(NPY_COEFF_Q5_LOGf); + @vtype@ loge2 = _mm@vsize@_set1_ps(NPY_LOGE2f); + @vtype@ nan = _mm@vsize@_set1_ps(NPY_NANF); + @vtype@ neg_nan = _mm@vsize@_set1_ps(-NPY_NANF); + @vtype@ neg_inf = _mm@vsize@_set1_ps(-NPY_INFINITYF); + @vtype@ inf = _mm@vsize@_set1_ps(NPY_INFINITYF); + @vtype@ zeros_f = _mm@vsize@_set1_ps(0.0f); + @vtype@ ones_f = _mm@vsize@_set1_ps(1.0f); + @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)indexarr); + @vtype@ poly, num_poly, denom_poly, exponent; + + @mask@ inf_mask, nan_mask, sqrt2_mask, zero_mask, negx_mask; + @mask@ invalid_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); + @mask@ divide_by_zero_mask = invalid_mask; + @mask@ load_mask = @isa@_get_full_load_mask_ps(); + npy_intp num_remaining_elements = array_size; + + while (num_remaining_elements > 0) { + + if (num_remaining_elements < num_lanes) { + load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, + num_lanes); + } + + @vtype@ x_in; + if (stride == 1) { + x_in = @isa@_masked_load_ps(load_mask, ip); + } + else { + x_in = @isa@_masked_gather_ps(zeros_f, ip, vindex, load_mask); + } + + negx_mask = _mm@vsize@_cmp_ps@vsub@(x_in, zeros_f, _CMP_LT_OQ); + zero_mask = _mm@vsize@_cmp_ps@vsub@(x_in, zeros_f, _CMP_EQ_OQ); + inf_mask = _mm@vsize@_cmp_ps@vsub@(x_in, inf, _CMP_EQ_OQ); + nan_mask = _mm@vsize@_cmp_ps@vsub@(x_in, x_in, _CMP_NEQ_UQ); + divide_by_zero_mask = @or_masks@(divide_by_zero_mask, + @and_masks@(zero_mask, load_mask)); + invalid_mask = @or_masks@(invalid_mask, negx_mask); + + @vtype@ x = @isa@_set_masked_lanes_ps(x_in, zeros_f, negx_mask); + + /* set x = normalized mantissa */ + exponent = @isa@_get_exponent(x); + x = @isa@_get_mantissa(x); + + /* if x < sqrt(2) {exp = exp-1; x = 2*x} */ + sqrt2_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(NPY_SQRT1_2f), _CMP_LE_OQ); + x = @isa@_blend(x, _mm@vsize@_add_ps(x,x), sqrt2_mask); + exponent = @isa@_blend(exponent, + _mm@vsize@_sub_ps(exponent,ones_f), sqrt2_mask); + + /* x = x - 1 */ + x = _mm@vsize@_sub_ps(x, ones_f); + + /* Polynomial approximation for log(1+x) */ + num_poly = @fmadd@(log_p5, x, log_p4); + num_poly = @fmadd@(num_poly, x, log_p3); + num_poly = @fmadd@(num_poly, x, log_p2); + num_poly = @fmadd@(num_poly, x, log_p1); + num_poly = @fmadd@(num_poly, x, log_p0); + denom_poly = @fmadd@(log_q5, x, log_q4); + denom_poly = @fmadd@(denom_poly, x, log_q3); + denom_poly = @fmadd@(denom_poly, x, log_q2); + denom_poly = @fmadd@(denom_poly, x, log_q1); + denom_poly = @fmadd@(denom_poly, x, log_q0); + poly = _mm@vsize@_div_ps(num_poly, denom_poly); + poly = @fmadd@(exponent, loge2, poly); + + /* + * x < 0.0f; return -NAN + * x = +/- NAN; return NAN + * x = 0.0f; return -INF + */ + poly = @isa@_set_masked_lanes_ps(poly, nan, nan_mask); + poly = @isa@_set_masked_lanes_ps(poly, neg_nan, negx_mask); + poly = @isa@_set_masked_lanes_ps(poly, neg_inf, zero_mask); + poly = @isa@_set_masked_lanes_ps(poly, inf, inf_mask); + + @masked_store@(op, @cvtps_epi32@(load_mask), poly); + + ip += num_lanes*stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } + + if (@mask_to_int@(invalid_mask)) { + npy_set_floatstatus_invalid(); + } + if (@mask_to_int@(divide_by_zero_mask)) { + npy_set_floatstatus_divbyzero(); + } +} +#endif // @CHK@ +/**end repeat**/ + +#ifdef SIMD_AVX512F_NOCLANG_BUG +/* + * Vectorized implementation of exp double using AVX512 + * Reference: Tang, P.T.P., "Table-driven implementation of the + * exponential function in IEEE floating-point + * arithmetic," ACM Transactions on Mathematical + * Software, vol. 15, pp. 144-157, 1989. + * 1) if x > mTH_max or x is INF; return INF (overflow) + * 2) if x < mTH_min; return 0.0f (underflow) + * 3) if abs(x) < mTH_nearzero; return 1.0f + x + * 4) if x is Nan; return Nan + * 5) Range reduction: + * x = (32m + j)ln2 / 32 + r; r in [-ln2/64, ln2/64] + * 6) exp(r) - 1 is approximated by a polynomial function p(r) + * exp(x) = 2^m(2^(j/32) + 2^(j/32)p(r)); + */ +static void +AVX512F_exp_DOUBLE(npy_double * op, + npy_double * ip, + const npy_intp array_size, + const npy_intp steps) +{ + npy_intp num_remaining_elements = array_size; + const npy_intp stride = steps / (npy_intp)sizeof(npy_double); + const npy_int num_lanes = 64 / (npy_intp)sizeof(npy_double); + npy_int32 indexarr[8]; + for (npy_int32 ii = 0; ii < 8; ii++) { + indexarr[ii] = ii*stride; + } + + __m512d InvLn2N = _mm512_set1_pd(NPY_INV_LN2_MUL_32); + __m512d mShift = _mm512_set1_pd(NPY_RINT_CVT_MAGIC); + __m512d mNegL1 = _mm512_set1_pd(NPY_TANG_NEG_L1); + __m512d mNegL2 = _mm512_set1_pd(NPY_TANG_NEG_L2); + __m512i mMod = _mm512_set1_epi64(0x1f); + __m512d mA1 = _mm512_set1_pd(NPY_TANG_A1); + __m512d mA2 = _mm512_set1_pd(NPY_TANG_A2); + __m512d mA3 = _mm512_set1_pd(NPY_TANG_A3); + __m512d mA4 = _mm512_set1_pd(NPY_TANG_A4); + __m512d mA5 = _mm512_set1_pd(NPY_TANG_A5); + __m512d mTH_nearzero = _mm512_set1_pd(0x1p-54); + __m512d mTH_max = _mm512_set1_pd(0x1.62e42fefa39efp+9); + __m512d mTH_min = _mm512_set1_pd(-0x1.74910d52d3053p+9); + __m512d mTH_inf = _mm512_set1_pd(NPY_INFINITY); + __m512d zeros_d = _mm512_set1_pd(0.0f); + __m512d ones_d = _mm512_set1_pd(1.0f); + __m256i vindex = _mm256_loadu_si256((__m256i*)&indexarr[0]); + + __m512d mTable_top_0 = _mm512_loadu_pd(&(EXP_Table_top[8*0])); + __m512d mTable_top_1 = _mm512_loadu_pd(&(EXP_Table_top[8*1])); + __m512d mTable_top_2 = _mm512_loadu_pd(&(EXP_Table_top[8*2])); + __m512d mTable_top_3 = _mm512_loadu_pd(&(EXP_Table_top[8*3])); + __m512d mTable_tail_0 = _mm512_loadu_pd(&(EXP_Table_tail[8*0])); + __m512d mTable_tail_1 = _mm512_loadu_pd(&(EXP_Table_tail[8*1])); + __m512d mTable_tail_2 = _mm512_loadu_pd(&(EXP_Table_tail[8*2])); + __m512d mTable_tail_3 = _mm512_loadu_pd(&(EXP_Table_tail[8*3])); + + __mmask8 overflow_mask = avx512_get_partial_load_mask_pd(0, num_lanes); + __mmask8 underflow_mask = avx512_get_partial_load_mask_pd(0, num_lanes); + __mmask8 load_mask = avx512_get_full_load_mask_pd(); + __mmask8 xmin_mask, xmax_mask, inf_mask, nan_mask, nearzero_mask; + + while (num_remaining_elements > 0) { + if (num_remaining_elements < num_lanes) { + load_mask = avx512_get_partial_load_mask_pd(num_remaining_elements, + num_lanes); + } + + __m512d x; + if (1 == stride) { + x = avx512_masked_load_pd(load_mask, ip); + } + else { + x = avx512_masked_gather_pd(zeros_d, ip, vindex, load_mask); + } + + nan_mask = _mm512_cmp_pd_mask(x, x, _CMP_NEQ_UQ); + x = avx512_set_masked_lanes_pd(x, zeros_d, nan_mask); + xmax_mask = _mm512_cmp_pd_mask(x, mTH_max, _CMP_GT_OQ); + xmin_mask = _mm512_cmp_pd_mask(x, mTH_min, _CMP_LT_OQ); + inf_mask = _mm512_cmp_pd_mask(x, mTH_inf, _CMP_EQ_OQ); + __m512i x_abs = _mm512_and_epi64(_mm512_castpd_si512(x), + _mm512_set1_epi64(0x7FFFFFFFFFFFFFFF)); + nearzero_mask = _mm512_cmp_pd_mask(_mm512_castsi512_pd(x_abs), + mTH_nearzero, _CMP_LT_OQ); + nearzero_mask = _mm512_kxor(nearzero_mask, nan_mask); + overflow_mask = _mm512_kor(overflow_mask, + _mm512_kxor(xmax_mask, inf_mask)); + underflow_mask = _mm512_kor(underflow_mask, xmin_mask); + x = avx512_set_masked_lanes_pd(x, zeros_d, + _mm512_kor(_mm512_kor(nan_mask, xmin_mask), + _mm512_kor(xmax_mask, nearzero_mask))); + + /* z = x * 32/ln2 */ + __m512d z = _mm512_mul_pd(x, InvLn2N); + + /* round to nearest */ + __m512d kd = _mm512_add_pd(z, mShift); + __m512i ki = _mm512_castpd_si512(kd); + kd = _mm512_sub_pd(kd, mShift); + + /* r = (x + kd*mNegL1) + kd*mNegL2 */ + __m512d r1 = _mm512_fmadd_pd(kd, mNegL1, x); + __m512d r2 = _mm512_mul_pd(kd, mNegL2); + __m512d r = _mm512_add_pd(r1,r2); + + /* Polynomial approximation for exp(r) - 1 */ + __m512d q = _mm512_fmadd_pd(mA5, r, mA4); + q = _mm512_fmadd_pd(q, r, mA3); + q = _mm512_fmadd_pd(q, r, mA2); + q = _mm512_fmadd_pd(q, r, mA1); + q = _mm512_mul_pd(q, r); + __m512d p = _mm512_fmadd_pd(r, q, r2);; + p = _mm512_add_pd(r1, p); + + /* Get 2^(j/32) from lookup table */ + __m512i j = _mm512_and_epi64(ki, mMod); + __m512d top = avx512_permute_x4var_pd(mTable_top_0, mTable_top_1, + mTable_top_2, mTable_top_3, j); + __m512d tail = avx512_permute_x4var_pd(mTable_tail_0, mTable_tail_1, + mTable_tail_2, mTable_tail_3, j); + + /* + * s = top + tail; + * exp(x) = 2^m * (top + (tail + s * p)); + */ + __m512d s = _mm512_add_pd(top, tail); + __m512d res = _mm512_fmadd_pd(s, p, tail); + res = _mm512_add_pd(res, top); + res= _mm512_scalef_pd(res, _mm512_div_pd(kd, _mm512_set1_pd(32))); + + /* return special cases */ + res = avx512_set_masked_lanes_pd(res, _mm512_add_pd(x, ones_d), + nearzero_mask); + res = avx512_set_masked_lanes_pd(res, _mm512_set1_pd(NPY_NAN), + nan_mask); + res = avx512_set_masked_lanes_pd(res, mTH_inf, xmax_mask); + res = avx512_set_masked_lanes_pd(res, zeros_d, xmin_mask); + + _mm512_mask_storeu_pd(op, load_mask, res); + + ip += num_lanes * stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } + if (overflow_mask) { + npy_set_floatstatus_overflow(); + } + + if (underflow_mask) { + npy_set_floatstatus_underflow(); + } +} +/* + * Vectorized implementation of log double using AVX512 + * Reference: + * [1] Tang, Ping Tak Peter. Table-lookup algorithms for elementary functions + * and their error analysis. No. CONF-9106103-1. Argonne National Lab., + * IL (USA), 1991. + * [2] Tang, Ping-Tak Peter. "Table-driven implementation of the logarithm + * function in IEEE floating-point arithmetic." ACM Transactions on + * Mathematical Software (TOMS) 16.4 (1990): 378-400. + * [3] Muller, Jean-Michel. "Elementary functions: algorithms and + * implementation." (2016). + * 1) if x = 0; return -INF + * 2) if x < 0; return NAN + * 3) if x is INF; return INF + * 4) if x is NAN; return NAN + * 5) if x on (1.0 - 0x1p-4, 1.0 + 0x1.09p-4), calling npy_log() + * 6) Range reduction: + * log(x) = log(2^m * z) + * = mln2 + log(z) + * 7) log(z) = log(z / c_k) + log(c_k); + * where c_k = 1 + k/64, k = 0,1,...,64 + * s.t. |x - c_k| <= 1/128 when x on[1,2]. + * 8) r = 2(x - c_k)/(x + c_k) + * log(x/c_k) = log((1 + r/2) / (1 - r/2)) + * = p(r) + * = 2((r/2) + 1/3*(r/2)^3 + 1/5*(r/2)^5 + ...) + */ +static void +AVX512F_log_DOUBLE(npy_double * op, + npy_double * ip, + const npy_intp array_size, + const npy_intp steps) +{ + npy_intp num_remaining_elements = array_size; + const npy_intp stride = steps / (npy_intp)sizeof(npy_double); + const npy_int num_lanes = 64 / (npy_intp)sizeof(npy_double); + npy_int32 indexarr[8]; + for (npy_int32 ii = 0; ii < 8; ii++) { + indexarr[ii] = ii*stride; + } + + __m512d zeros_d = _mm512_set1_pd(0.0f); + __m512d ones_d = _mm512_set1_pd(1.0f); + __m512d mInf = _mm512_set1_pd(NPY_INFINITY); + __m512d mInv64 = _mm512_castsi512_pd(_mm512_set1_epi64(0x3f90000000000000)); + __m512d mNeg_nan = _mm512_set1_pd(-NPY_NAN); + __m512d mNan = _mm512_set1_pd(NPY_NAN); + __m512d mNeg_inf = _mm512_set1_pd(-NPY_INFINITY); + __m512d mA1 = _mm512_set1_pd(NPY_TANG_LOG_A1); + __m512d mA2 = _mm512_set1_pd(NPY_TANG_LOG_A2); + __m512d mA3 = _mm512_set1_pd(NPY_TANG_LOG_A3); + __m512d mA4 = _mm512_set1_pd(NPY_TANG_LOG_A4); + __m512d mLN2HI = _mm512_set1_pd(NPY_TANG_LOG_LN2HI); + __m512d mLN2LO = _mm512_set1_pd(NPY_TANG_LOG_LN2LO); + + __m512d mTo_glibc_min = _mm512_set1_pd(1.0 - 0x1p-4); + __m512d mTo_glibc_max = _mm512_set1_pd(1.0 + 0x1.09p-4); + __m256i vindex = _mm256_loadu_si256((__m256i*)&indexarr[0]); + + /* Load lookup table data */ + /**begin repeat + * #i = 0, 1, 2, 3, 4, 5, 6, 7# + */ + + __m512d mLUT_TOP_@i@ = _mm512_loadu_pd(&(LOG_TABLE_TOP[8*@i@])); + __m512d mLUT_TAIL_@i@ = _mm512_loadu_pd(&(LOG_TABLE_TAIL[8*@i@])); + + /**end repeat**/ + + __mmask8 load_mask = avx512_get_full_load_mask_pd(); + __mmask8 invalid_mask = avx512_get_partial_load_mask_pd(0, num_lanes); + __mmask8 divide_by_zero_mask = invalid_mask; + + __mmask8 inf_mask, nan_mask, zero_mask, negx_mask, denormal_mask, + glibc_mask; + + __m512d x_in; + while (num_remaining_elements > 0) { + if (num_remaining_elements < num_lanes) { + load_mask = avx512_get_partial_load_mask_pd(num_remaining_elements, + num_lanes); + } + + if (1 == stride) { + x_in = avx512_masked_load_pd(load_mask, ip); + } + else { + x_in = avx512_masked_gather_pd(zeros_d, ip, vindex, load_mask); + } + + /* call glibc when x on [1.0 - 0x1p-4, 1.0 + 0x1.09p-4] */ + __mmask8 m1 = _mm512_cmp_pd_mask(x_in, mTo_glibc_max, _CMP_LT_OQ); + __mmask8 m2 = _mm512_cmp_pd_mask(x_in, mTo_glibc_min, _CMP_GT_OQ); + glibc_mask = m1 & m2; + + if (glibc_mask != 0xFF) { + zero_mask = _mm512_cmp_pd_mask(x_in, zeros_d, _CMP_EQ_OQ); + inf_mask = _mm512_cmp_pd_mask(x_in, mInf, _CMP_EQ_OQ); + negx_mask = _mm512_cmp_pd_mask(x_in, zeros_d, _CMP_LT_OQ); + nan_mask = _mm512_cmp_pd_mask(x_in, x_in, _CMP_NEQ_UQ); + + divide_by_zero_mask = divide_by_zero_mask | (zero_mask & load_mask); + invalid_mask = invalid_mask | negx_mask; + + __m512d x = avx512_set_masked_lanes_pd(x_in, zeros_d, negx_mask); + __m512i ix = _mm512_castpd_si512(x); + + /* Normalize x when it is denormal */ + __m512i top12 = _mm512_and_epi64(ix, + _mm512_set1_epi64(0xfff0000000000000)); + denormal_mask = _mm512_cmp_epi64_mask(top12, _mm512_set1_epi64(0), + _CMP_EQ_OQ); + denormal_mask = (~zero_mask) & denormal_mask; + ix = _mm512_castpd_si512(_mm512_mask_mul_pd(x, denormal_mask, + x, _mm512_set1_pd(0x1p52))); + ix = _mm512_mask_sub_epi64(ix, denormal_mask, + ix, _mm512_set1_epi64(52ULL << 52)); + + /* + * x = 2^k * z; where z in range [1,2] + */ + __m512i tmp = _mm512_sub_epi64(ix, + _mm512_set1_epi64(0x3ff0000000000000)); + __m512i i = _mm512_and_epi64(_mm512_srai_epi64(tmp, 52 - 6), + _mm512_set1_epi64(0x3fULL)); + __m512i ik = _mm512_srai_epi64(tmp, 52); + __m512d z = _mm512_castsi512_pd(_mm512_sub_epi64(ix, _mm512_and_epi64(tmp, + _mm512_set1_epi64(0xfff0000000000000)))); + /* c = i/64 + 1 */ + __m256i i_32 = _mm512_cvtepi64_epi32(i); + __m512d c = _mm512_fmadd_pd(_mm512_cvtepi32_pd(i_32), mInv64, ones_d); + + /* u = 2 * (z - c) / (z + c) */ + __m512d u = _mm512_div_pd(_mm512_sub_pd(z, c), _mm512_add_pd(z, c)); + u = _mm512_mul_pd(_mm512_set1_pd(2.0), u); + + /* v = u * u */ + __m512d v = _mm512_mul_pd(u,u); + + /* log(z/c) = u + u*v*(A1 + v*(A2 + v*(A3 + v*A4))) */ + __m512d res = _mm512_fmadd_pd(v, mA4, mA3); + res = _mm512_fmadd_pd(v, res, mA2); + res = _mm512_fmadd_pd(v, res, mA1); + res = _mm512_mul_pd(v, res); + res = _mm512_fmadd_pd(u, res, u); + + /* Load lookup table data */ + __m512d c_hi = avx512_permute_x8var_pd(mLUT_TOP_0, mLUT_TOP_1, + mLUT_TOP_2, mLUT_TOP_3, mLUT_TOP_4, mLUT_TOP_5, + mLUT_TOP_6, mLUT_TOP_7, i); + __m512d c_lo = avx512_permute_x8var_pd(mLUT_TAIL_0, mLUT_TAIL_1, + mLUT_TAIL_2, mLUT_TAIL_3, mLUT_TAIL_4, mLUT_TAIL_5, + mLUT_TAIL_6, mLUT_TAIL_7, i); + + /* + * log(x) = k * ln2_hi + c_hi + + * k * ln2_lo + c_lo + + * log(z/c) + */ + __m256i ik_32 = _mm512_cvtepi64_epi32(ik); + __m512d k = _mm512_cvtepi32_pd(ik_32); + __m512d tt = _mm512_fmadd_pd(k, mLN2HI, c_hi); + __m512d tt2 = _mm512_fmadd_pd(k, mLN2LO, c_lo); + tt = _mm512_add_pd(tt, tt2); + res = _mm512_add_pd(tt, res); + + /* return special cases */ + res = avx512_set_masked_lanes_pd(res, mNan, nan_mask); + res = avx512_set_masked_lanes_pd(res, mNeg_nan, negx_mask); + res = avx512_set_masked_lanes_pd(res, mNeg_inf, zero_mask); + res = avx512_set_masked_lanes_pd(res, mInf, inf_mask); + + _mm512_mask_storeu_pd(op, load_mask, res); + } + + /* call glibc's log func when x around 1.0f */ + if (glibc_mask != 0) { + double NPY_DECL_ALIGNED(64) ip_fback[8]; + _mm512_store_pd(ip_fback, x_in); + + for (int ii = 0; ii < 8; ++ii, glibc_mask >>= 1) { + if (glibc_mask & 0x01) { + op[ii] = npy_log(ip_fback[ii]); + } + } + } + ip += num_lanes * stride; + op += num_lanes; + num_remaining_elements -= num_lanes; + } + + if (invalid_mask) { + npy_set_floatstatus_invalid(); + } + if (divide_by_zero_mask) { + npy_set_floatstatus_divbyzero(); + } +} +#endif // AVX512F_NOCLANG_BUG + +#ifdef SIMD_AVX512_SKX +/**begin repeat + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + * #num_lanes = 16, 8# + * #vsuffix = ps, pd# + * #mask = __mmask16, __mmask8# + * #vtype1 = __m512, __m512d# + * #vtype2 = __m512i, __m256i# + * #scale = 4, 8# + * #vindextype = __m512i, __m256i# + * #vindexsize = 512, 256# + * #vindexload = _mm512_loadu_si512, _mm256_loadu_si256# + * #vtype2_load = _mm512_maskz_loadu_epi32, _mm256_maskz_loadu_epi32# + * #vtype2_gather = _mm512_mask_i32gather_epi32, _mm256_mmask_i32gather_epi32# + * #vtype2_store = _mm512_mask_storeu_epi32, _mm256_mask_storeu_epi32# + * #vtype2_scatter = _mm512_mask_i32scatter_epi32, _mm256_mask_i32scatter_epi32# + * #setzero = _mm512_setzero_epi32, _mm256_setzero_si256# + */ +static NPY_INLINE void +AVX512_SKX_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) +{ + const npy_intp stride_ip1 = steps[0]/(npy_intp)sizeof(@type@); + const npy_intp stride_ip2 = steps[1]/(npy_intp)sizeof(int); + const npy_intp stride_op = steps[2]/(npy_intp)sizeof(@type@); + const npy_intp array_size = dimensions[0]; + npy_intp num_remaining_elements = array_size; + @type@* ip1 = (@type@*) args[0]; + int* ip2 = (int*) args[1]; + @type@* op = (@type@*) args[2]; + + @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP + */ + + npy_int32 index_ip1[@num_lanes@], index_ip2[@num_lanes@], index_op[@num_lanes@]; + for (npy_int32 ii = 0; ii < @num_lanes@; ii++) { + index_ip1[ii] = ii*stride_ip1; + index_ip2[ii] = ii*stride_ip2; + index_op[ii] = ii*stride_op; + } + @vindextype@ vindex_ip1 = @vindexload@((@vindextype@*)&index_ip1[0]); + @vindextype@ vindex_ip2 = @vindexload@((@vindextype@*)&index_ip2[0]); + @vindextype@ vindex_op = @vindexload@((@vindextype@*)&index_op[0]); + @vtype1@ zeros_f = _mm512_setzero_@vsuffix@(); + @vtype2@ zeros = @setzero@(); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + } + @vtype1@ x1; + @vtype2@ x2; + if (stride_ip1 == 1) { + x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); + } + else { + x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip1, vindex_ip1, load_mask); + } + if (stride_ip2 == 1) { + x2 = @vtype2_load@(load_mask, ip2); + } + else { + x2 = @vtype2_gather@(zeros, load_mask, vindex_ip2, ip2, 4); + } + + @vtype1@ out = _mm512_scalef_@vsuffix@(x1, _mm512_cvtepi32_@vsuffix@(x2)); + + if (stride_op == 1) { + _mm512_mask_storeu_@vsuffix@(op, load_mask, out); + } + else { + /* scatter! */ + _mm512_mask_i32scatter_@vsuffix@(op, load_mask, vindex_op, out, @scale@); + } + + ip1 += @num_lanes@*stride_ip1; + ip2 += @num_lanes@*stride_ip2; + op += @num_lanes@*stride_op; + num_remaining_elements -= @num_lanes@; + } +} + +static NPY_INLINE void +AVX512_SKX_frexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) +{ + const npy_intp stride_ip1 = steps[0]/(npy_intp)sizeof(@type@); + const npy_intp stride_op1 = steps[1]/(npy_intp)sizeof(@type@); + const npy_intp stride_op2 = steps[2]/(npy_intp)sizeof(int); + const npy_intp array_size = dimensions[0]; + npy_intp num_remaining_elements = array_size; + @type@* ip1 = (@type@*) args[0]; + @type@* op1 = (@type@*) args[1]; + int* op2 = (int*) args[2]; + + @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); + + /* + * Note: while generally indices are npy_intp, we ensure that our maximum index + * will fit in an int32 as a precondition for this function via + * IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP + */ + + npy_int32 index_ip1[@num_lanes@], index_op1[@num_lanes@], index_op2[@num_lanes@]; + for (npy_int32 ii = 0; ii < @num_lanes@; ii++) { + index_ip1[ii] = ii*stride_ip1; + index_op1[ii] = ii*stride_op1; + index_op2[ii] = ii*stride_op2; + } + @vindextype@ vindex_ip1 = @vindexload@((@vindextype@*)&index_ip1[0]); + @vindextype@ vindex_op1 = @vindexload@((@vindextype@*)&index_op1[0]); + @vindextype@ vindex_op2 = @vindexload@((@vindextype@*)&index_op2[0]); + @vtype1@ zeros_f = _mm512_setzero_@vsuffix@(); + + while (num_remaining_elements > 0) { + if (num_remaining_elements < @num_lanes@) { + load_mask = avx512_get_partial_load_mask_@vsuffix@( + num_remaining_elements, @num_lanes@); + } + @vtype1@ x1; + if (stride_ip1 == 1) { + x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); + } + else { + x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip1, vindex_ip1, load_mask); + } + + /* + * The x86 instructions vpgetmant and vpgetexp do not conform + * with NumPy's output for special floating points: NAN, +/-INF, +/-0.0 + * We mask these values with spmask to avoid invalid exceptions. + */ + @mask@ spmask =_mm512_knot(_mm512_fpclass_@vsuffix@_mask( + x1, 0b10011111)); + @vtype1@ out1 = _mm512_maskz_getmant_@vsuffix@( + spmask, x1, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src); + out1 = _mm512_mask_mov_@vsuffix@(x1, spmask, out1); + @vtype2@ out2 = _mm512_cvt@vsuffix@_epi32( + _mm512_maskz_add_@vsuffix@(spmask, _mm512_set1_@vsuffix@(1.0), + _mm512_maskz_getexp_@vsuffix@(spmask, x1))); + if (stride_op1 == 1) { + _mm512_mask_storeu_@vsuffix@(op1, load_mask, out1); + } + else { + _mm512_mask_i32scatter_@vsuffix@(op1, load_mask, vindex_op1, out1, @scale@); + } + if (stride_op2 == 1) { + @vtype2_store@(op2, load_mask, out2); + } + else { + @vtype2_scatter@(op2, load_mask, vindex_op2, out2, 4); + } + + ip1 += @num_lanes@*stride_ip1; + op1 += @num_lanes@*stride_op1; + op2 += @num_lanes@*stride_op2; + num_remaining_elements -= @num_lanes@; + } +} +/**end repeat**/ +#endif // SIMD_AVX512_SKX + + +/******************************************************************************** + ** Defining ufunc inner functions + ********************************************************************************/ +/**begin repeat + * #func = exp, log# + * #scalarf = npy_expf, npy_logf# + */ +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_@func@) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) +{ +#if defined(SIMD_AVX2_FMA3) || defined(SIMD_AVX512F) + // third arg in `IS_OUTPUT_BLOCKABLE_UNARY` is dummy + // TODO: get ride of this macro during the move to NPYV + if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_float), sizeof(npy_float), 64)) { + simd_@func@_FLOAT((npy_float*)args[1], (npy_float*)args[0], dimensions[0], steps[0]); + } + else { + UNARY_LOOP { + /* + * We use the AVX function to compute exp/log for scalar elements as well. + * This is needed to ensure the output of strided and non-strided + * cases match. SIMD code handles strided input cases, but not + * strided output. + */ + simd_@func@_FLOAT((npy_float *)op1, (npy_float *)ip1, 1, steps[0]); + } + } +#else + UNARY_LOOP { + const npy_float in1 = *(npy_float *)ip1; + *(npy_float *)op1 = @scalarf@(in1); + } +#endif +} +/**end repeat**/ + +/**begin repeat + * #func = exp, log# + * #scalar = npy_exp, npy_log# + */ +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(DOUBLE_@func@) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) +{ +#ifdef SIMD_AVX512F_NOCLANG_BUG + if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_double), sizeof(npy_double), 64)) { + AVX512F_@func@_DOUBLE((npy_double*)args[1], (npy_double*)args[0], dimensions[0], steps[0]); + return; + } +#endif + UNARY_LOOP { + const npy_double in1 = *(npy_double *)ip1; + *(npy_double *)op1 = @scalar@(in1); + } +} +/**end repeat**/ + +/**begin repeat + * Float types + * #type = npy_float, npy_double# + * #TYPE = FLOAT, DOUBLE# + * #c = f, # + * #C = F, # + */ +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_frexp) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ +#ifdef SIMD_AVX512_SKX + if (IS_UNARY_TWO_OUT_SMALL_STEPS_AND_NOMEMOVERLAP) { + AVX512_SKX_frexp_@TYPE@(args, dimensions, steps); + return; + } +#endif + UNARY_LOOP_TWO_OUT { + const @type@ in1 = *(@type@ *)ip1; + *((@type@ *)op1) = npy_frexp@c@(in1, (int *)op2); + } +} + +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_ldexp) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(func)) +{ +#ifdef SIMD_AVX512_SKX + if (IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP) { + AVX512_SKX_ldexp_@TYPE@(args, dimensions, steps); + return; + } +#endif + BINARY_LOOP { + const @type@ in1 = *(@type@ *)ip1; + const int in2 = *(int *)ip2; + *((@type@ *)op1) = npy_ldexp@c@(in1, in2); + } +} +/**end repeat**/ diff --git a/numpy/core/src/umath/loops_trigonometric.dispatch.c.src b/numpy/core/src/umath/loops_trigonometric.dispatch.c.src new file mode 100644 index 000000000000..8c2c83e7c998 --- /dev/null +++ b/numpy/core/src/umath/loops_trigonometric.dispatch.c.src @@ -0,0 +1,230 @@ +/*@targets + ** $maxopt baseline + ** (avx2 fma3) avx512f + ** vsx2 + ** neon_vfpv4 + **/ +#include "numpy/npy_math.h" +#include "simd/simd.h" +#include "loops_utils.h" +#include "loops.h" +/* + * TODO: + * - use vectorized version of Payne-Hanek style reduction for large elements or + * when there's no native FUSED support instead of fallback to libc + */ +#if NPY_SIMD_FMA3 // native support +/* + * Vectorized Cody-Waite range reduction technique + * Performs the reduction step x* = x - y*C in three steps: + * 1) x* = x - y*c1 + * 2) x* = x - y*c2 + * 3) x* = x - y*c3 + * c1, c2 are exact floating points, c3 = C - c1 - c2 simulates higher precision + */ +NPY_FINLINE npyv_f32 +simd_range_reduction_f32(npyv_f32 x, npyv_f32 y, npyv_f32 c1, npyv_f32 c2, npyv_f32 c3) +{ + npyv_f32 reduced_x = npyv_muladd_f32(y, c1, x); + reduced_x = npyv_muladd_f32(y, c2, reduced_x); + reduced_x = npyv_muladd_f32(y, c3, reduced_x); + return reduced_x; +} +/* + * Approximate cosine algorithm for x \in [-PI/4, PI/4] + * Maximum ULP across all 32-bit floats = 0.875 + */ +NPY_FINLINE npyv_f32 +simd_cosine_poly_f32(npyv_f32 x2) +{ + const npyv_f32 invf8 = npyv_setall_f32(0x1.98e616p-16f); + const npyv_f32 invf6 = npyv_setall_f32(-0x1.6c06dcp-10f); + const npyv_f32 invf4 = npyv_setall_f32(0x1.55553cp-05f); + const npyv_f32 invf2 = npyv_setall_f32(-0x1.000000p-01f); + const npyv_f32 invf0 = npyv_setall_f32(0x1.000000p+00f); + + npyv_f32 r = npyv_muladd_f32(invf8, x2, invf6); + r = npyv_muladd_f32(r, x2, invf4); + r = npyv_muladd_f32(r, x2, invf2); + r = npyv_muladd_f32(r, x2, invf0); + return r; +} +/* + * Approximate sine algorithm for x \in [-PI/4, PI/4] + * Maximum ULP across all 32-bit floats = 0.647 + * Polynomial approximation based on unpublished work by T. Myklebust + */ +NPY_FINLINE npyv_f32 +simd_sine_poly_f32(npyv_f32 x, npyv_f32 x2) +{ + const npyv_f32 invf9 = npyv_setall_f32(0x1.7d3bbcp-19f); + const npyv_f32 invf7 = npyv_setall_f32(-0x1.a06bbap-13f); + const npyv_f32 invf5 = npyv_setall_f32(0x1.11119ap-07f); + const npyv_f32 invf3 = npyv_setall_f32(-0x1.555556p-03f); + + npyv_f32 r = npyv_muladd_f32(invf9, x2, invf7); + r = npyv_muladd_f32(r, x2, invf5); + r = npyv_muladd_f32(r, x2, invf3); + r = npyv_muladd_f32(r, x2, npyv_zero_f32()); + r = npyv_muladd_f32(r, x, x); + return r; +} +/* + * Vectorized approximate sine/cosine algorithms: The following code is a + * vectorized version of the algorithm presented here: + * https://stackoverflow.com/questions/30463616/payne-hanek-algorithm-implementation-in-c/30465751#30465751 + * (1) Load data in registers and generate mask for elements that are + * within range [-71476.0625f, 71476.0625f] for cosine and [-117435.992f, + * 117435.992f] for sine. + * (2) For elements within range, perform range reduction using Cody-Waite's + * method: x* = x - y*PI/2, where y = rint(x*2/PI). x* \in [-PI/4, PI/4]. + * (3) Map cos(x) to (+/-)sine or (+/-)cosine of x* based on the quadrant k = + * int(y). + * (4) For elements outside that range, Cody-Waite reduction performs poorly + * leading to catastrophic cancellation. We compute cosine by calling glibc in + * a scalar fashion. + * (5) Vectorized implementation has a max ULP of 1.49 and performs at least + * 5-7x(x86) - 2.5-3x(Power) - 1-2x(Arm) faster than scalar implementations + * when magnitude of all elements in the array < 71476.0625f (117435.992f for sine). + * Worst case performance is when all the elements are large leading to about 1-2% reduction in + * performance. + */ +typedef enum +{ + SIMD_COMPUTE_SIN, + SIMD_COMPUTE_COS +} SIMD_TRIG_OP; + +static void SIMD_MSVC_NOINLINE +simd_sincos_f32(const float *src, npy_intp ssrc, float *dst, npy_intp sdst, + npy_intp len, SIMD_TRIG_OP trig_op) +{ + // Load up frequently used constants + const npyv_f32 zerosf = npyv_zero_f32(); + const npyv_s32 ones = npyv_setall_s32(1); + const npyv_s32 twos = npyv_setall_s32(2); + const npyv_f32 two_over_pi = npyv_setall_f32(0x1.45f306p-1f); + const npyv_f32 codyw_pio2_highf = npyv_setall_f32(-0x1.921fb0p+00f); + const npyv_f32 codyw_pio2_medf = npyv_setall_f32(-0x1.5110b4p-22f); + const npyv_f32 codyw_pio2_lowf = npyv_setall_f32(-0x1.846988p-48f); + const npyv_f32 rint_cvt_magic = npyv_setall_f32(0x1.800000p+23f); + // Cody-Waite's range + float max_codi = 117435.992f; + if (trig_op == SIMD_COMPUTE_COS) { + max_codi = 71476.0625f; + } + const npyv_f32 max_cody = npyv_setall_f32(max_codi); + const int vstep = npyv_nlanes_f32; + + for (; len > 0; len -= vstep, src += ssrc*vstep, dst += sdst*vstep) { + npyv_f32 x_in; + if (ssrc == 1) { + x_in = npyv_load_tillz_f32(src, len); + } else { + x_in = npyv_loadn_tillz_f32(src, ssrc, len); + } + npyv_b32 simd_mask = npyv_cmple_f32(npyv_abs_f32(x_in), max_cody); + npy_uint64 simd_maski = npyv_tobits_b32(simd_mask); + /* + * For elements outside of this range, Cody-Waite's range reduction + * becomes inaccurate and we will call libc to compute cosine for + * these numbers + */ + if (simd_maski != 0) { + npyv_b32 nnan_mask = npyv_notnan_f32(x_in); + npyv_f32 x = npyv_select_f32(npyv_and_b32(nnan_mask, simd_mask), x_in, zerosf); + + npyv_f32 quadrant = npyv_mul_f32(x, two_over_pi); + // round to nearest, -0.0f -> +0.0f, and |a| must be <= 0x1.0p+22 + quadrant = npyv_add_f32(quadrant, rint_cvt_magic); + quadrant = npyv_sub_f32(quadrant, rint_cvt_magic); + + // Cody-Waite's range reduction algorithm + npyv_f32 reduced_x = simd_range_reduction_f32( + x, quadrant, codyw_pio2_highf, codyw_pio2_medf, codyw_pio2_lowf + ); + npyv_f32 reduced_x2 = npyv_square_f32(reduced_x); + + // compute cosine and sine + npyv_f32 cos = simd_cosine_poly_f32(reduced_x2); + npyv_f32 sin = simd_sine_poly_f32(reduced_x, reduced_x2); + + npyv_s32 iquadrant = npyv_round_s32_f32(quadrant); + if (trig_op == SIMD_COMPUTE_COS) { + iquadrant = npyv_add_s32(iquadrant, ones); + } + // blend sin and cos based on the quadrant + npyv_b32 sine_mask = npyv_cmpeq_s32(npyv_and_s32(iquadrant, ones), npyv_zero_s32()); + cos = npyv_select_f32(sine_mask, sin, cos); + + // multiply by -1 for appropriate elements + npyv_b32 negate_mask = npyv_cmpeq_s32(npyv_and_s32(iquadrant, twos), twos); + cos = npyv_ifsub_f32(negate_mask, zerosf, cos, cos); + cos = npyv_select_f32(nnan_mask, cos, npyv_setall_f32(NPY_NANF)); + + if (sdst == 1) { + npyv_store_till_f32(dst, len, cos); + } else { + npyv_storen_till_f32(dst, sdst, len, cos); + } + } + if (simd_maski != ((1 << vstep) - 1)) { + float NPY_DECL_ALIGNED(NPY_SIMD_WIDTH) ip_fback[npyv_nlanes_f32]; + npyv_storea_f32(ip_fback, x_in); + + // process elements using libc for large elements + if (trig_op == SIMD_COMPUTE_COS) { + for (unsigned i = 0; i < npyv_nlanes_f32; ++i) { + if ((simd_maski >> i) & 1) { + continue; + } + dst[sdst*i] = npy_cosf(ip_fback[i]); + } + } + else { + for (unsigned i = 0; i < npyv_nlanes_f32; ++i) { + if ((simd_maski >> i) & 1) { + continue; + } + dst[sdst*i] = npy_sinf(ip_fback[i]); + } + } + } + } + npyv_cleanup(); +} +#endif // NPY_SIMD_FMA3 + +/**begin repeat + * #func = cos, sin# + * #enum = SIMD_COMPUTE_COS, SIMD_COMPUTE_SIN# + */ +NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(FLOAT_@func@) +(char **args, npy_intp const *dimensions, npy_intp const *steps, void *NPY_UNUSED(data)) +{ + const float *src = (float*)args[0]; + float *dst = (float*)args[1]; + + const int lsize = sizeof(src[0]); + const npy_intp ssrc = steps[0] / lsize; + const npy_intp sdst = steps[1] / lsize; + npy_intp len = dimensions[0]; + assert(steps[0] % lsize == 0 && steps[1] % lsize == 0); +#if NPY_SIMD_FMA3 + if (is_mem_overlap(src, steps[0], dst, steps[1], len) || + !npyv_loadable_stride_f32(ssrc) || !npyv_storable_stride_f32(sdst) + ) { + for (; len > 0; --len, src += ssrc, dst += sdst) { + simd_sincos_f32(src, 1, dst, 1, 1, @enum@); + } + } else { + simd_sincos_f32(src, ssrc, dst, sdst, len, @enum@); + } +#else + for (; len > 0; --len, src += ssrc, dst += sdst) { + const float src0 = *src; + *dst = npy_@func@f(src0); + } +#endif +} +/**end repeat**/ diff --git a/numpy/core/src/umath/loops_utils.h b/numpy/core/src/umath/loops_utils.h deleted file mode 100644 index f5540bdae2fd..000000000000 --- a/numpy/core/src/umath/loops_utils.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef _NPY_UMATH_LOOPS_UTILS_H_ -#define _NPY_UMATH_LOOPS_UTILS_H_ - -#include "numpy/npy_common.h" // NPY_FINLINE -/* - * nomemoverlap - returns false if two strided arrays have an overlapping - * region in memory. ip_size/op_size = size of the arrays which can be negative - * indicating negative steps. - */ -NPY_FINLINE npy_bool -nomemoverlap(char *ip, npy_intp ip_size, char *op, npy_intp op_size) -{ - char *ip_start, *ip_end, *op_start, *op_end; - if (ip_size < 0) { - ip_start = ip + ip_size; - ip_end = ip; - } - else { - ip_start = ip; - ip_end = ip + ip_size; - } - if (op_size < 0) { - op_start = op + op_size; - op_end = op; - } - else { - op_start = op; - op_end = op + op_size; - } - return (ip_start == op_start && op_end == ip_end) || - (ip_start > op_end) || (op_start > ip_end); -} - -// returns true if two strided arrays have an overlapping region in memory -// same as `nomemoverlap()` but requires array length and step sizes -NPY_FINLINE npy_bool -is_mem_overlap(const void *src, npy_intp src_step, const void *dst, npy_intp dst_step, npy_intp len) -{ - return !(nomemoverlap((char*)src, src_step*len, (char*)dst, dst_step*len)); -} - -#endif // _NPY_UMATH_LOOPS_UTILS_H_ diff --git a/numpy/core/src/umath/loops_utils.h.src b/numpy/core/src/umath/loops_utils.h.src new file mode 100644 index 000000000000..1a2a5a32ba20 --- /dev/null +++ b/numpy/core/src/umath/loops_utils.h.src @@ -0,0 +1,224 @@ +#ifndef _NPY_UMATH_LOOPS_UTILS_H_ +#define _NPY_UMATH_LOOPS_UTILS_H_ + +#include "numpy/npy_common.h" // NPY_FINLINE +#include "numpy/halffloat.h" // npy_half_to_float + +/** + * Old versions of MSVC causes ambiguous link errors when we deal with large SIMD kernels + * which lead to break the build, probably releated to the following bug: + * https://developercommunity.visualstudio.com/content/problem/415095/internal-compiler-error-with-perfectly-forwarded-r.html + */ +#if defined(_MSC_VER) && _MSC_VER < 1916 + #define SIMD_MSVC_NOINLINE __declspec(noinline) +#else + #define SIMD_MSVC_NOINLINE +#endif +/* + * nomemoverlap - returns false if two strided arrays have an overlapping + * region in memory. ip_size/op_size = size of the arrays which can be negative + * indicating negative steps. + */ +NPY_FINLINE npy_bool +nomemoverlap(char *ip, npy_intp ip_size, char *op, npy_intp op_size) +{ + char *ip_start, *ip_end, *op_start, *op_end; + if (ip_size < 0) { + ip_start = ip + ip_size; + ip_end = ip; + } + else { + ip_start = ip; + ip_end = ip + ip_size; + } + if (op_size < 0) { + op_start = op + op_size; + op_end = op; + } + else { + op_start = op; + op_end = op + op_size; + } + return (ip_start == op_start && op_end == ip_end) || + (ip_start > op_end) || (op_start > ip_end); +} + +// returns true if two strided arrays have an overlapping region in memory +// same as `nomemoverlap()` but requires array length and step sizes +NPY_FINLINE npy_bool +is_mem_overlap(const void *src, npy_intp src_step, const void *dst, npy_intp dst_step, npy_intp len) +{ + return !(nomemoverlap((char*)src, src_step*len, (char*)dst, dst_step*len)); +} + +/* + * cutoff blocksize for pairwise summation + * decreasing it decreases errors slightly as more pairs are summed but + * also lowers performance, as the inner loop is unrolled eight times it is + * effectively 16 + */ +#define PW_BLOCKSIZE 128 + +/**begin repeat + * Float types + * #type = npy_float, npy_double, npy_longdouble, npy_float# + * #dtype = npy_float, npy_double, npy_longdouble, npy_half# + * #TYPE = FLOAT, DOUBLE, LONGDOUBLE, HALF# + * #c = f, , l, # + * #C = F, , L, # + * #trf = , , , npy_half_to_float# + */ + +/* + * Pairwise summation, rounding error O(lg n) instead of O(n). + * The recursion depth is O(lg n) as well. + * when updating also update similar complex floats summation + */ +static NPY_INLINE @type@ +@TYPE@_pairwise_sum(char *a, npy_intp n, npy_intp stride) +{ + if (n < 8) { + npy_intp i; + @type@ res = 0.; + + for (i = 0; i < n; i++) { + res += @trf@(*((@dtype@*)(a + i * stride))); + } + return res; + } + else if (n <= PW_BLOCKSIZE) { + npy_intp i; + @type@ r[8], res; + + /* + * sum a block with 8 accumulators + * 8 times unroll reduces blocksize to 16 and allows vectorization with + * avx without changing summation ordering + */ + r[0] = @trf@(*((@dtype@ *)(a + 0 * stride))); + r[1] = @trf@(*((@dtype@ *)(a + 1 * stride))); + r[2] = @trf@(*((@dtype@ *)(a + 2 * stride))); + r[3] = @trf@(*((@dtype@ *)(a + 3 * stride))); + r[4] = @trf@(*((@dtype@ *)(a + 4 * stride))); + r[5] = @trf@(*((@dtype@ *)(a + 5 * stride))); + r[6] = @trf@(*((@dtype@ *)(a + 6 * stride))); + r[7] = @trf@(*((@dtype@ *)(a + 7 * stride))); + + for (i = 8; i < n - (n % 8); i += 8) { + /* small blocksizes seems to mess with hardware prefetch */ + NPY_PREFETCH(a + (i + 512/(npy_intp)sizeof(@dtype@))*stride, 0, 3); + r[0] += @trf@(*((@dtype@ *)(a + (i + 0) * stride))); + r[1] += @trf@(*((@dtype@ *)(a + (i + 1) * stride))); + r[2] += @trf@(*((@dtype@ *)(a + (i + 2) * stride))); + r[3] += @trf@(*((@dtype@ *)(a + (i + 3) * stride))); + r[4] += @trf@(*((@dtype@ *)(a + (i + 4) * stride))); + r[5] += @trf@(*((@dtype@ *)(a + (i + 5) * stride))); + r[6] += @trf@(*((@dtype@ *)(a + (i + 6) * stride))); + r[7] += @trf@(*((@dtype@ *)(a + (i + 7) * stride))); + } + + /* accumulate now to avoid stack spills for single peel loop */ + res = ((r[0] + r[1]) + (r[2] + r[3])) + + ((r[4] + r[5]) + (r[6] + r[7])); + + /* do non multiple of 8 rest */ + for (; i < n; i++) { + res += @trf@(*((@dtype@ *)(a + i * stride))); + } + return res; + } + else { + /* divide by two but avoid non-multiples of unroll factor */ + npy_intp n2 = n / 2; + + n2 -= n2 % 8; + return @TYPE@_pairwise_sum(a, n2, stride) + + @TYPE@_pairwise_sum(a + n2 * stride, n - n2, stride); + } +} + +/**end repeat**/ + +/**begin repeat + * complex types + * #TYPE = CFLOAT, CDOUBLE, CLONGDOUBLE# + * #ftype = npy_float, npy_double, npy_longdouble# + * #c = f, , l# + * #C = F, , L# + * #SIMD = 1, 1, 0# + */ +/* similar to pairwise sum of real floats */ +static NPY_INLINE void +@TYPE@_pairwise_sum(@ftype@ *rr, @ftype@ * ri, char * a, npy_intp n, + npy_intp stride) +{ + assert(n % 2 == 0); + if (n < 8) { + npy_intp i; + + *rr = 0.; + *ri = 0.; + for (i = 0; i < n; i += 2) { + *rr += *((@ftype@ *)(a + i * stride + 0)); + *ri += *((@ftype@ *)(a + i * stride + sizeof(@ftype@))); + } + return; + } + else if (n <= PW_BLOCKSIZE) { + npy_intp i; + @ftype@ r[8]; + + /* + * sum a block with 8 accumulators + * 8 times unroll reduces blocksize to 16 and allows vectorization with + * avx without changing summation ordering + */ + r[0] = *((@ftype@ *)(a + 0 * stride)); + r[1] = *((@ftype@ *)(a + 0 * stride + sizeof(@ftype@))); + r[2] = *((@ftype@ *)(a + 2 * stride)); + r[3] = *((@ftype@ *)(a + 2 * stride + sizeof(@ftype@))); + r[4] = *((@ftype@ *)(a + 4 * stride)); + r[5] = *((@ftype@ *)(a + 4 * stride + sizeof(@ftype@))); + r[6] = *((@ftype@ *)(a + 6 * stride)); + r[7] = *((@ftype@ *)(a + 6 * stride + sizeof(@ftype@))); + + for (i = 8; i < n - (n % 8); i += 8) { + /* small blocksizes seems to mess with hardware prefetch */ + NPY_PREFETCH(a + (i + 512/(npy_intp)sizeof(@ftype@))*stride, 0, 3); + r[0] += *((@ftype@ *)(a + (i + 0) * stride)); + r[1] += *((@ftype@ *)(a + (i + 0) * stride + sizeof(@ftype@))); + r[2] += *((@ftype@ *)(a + (i + 2) * stride)); + r[3] += *((@ftype@ *)(a + (i + 2) * stride + sizeof(@ftype@))); + r[4] += *((@ftype@ *)(a + (i + 4) * stride)); + r[5] += *((@ftype@ *)(a + (i + 4) * stride + sizeof(@ftype@))); + r[6] += *((@ftype@ *)(a + (i + 6) * stride)); + r[7] += *((@ftype@ *)(a + (i + 6) * stride + sizeof(@ftype@))); + } + + /* accumulate now to avoid stack spills for single peel loop */ + *rr = ((r[0] + r[2]) + (r[4] + r[6])); + *ri = ((r[1] + r[3]) + (r[5] + r[7])); + + /* do non multiple of 8 rest */ + for (; i < n; i+=2) { + *rr += *((@ftype@ *)(a + i * stride + 0)); + *ri += *((@ftype@ *)(a + i * stride + sizeof(@ftype@))); + } + return; + } + else { + /* divide by two but avoid non-multiples of unroll factor */ + @ftype@ rr1, ri1, rr2, ri2; + npy_intp n2 = n / 2; + + n2 -= n2 % 8; + @TYPE@_pairwise_sum(&rr1, &ri1, a, n2, stride); + @TYPE@_pairwise_sum(&rr2, &ri2, a + n2 * stride, n - n2, stride); + *rr = rr1 + rr2; + *ri = ri1 + ri2; + return; + } +} +/**end repeat**/ + +#endif // _NPY_UMATH_LOOPS_UTILS_H_ diff --git a/numpy/core/src/umath/npy_simd_data.h b/numpy/core/src/umath/npy_simd_data.h index 45487d0a8553..62438d7a3fa8 100644 --- a/numpy/core/src/umath/npy_simd_data.h +++ b/numpy/core/src/umath/npy_simd_data.h @@ -1,6 +1,6 @@ #ifndef __NPY_SIMD_DATA_H_ #define __NPY_SIMD_DATA_H_ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +#if defined NPY_HAVE_AVX512F #if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) /* * Constants used in vector implementation of float64 exp(x) @@ -119,30 +119,14 @@ static npy_uint64 EXP_Table_tail[32] = { #define NPY_COEFF_Q3_LOGf 9.864942958519418960339e-01f #define NPY_COEFF_Q4_LOGf 1.546476374983906719538e-01f #define NPY_COEFF_Q5_LOGf 5.875095403124574342950e-03f -/* - * Constants used in vector implementation of sinf/cosf(x) - */ -#define NPY_TWO_O_PIf 0x1.45f306p-1f -#define NPY_CODY_WAITE_PI_O_2_HIGHf -0x1.921fb0p+00f -#define NPY_CODY_WAITE_PI_O_2_MEDf -0x1.5110b4p-22f -#define NPY_CODY_WAITE_PI_O_2_LOWf -0x1.846988p-48f -#define NPY_COEFF_INVF0_COSINEf 0x1.000000p+00f -#define NPY_COEFF_INVF2_COSINEf -0x1.000000p-01f -#define NPY_COEFF_INVF4_COSINEf 0x1.55553cp-05f -#define NPY_COEFF_INVF6_COSINEf -0x1.6c06dcp-10f -#define NPY_COEFF_INVF8_COSINEf 0x1.98e616p-16f -#define NPY_COEFF_INVF3_SINEf -0x1.555556p-03f -#define NPY_COEFF_INVF5_SINEf 0x1.11119ap-07f -#define NPY_COEFF_INVF7_SINEf -0x1.a06bbap-13f -#define NPY_COEFF_INVF9_SINEf 0x1.7d3bbcp-19f /* * Lookup table of log(c_k) - * Reference form: Tang, Ping-Tak Peter. "Table-driven implementation of the - * logarithm function in IEEE floating-point arithmetic." ACM Transactions + * Reference form: Tang, Ping-Tak Peter. "Table-driven implementation of the + * logarithm function in IEEE floating-point arithmetic." ACM Transactions * on Mathematical Software (TOMS) 16.4 (1990): 378-400. */ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +#if defined NPY_HAVE_AVX512F #if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) static npy_uint64 LOG_TABLE_TOP[64] = { 0x0000000000000000, diff --git a/numpy/core/src/umath/override.c b/numpy/core/src/umath/override.c index a0090e302324..d247c263986f 100644 --- a/numpy/core/src/umath/override.c +++ b/numpy/core/src/umath/override.c @@ -8,6 +8,7 @@ #include "override.h" #include "ufunc_override.h" + /* * For each positional argument and each argument in a possible "out" * keyword, look for overrides of the standard ufunc behaviour, i.e., @@ -22,25 +23,16 @@ * Returns -1 on failure. */ static int -get_array_ufunc_overrides(PyObject *args, PyObject *kwds, +get_array_ufunc_overrides(PyObject *in_args, PyObject *out_args, PyObject **with_override, PyObject **methods) { int i; int num_override_args = 0; - int narg, nout = 0; - PyObject *out_kwd_obj; - PyObject **arg_objs, **out_objs; + int narg, nout; - narg = PyTuple_Size(args); - if (narg < 0) { - return -1; - } - arg_objs = PySequence_Fast_ITEMS(args); - - nout = PyUFuncOverride_GetOutObjects(kwds, &out_kwd_obj, &out_objs); - if (nout < 0) { - return -1; - } + narg = (int)PyTuple_GET_SIZE(in_args); + /* It is valid for out_args to be NULL: */ + nout = (out_args != NULL) ? (int)PyTuple_GET_SIZE(out_args) : 0; for (i = 0; i < narg + nout; ++i) { PyObject *obj; @@ -48,10 +40,10 @@ get_array_ufunc_overrides(PyObject *args, PyObject *kwds, int new_class = 1; if (i < narg) { - obj = arg_objs[i]; + obj = PyTuple_GET_ITEM(in_args, i); } else { - obj = out_objs[i - narg]; + obj = PyTuple_GET_ITEM(out_args, i - narg); } /* * Have we seen this class before? If so, ignore. @@ -86,7 +78,6 @@ get_array_ufunc_overrides(PyObject *args, PyObject *kwds, ++num_override_args; } } - Py_DECREF(out_kwd_obj); return num_override_args; fail: @@ -94,359 +85,116 @@ get_array_ufunc_overrides(PyObject *args, PyObject *kwds, Py_DECREF(with_override[i]); Py_DECREF(methods[i]); } - Py_DECREF(out_kwd_obj); return -1; } -/* - * The following functions normalize ufunc arguments. The work done is similar - * to what is done inside ufunc_object by get_ufunc_arguments for __call__ and - * generalized ufuncs, and by PyUFunc_GenericReduction for the other methods. - * It would be good to unify (see gh-8892). - */ /* - * ufunc() and ufunc.outer() accept 'sig' or 'signature'; - * normalize to 'signature' + * Build a dictionary from the keyword arguments, but replace out with the + * normalized version (and always pass it even if it was passed by position). */ static int -normalize_signature_keyword(PyObject *normal_kwds) -{ - PyObject *obj = _PyDict_GetItemStringWithError(normal_kwds, "sig"); - if (obj == NULL && PyErr_Occurred()){ - return -1; - } - if (obj != NULL) { - PyObject *sig = _PyDict_GetItemStringWithError(normal_kwds, "signature"); - if (sig == NULL && PyErr_Occurred()) { - return -1; - } - if (sig) { - PyErr_SetString(PyExc_TypeError, - "cannot specify both 'sig' and 'signature'"); - return -1; - } - /* - * No INCREF or DECREF needed: got a borrowed reference above, - * and, unlike e.g. PyList_SetItem, PyDict_SetItem INCREF's it. - */ - PyDict_SetItemString(normal_kwds, "signature", obj); - PyDict_DelItemString(normal_kwds, "sig"); - } - return 0; -} - -static int -normalize___call___args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) +initialize_normal_kwds(PyObject *out_args, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, + PyObject *normal_kwds) { - /* - * ufunc.__call__(*args, **kwds) - */ - npy_intp i; - int not_all_none; - npy_intp nin = ufunc->nin; - npy_intp nout = ufunc->nout; - npy_intp nargs = PyTuple_GET_SIZE(args); - npy_intp nkwds = PyDict_Size(*normal_kwds); - PyObject *obj; - - if (nargs < nin) { - PyErr_Format(PyExc_TypeError, - "ufunc() missing %"NPY_INTP_FMT" of %"NPY_INTP_FMT - "required positional argument(s)", nin - nargs, nin); - return -1; - } - if (nargs > nin+nout) { - PyErr_Format(PyExc_TypeError, - "ufunc() takes from %"NPY_INTP_FMT" to %"NPY_INTP_FMT - "arguments but %"NPY_INTP_FMT" were given", - nin, nin+nout, nargs); - return -1; - } - - *normal_args = PyTuple_GetSlice(args, 0, nin); - if (*normal_args == NULL) { - return -1; - } - - /* If we have more args than nin, they must be the output variables.*/ - if (nargs > nin) { - if (nkwds > 0) { - PyObject *out_kwd = _PyDict_GetItemStringWithError(*normal_kwds, "out"); - if (out_kwd == NULL && PyErr_Occurred()) { - return -1; - } - else if (out_kwd) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('out') and position " - "(%"NPY_INTP_FMT")", nin); + if (kwnames != NULL) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(kwnames); i++) { + if (PyDict_SetItem(normal_kwds, + PyTuple_GET_ITEM(kwnames, i), args[i + len_args]) < 0) { return -1; } } - for (i = nin; i < nargs; i++) { - not_all_none = (PyTuple_GET_ITEM(args, i) != Py_None); - if (not_all_none) { - break; - } - } - if (not_all_none) { - if (nargs - nin == nout) { - obj = PyTuple_GetSlice(args, nin, nargs); - } - else { - PyObject *item; - - obj = PyTuple_New(nout); - if (obj == NULL) { - return -1; - } - for (i = 0; i < nout; i++) { - if (i + nin < nargs) { - item = PyTuple_GET_ITEM(args, nin+i); - } - else { - item = Py_None; - } - Py_INCREF(item); - PyTuple_SET_ITEM(obj, i, item); - } - } - PyDict_SetItemString(*normal_kwds, "out", obj); - Py_DECREF(obj); - } } - /* gufuncs accept either 'axes' or 'axis', but not both */ - if (nkwds >= 2) { - PyObject *axis_kwd = _PyDict_GetItemStringWithError(*normal_kwds, "axis"); - if (axis_kwd == NULL && PyErr_Occurred()) { + static PyObject *out_str = NULL; + if (out_str == NULL) { + out_str = PyUnicode_InternFromString("out"); + if (out_str == NULL) { return -1; } - PyObject *axes_kwd = _PyDict_GetItemStringWithError(*normal_kwds, "axes"); - if (axes_kwd == NULL && PyErr_Occurred()) { - return -1; - } - if (axis_kwd && axes_kwd) { - PyErr_SetString(PyExc_TypeError, - "cannot specify both 'axis' and 'axes'"); - return -1; - } - } - /* finally, ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - return nkwds == 0 ? 0 : normalize_signature_keyword(*normal_kwds); -} - -static int -normalize_reduce_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* - * ufunc.reduce(a[, axis, dtype, out, keepdims]) - */ - npy_intp nargs = PyTuple_GET_SIZE(args); - npy_intp i; - PyObject *obj; - static PyObject *NoValue = NULL; - static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims", - "initial", "where"}; - - npy_cache_import("numpy", "_NoValue", &NoValue); - if (NoValue == NULL) return -1; - - if (nargs < 1 || nargs > 7) { - PyErr_Format(PyExc_TypeError, - "ufunc.reduce() takes from 1 to 7 positional " - "arguments but %"NPY_INTP_FMT" were given", nargs); - return -1; - } - *normal_args = PyTuple_GetSlice(args, 0, 1); - if (*normal_args == NULL) { - return -1; } - for (i = 1; i < nargs; i++) { - PyObject *kwd = _PyDict_GetItemStringWithError(*normal_kwds, kwlist[i]); - if (kwd == NULL && PyErr_Occurred()) { + if (out_args != NULL) { + /* Replace `out` argument with the normalized version */ + int res = PyDict_SetItem(normal_kwds, out_str, out_args); + if (res < 0) { return -1; } - else if (kwd) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('%s') and position " - "(%"NPY_INTP_FMT")", kwlist[i], i); + } + else { + /* Ensure that `out` is not present. */ + int res = PyDict_Contains(normal_kwds, out_str); + if (res < 0) { return -1; } - obj = PyTuple_GET_ITEM(args, i); - if (i == 3) { - /* remove out=None */ - if (obj == Py_None) { - continue; - } - obj = PyTuple_GetSlice(args, 3, 4); - } - /* Remove initial=np._NoValue */ - if (i == 5 && obj == NoValue) { - continue; - } - PyDict_SetItemString(*normal_kwds, kwlist[i], obj); - if (i == 3) { - Py_DECREF(obj); + if (res) { + return PyDict_DelItem(normal_kwds, out_str); } } return 0; } +/* + * ufunc() and ufunc.outer() accept 'sig' or 'signature'. We guarantee + * that it is passed as 'signature' by renaming 'sig' if present. + * Note that we have already validated that only one of them was passed + * before checking for overrides. + */ static int -normalize_accumulate_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) +normalize_signature_keyword(PyObject *normal_kwds) { - /* - * ufunc.accumulate(a[, axis, dtype, out]) - */ - npy_intp nargs = PyTuple_GET_SIZE(args); - npy_intp i; - PyObject *obj; - static char *kwlist[] = {"array", "axis", "dtype", "out", "keepdims"}; - - if (nargs < 1 || nargs > 4) { - PyErr_Format(PyExc_TypeError, - "ufunc.accumulate() takes from 1 to 4 positional " - "arguments but %"NPY_INTP_FMT" were given", nargs); - return -1; - } - *normal_args = PyTuple_GetSlice(args, 0, 1); - if (*normal_args == NULL) { + /* If the keywords include `sig` rename to `signature`. */ + PyObject* obj = _PyDict_GetItemStringWithError(normal_kwds, "sig"); + if (obj == NULL && PyErr_Occurred()) { return -1; } - - for (i = 1; i < nargs; i++) { - PyObject *kwd = _PyDict_GetItemStringWithError(*normal_kwds, kwlist[i]); - if (kwd == NULL && PyErr_Occurred()) { + if (obj != NULL) { + /* + * No INCREF or DECREF needed: got a borrowed reference above, + * and, unlike e.g. PyList_SetItem, PyDict_SetItem INCREF's it. + */ + if (PyDict_SetItemString(normal_kwds, "signature", obj) < 0) { return -1; } - else if (kwd) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('%s') and position " - "(%"NPY_INTP_FMT")", kwlist[i], i); + if (PyDict_DelItemString(normal_kwds, "sig") < 0) { return -1; } - obj = PyTuple_GET_ITEM(args, i); - if (i == 3) { - /* remove out=None */ - if (obj == Py_None) { - continue; - } - obj = PyTuple_GetSlice(args, 3, 4); - } - PyDict_SetItemString(*normal_kwds, kwlist[i], obj); - if (i == 3) { - Py_DECREF(obj); - } } return 0; } + static int -normalize_reduceat_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) +copy_positional_args_to_kwargs(const char **keywords, + PyObject *const *args, Py_ssize_t len_args, + PyObject *normal_kwds) { - /* - * ufunc.reduceat(a, indices[, axis, dtype, out]) - * the number of arguments has been checked in PyUFunc_GenericReduction. - */ - npy_intp i; - npy_intp nargs = PyTuple_GET_SIZE(args); - PyObject *obj; - static char *kwlist[] = {"array", "indices", "axis", "dtype", "out"}; - - if (nargs < 2 || nargs > 5) { - PyErr_Format(PyExc_TypeError, - "ufunc.reduceat() takes from 2 to 4 positional " - "arguments but %"NPY_INTP_FMT" were given", nargs); - return -1; - } - /* a and indices */ - *normal_args = PyTuple_GetSlice(args, 0, 2); - if (*normal_args == NULL) { - return -1; - } - - for (i = 2; i < nargs; i++) { - PyObject *kwd = _PyDict_GetItemStringWithError(*normal_kwds, kwlist[i]); - if (kwd == NULL && PyErr_Occurred()) { - return -1; - } - else if (kwd) { - PyErr_Format(PyExc_TypeError, - "argument given by name ('%s') and position " - "(%"NPY_INTP_FMT")", kwlist[i], i); - return -1; + for (Py_ssize_t i = 0; i < len_args; i++) { + if (keywords[i] == NULL) { + /* keyword argument is either input or output and not set here */ + continue; } - obj = PyTuple_GET_ITEM(args, i); - if (i == 4) { - /* remove out=None */ - if (obj == Py_None) { + if (NPY_UNLIKELY(i == 5)) { + /* + * This is only relevant for reduce, which is the only one with + * 5 keyword arguments. + */ + static PyObject *NoValue = NULL; + assert(strcmp(keywords[i], "initial") == 0); + npy_cache_import("numpy", "_NoValue", &NoValue); + if (args[i] == NoValue) { continue; } - obj = PyTuple_GetSlice(args, 4, 5); } - PyDict_SetItemString(*normal_kwds, kwlist[i], obj); - if (i == 4) { - Py_DECREF(obj); + + int res = PyDict_SetItemString(normal_kwds, keywords[i], args[i]); + if (res < 0) { + return -1; } } return 0; } -static int -normalize_outer_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* - * ufunc.outer(*args, **kwds) - * all positional arguments should be inputs. - * for the keywords, we only need to check 'sig' vs 'signature'. - */ - npy_intp nin = ufunc->nin; - npy_intp nargs = PyTuple_GET_SIZE(args); - - if (nargs < nin) { - PyErr_Format(PyExc_TypeError, - "ufunc.outer() missing %"NPY_INTP_FMT" of %"NPY_INTP_FMT - "required positional " "argument(s)", nin - nargs, nin); - return -1; - } - if (nargs > nin) { - PyErr_Format(PyExc_TypeError, - "ufunc.outer() takes %"NPY_INTP_FMT" arguments but" - "%"NPY_INTP_FMT" were given", nin, nargs); - return -1; - } - - *normal_args = PyTuple_GetSlice(args, 0, nin); - if (*normal_args == NULL) { - return -1; - } - /* ufuncs accept 'sig' or 'signature' normalize to 'signature' */ - return normalize_signature_keyword(*normal_kwds); -} - -static int -normalize_at_args(PyUFuncObject *ufunc, PyObject *args, - PyObject **normal_args, PyObject **normal_kwds) -{ - /* ufunc.at(a, indices[, b]) */ - npy_intp nargs = PyTuple_GET_SIZE(args); - - if (nargs < 2 || nargs > 3) { - PyErr_Format(PyExc_TypeError, - "ufunc.at() takes from 2 to 3 positional " - "arguments but %"NPY_INTP_FMT" were given", nargs); - return -1; - } - *normal_args = PyTuple_GetSlice(args, 0, nargs); - return (*normal_args == NULL); -} - /* * Check a set of args for the `__array_ufunc__` method. If more than one of * the input arguments implements `__array_ufunc__`, they are tried in the @@ -460,31 +208,26 @@ normalize_at_args(PyUFuncObject *ufunc, PyObject *args, */ NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result) + PyObject *in_args, PyObject *out_args, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, + PyObject **result) { - int i; - int j; int status; int num_override_args; PyObject *with_override[NPY_MAXARGS]; PyObject *array_ufunc_methods[NPY_MAXARGS]; - PyObject *out; - PyObject *method_name = NULL; - PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ PyObject *normal_kwds = NULL; PyObject *override_args = NULL; - Py_ssize_t len; /* * Check inputs for overrides */ num_override_args = get_array_ufunc_overrides( - args, kwds, with_override, array_ufunc_methods); + in_args, out_args, with_override, array_ufunc_methods); if (num_override_args == -1) { goto fail; } @@ -495,104 +238,58 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, } /* - * Normalize ufunc arguments. + * Normalize ufunc arguments, note that any input and output arguments + * have already been stored in `in_args` and `out_args`. */ - - /* Build new kwds */ - if (kwds && PyDict_CheckExact(kwds)) { - - /* ensure out is always a tuple */ - normal_kwds = PyDict_Copy(kwds); - out = _PyDict_GetItemStringWithError(normal_kwds, "out"); - if (out == NULL && PyErr_Occurred()) { - goto fail; - } - else if (out) { - int nout = ufunc->nout; - - if (PyTuple_CheckExact(out)) { - int all_none = 1; - - if (PyTuple_GET_SIZE(out) != nout) { - PyErr_Format(PyExc_ValueError, - "The 'out' tuple must have exactly " - "%d entries: one per ufunc output", nout); - goto fail; - } - for (i = 0; i < PyTuple_GET_SIZE(out); i++) { - all_none = (PyTuple_GET_ITEM(out, i) == Py_None); - if (!all_none) { - break; - } - } - if (all_none) { - PyDict_DelItemString(normal_kwds, "out"); - } - } - else { - /* not a tuple */ - if (nout > 1) { - PyErr_SetString(PyExc_TypeError, - "'out' must be a tuple of arguments"); - goto fail; - } - if (out != Py_None) { - /* not already a tuple and not None */ - PyObject *out_tuple = PyTuple_New(1); - - if (out_tuple == NULL) { - goto fail; - } - /* out was borrowed ref; make it permanent */ - Py_INCREF(out); - /* steals reference */ - PyTuple_SET_ITEM(out_tuple, 0, out); - PyDict_SetItemString(normal_kwds, "out", out_tuple); - Py_DECREF(out_tuple); - } - else { - /* out=None; remove it */ - PyDict_DelItemString(normal_kwds, "out"); - } - } - } - } - else { - normal_kwds = PyDict_New(); - } + normal_kwds = PyDict_New(); if (normal_kwds == NULL) { goto fail; } + if (initialize_normal_kwds(out_args, + args, len_args, kwnames, normal_kwds) < 0) { + goto fail; + } - /* decide what to do based on the method. */ + /* + * Reduce-like methods can pass keyword arguments also by position, + * in which case the additional positional arguments have to be copied + * into the keyword argument dictionary. The `__call__` and `__outer__` + * method have to normalize sig and signature. + */ /* ufunc.__call__ */ if (strcmp(method, "__call__") == 0) { - status = normalize___call___args(ufunc, args, &normal_args, - &normal_kwds); + status = normalize_signature_keyword(normal_kwds); } /* ufunc.reduce */ else if (strcmp(method, "reduce") == 0) { - status = normalize_reduce_args(ufunc, args, &normal_args, - &normal_kwds); + static const char *keywords[] = { + NULL, "axis", "dtype", NULL, "keepdims", + "initial", "where"}; + status = copy_positional_args_to_kwargs(keywords, + args, len_args, normal_kwds); } /* ufunc.accumulate */ else if (strcmp(method, "accumulate") == 0) { - status = normalize_accumulate_args(ufunc, args, &normal_args, - &normal_kwds); + static const char *keywords[] = { + NULL, "axis", "dtype", NULL}; + status = copy_positional_args_to_kwargs(keywords, + args, len_args, normal_kwds); } /* ufunc.reduceat */ else if (strcmp(method, "reduceat") == 0) { - status = normalize_reduceat_args(ufunc, args, &normal_args, - &normal_kwds); + static const char *keywords[] = { + NULL, NULL, "axis", "dtype", NULL}; + status = copy_positional_args_to_kwargs(keywords, + args, len_args, normal_kwds); } - /* ufunc.outer */ + /* ufunc.outer (identical to call) */ else if (strcmp(method, "outer") == 0) { - status = normalize_outer_args(ufunc, args, &normal_args, &normal_kwds); + status = normalize_signature_keyword(normal_kwds); } /* ufunc.at */ else if (strcmp(method, "at") == 0) { - status = normalize_at_args(ufunc, args, &normal_args, &normal_kwds); + status = 0; } /* unknown method */ else { @@ -610,7 +307,7 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, goto fail; } - len = PyTuple_GET_SIZE(normal_args); + int len = (int)PyTuple_GET_SIZE(in_args); /* Call __array_ufunc__ functions in correct order */ while (1) { @@ -621,14 +318,14 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, *result = NULL; /* Choose an overriding argument */ - for (i = 0; i < num_override_args; i++) { + for (int i = 0; i < num_override_args; i++) { override_obj = with_override[i]; if (override_obj == NULL) { continue; } /* Check for sub-types to the right of obj. */ - for (j = i + 1; j < num_override_args; j++) { + for (int j = i + 1; j < num_override_args; j++) { PyObject *other_obj = with_override[j]; if (other_obj != NULL && Py_TYPE(other_obj) != Py_TYPE(override_obj) && @@ -662,8 +359,8 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, PyTuple_SET_ITEM(override_args, 1, (PyObject *)ufunc); Py_INCREF(method_name); PyTuple_SET_ITEM(override_args, 2, method_name); - for (i = 0; i < len; i++) { - PyObject *item = PyTuple_GET_ITEM(normal_args, i); + for (int i = 0; i < len; i++) { + PyObject *item = PyTuple_GET_ITEM(in_args, i); Py_INCREF(item); PyTuple_SET_ITEM(override_args, i + 3, item); @@ -724,11 +421,10 @@ PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, fail: status = -1; cleanup: - for (i = 0; i < num_override_args; i++) { + for (int i = 0; i < num_override_args; i++) { Py_XDECREF(with_override[i]); Py_XDECREF(array_ufunc_methods[i]); } - Py_XDECREF(normal_args); Py_XDECREF(method_name); Py_XDECREF(normal_kwds); return status; diff --git a/numpy/core/src/umath/override.h b/numpy/core/src/umath/override.h index 68f3c6ef0814..4e9a323ca629 100644 --- a/numpy/core/src/umath/override.h +++ b/numpy/core/src/umath/override.h @@ -6,6 +6,9 @@ NPY_NO_EXPORT int PyUFunc_CheckOverride(PyUFuncObject *ufunc, char *method, - PyObject *args, PyObject *kwds, - PyObject **result); + PyObject *in_args, PyObject *out_args, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, + PyObject **result); + + #endif diff --git a/numpy/core/src/umath/scalarmath.c.src b/numpy/core/src/umath/scalarmath.c.src index 86dade0f180c..66f97a831431 100644 --- a/numpy/core/src/umath/scalarmath.c.src +++ b/numpy/core/src/umath/scalarmath.c.src @@ -737,6 +737,9 @@ _@name@_convert2_to_ctypes(PyObject *a, @type@ *arg1, { int ret; ret = _@name@_convert_to_ctype(a, arg1); + if (ret == -2) { + ret = -3; + } if (ret < 0) { return ret; } @@ -1029,7 +1032,7 @@ static PyObject * /**begin repeat * - * #name = cfloat, cdouble, clongdouble# + * #name = cfloat, cdouble# * */ @@ -1045,6 +1048,60 @@ static PyObject * /**end repeat**/ +/**begin repeat + * + * #oper = divmod, remainder# + * + */ + +/* +Complex numbers do not support remainder operations. Unfortunately, +the type inference for long doubles is complicated, and if a remainder +operation is not defined - if the relevant field is left NULL - then +operations between long doubles and objects lead to an infinite recursion +instead of a TypeError. This should ensure that once everything gets +converted to complex long doubles you correctly get a reasonably +informative TypeError. This fixes the last part of bug gh-18548. +*/ + +static PyObject * +clongdouble_@oper@(PyObject *a, PyObject *b) +{ + npy_clongdouble arg1, arg2; + + BINOP_GIVE_UP_IF_NEEDED(a, b, nb_@oper@, clongdouble_@oper@); + + switch(_clongdouble_convert2_to_ctypes(a, &arg1, b, &arg2)) { + case 0: + break; + case -1: + /* one of them can't be cast safely must be mixed-types*/ + return PyArray_Type.tp_as_number->nb_@oper@(a,b); + case -2: + /* use default handling */ + if (PyErr_Occurred()) { + return NULL; + } + return PyGenericArrType_Type.tp_as_number->nb_@oper@(a,b); + case -3: + /* + * special case for longdouble and clongdouble + * because they have a recursive getitem in their dtype + */ + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + /* + * here we do the actual calculation with arg1 and arg2 + * as a function call. + */ + PyErr_SetString(PyExc_TypeError, "complex long doubles do not support remainder"); + return NULL; +} + +/**end repeat**/ + /**begin repeat * * #name = half, float, double, longdouble, cfloat, cdouble, clongdouble# diff --git a/numpy/core/src/umath/simd.inc.src b/numpy/core/src/umath/simd.inc.src index 71ee7e07ee70..1a345b1fbaec 100644 --- a/numpy/core/src/umath/simd.inc.src +++ b/numpy/core/src/umath/simd.inc.src @@ -28,7 +28,6 @@ #undef __AVX512F__ #endif #endif -#include "simd/simd.h" #include "loops_utils.h" // nomemoverlap #include #include @@ -37,134 +36,6 @@ #define VECTOR_SIZE_BYTES 16 -/* - * MAX_STEP_SIZE is used to determine if we need to use SIMD version of the ufunc. - * Very large step size can be as slow as processing it using scalar. The - * value of 2097152 ( = 2MB) was chosen using 2 considerations: - * 1) Typical linux kernel page size is 4Kb, but sometimes it could also be 2MB - * which is == 2097152 Bytes. For a step size as large as this, surely all - * the loads/stores of gather/scatter instructions falls on 16 different pages - * which one would think would slow down gather/scatter instructions. - * 2) It additionally satisfies MAX_STEP_SIZE*16/esize < NPY_MAX_INT32 which - * allows us to use i32 version of gather/scatter (as opposed to the i64 version) - * without problems (step larger than NPY_MAX_INT32*esize/16 would require use of - * i64gather/scatter). esize = element size = 4/8 bytes for float/double. - */ -#define MAX_STEP_SIZE 2097152 - -#define IS_BINARY_STRIDE_ONE(esize, vsize) \ - ((steps[0] == esize) && \ - (steps[1] == esize) && \ - (steps[2] == esize) && \ - (abs_ptrdiff(args[2], args[0]) >= vsize) && \ - (abs_ptrdiff(args[2], args[1]) >= vsize)) - -/* - * stride is equal to element size and input and destination are equal or - * don't overlap within one register. The check of the steps against - * esize also quarantees that steps are >= 0. - */ -#define IS_BLOCKABLE_UNARY(esize, vsize) \ - (steps[0] == (esize) && steps[0] == steps[1] && \ - (npy_is_aligned(args[0], esize) && npy_is_aligned(args[1], esize)) && \ - ((abs_ptrdiff(args[1], args[0]) >= (vsize)) || \ - ((abs_ptrdiff(args[1], args[0]) == 0)))) - -/* - * Avoid using SIMD for very large step sizes for several reasons: - * 1) Supporting large step sizes requires use of i64gather/scatter_ps instructions, - * in which case we need two i64gather instructions and an additional vinsertf32x8 - * instruction to load a single zmm register (since one i64gather instruction - * loads into a ymm register). This is not ideal for performance. - * 2) Gather and scatter instructions can be slow when the loads/stores - * cross page boundaries. - * - * We instead rely on i32gather/scatter_ps instructions which use a 32-bit index - * element. The index needs to be < INT_MAX to avoid overflow. MAX_STEP_SIZE - * ensures this. The condition also requires that the input and output arrays - * should have no overlap in memory. - */ -#define IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP \ - ((labs(steps[0]) < MAX_STEP_SIZE) && \ - (labs(steps[1]) < MAX_STEP_SIZE) && \ - (labs(steps[2]) < MAX_STEP_SIZE) && \ - (nomemoverlap(args[0], steps[0] * dimensions[0], args[2], steps[2] * dimensions[0])) && \ - (nomemoverlap(args[1], steps[1] * dimensions[0], args[2], steps[2] * dimensions[0]))) - -#define IS_UNARY_TWO_OUT_SMALL_STEPS_AND_NOMEMOVERLAP \ - ((labs(steps[0]) < MAX_STEP_SIZE) && \ - (labs(steps[1]) < MAX_STEP_SIZE) && \ - (labs(steps[2]) < MAX_STEP_SIZE) && \ - (nomemoverlap(args[0], steps[0] * dimensions[0], args[2], steps[2] * dimensions[0])) && \ - (nomemoverlap(args[0], steps[0] * dimensions[0], args[1], steps[1] * dimensions[0]))) - -/* - * 1) Output should be contiguous, can handle strided input data - * 2) Input step should be smaller than MAX_STEP_SIZE for performance - * 3) Input and output arrays should have no overlap in memory - */ -#define IS_OUTPUT_BLOCKABLE_UNARY(esizein, esizeout, vsize) \ - ((steps[0] & (esizein-1)) == 0 && \ - steps[1] == (esizeout) && labs(steps[0]) < MAX_STEP_SIZE && \ - (nomemoverlap(args[1], steps[1] * dimensions[0], args[0], steps[0] * dimensions[0]))) - -#define IS_BLOCKABLE_REDUCE(esize, vsize) \ - (steps[1] == (esize) && abs_ptrdiff(args[1], args[0]) >= (vsize) && \ - npy_is_aligned(args[1], (esize)) && \ - npy_is_aligned(args[0], (esize))) - -#define IS_BLOCKABLE_BINARY(esize, vsize) \ - (steps[0] == steps[1] && steps[1] == steps[2] && steps[2] == (esize) && \ - npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[1], (esize)) && \ - npy_is_aligned(args[0], (esize)) && \ - (abs_ptrdiff(args[2], args[0]) >= (vsize) || \ - abs_ptrdiff(args[2], args[0]) == 0) && \ - (abs_ptrdiff(args[2], args[1]) >= (vsize) || \ - abs_ptrdiff(args[2], args[1]) >= 0)) - -#define IS_BLOCKABLE_BINARY_SCALAR1(esize, vsize) \ - (steps[0] == 0 && steps[1] == steps[2] && steps[2] == (esize) && \ - npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[1], (esize)) && \ - ((abs_ptrdiff(args[2], args[1]) >= (vsize)) || \ - (abs_ptrdiff(args[2], args[1]) == 0)) && \ - abs_ptrdiff(args[2], args[0]) >= (esize)) - -#define IS_BLOCKABLE_BINARY_SCALAR2(esize, vsize) \ - (steps[1] == 0 && steps[0] == steps[2] && steps[2] == (esize) && \ - npy_is_aligned(args[2], (esize)) && npy_is_aligned(args[0], (esize)) && \ - ((abs_ptrdiff(args[2], args[0]) >= (vsize)) || \ - (abs_ptrdiff(args[2], args[0]) == 0)) && \ - abs_ptrdiff(args[2], args[1]) >= (esize)) - -#undef abs_ptrdiff - -#define IS_BLOCKABLE_BINARY_BOOL(esize, vsize) \ - (steps[0] == (esize) && steps[0] == steps[1] && steps[2] == (1) && \ - npy_is_aligned(args[1], (esize)) && \ - npy_is_aligned(args[0], (esize))) - -#define IS_BLOCKABLE_BINARY_SCALAR1_BOOL(esize, vsize) \ - (steps[0] == 0 && steps[1] == (esize) && steps[2] == (1) && \ - npy_is_aligned(args[1], (esize))) - -#define IS_BLOCKABLE_BINARY_SCALAR2_BOOL(esize, vsize) \ - (steps[0] == (esize) && steps[1] == 0 && steps[2] == (1) && \ - npy_is_aligned(args[0], (esize))) - -/* align var to alignment */ -#define LOOP_BLOCK_ALIGN_VAR(var, type, alignment)\ - npy_intp i, peel = npy_aligned_block_offset(var, sizeof(type),\ - alignment, n);\ - for(i = 0; i < peel; i++) - -#define LOOP_BLOCKED(type, vsize)\ - for(; i < npy_blocked_end(peel, sizeof(type), vsize, n);\ - i += (vsize / sizeof(type))) - -#define LOOP_BLOCKED_END\ - for (; i < n; i++) - - /* * Dispatcher functions * decide whether the operation can be vectorized and run it @@ -183,31 +54,6 @@ * #esize = 8, 16# */ -/**begin repeat1 - * #func = add, subtract, multiply# - */ - -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE NPY_GCC_TARGET_AVX512F void -AVX512F_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps); -#endif - -static NPY_INLINE int -run_binary_avx512f_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) -{ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS - if (IS_BINARY_STRIDE_ONE(@esize@, 64)) { - AVX512F_@func@_@TYPE@(args, dimensions, steps); - return 1; - } - else - return 0; -#endif - return 0; -} - -/**end repeat1**/ - /**begin repeat1 * #func = square, absolute, conjugate# * #outsize = 1, 2, 1# @@ -273,42 +119,6 @@ run_binary_avx512f_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_in /**end repeat1**/ - -#if defined HAVE_ATTRIBUTE_TARGET_AVX512_SKX_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS && @EXISTS@ -static NPY_INLINE NPY_GCC_TARGET_AVX512_SKX void -AVX512_SKX_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps); - -static NPY_INLINE NPY_GCC_TARGET_AVX512_SKX void -AVX512_SKX_frexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps); -#endif - -static NPY_INLINE int -run_binary_avx512_skx_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512_SKX_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS && @EXISTS@ - if (IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP) { - AVX512_SKX_ldexp_@TYPE@(args, dimensions, steps); - return 1; - } - else - return 0; -#endif - return 0; -} - -static NPY_INLINE int -run_unary_two_out_avx512_skx_frexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512_SKX_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS && @EXISTS@ - if (IS_UNARY_TWO_OUT_SMALL_STEPS_AND_NOMEMOVERLAP) { - AVX512_SKX_frexp_@TYPE@(args, dimensions, steps); - return 1; - } - else - return 0; -#endif - return 0; -} /**end repeat**/ /**begin repeat @@ -384,93 +194,8 @@ run_unary_@isa@_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp /**end repeat2**/ /**end repeat1**/ - -/**begin repeat1 - * #func = exp, log# - */ - -#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE void -@ISA@_@func@_FLOAT(npy_float *, npy_float *, const npy_intp n, const npy_intp stride); -#endif - -static NPY_INLINE int -run_unary_@isa@_@func@_FLOAT(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ -#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS - if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_float), sizeof(npy_float), @REGISTER_SIZE@)) { - @ISA@_@func@_FLOAT((npy_float*)args[1], (npy_float*)args[0], dimensions[0], steps[0]); - return 1; - } - else - return 0; -#endif - return 0; -} - -/**end repeat1**/ - -#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE void -@ISA@_sincos_FLOAT(npy_float *, npy_float *, const npy_intp n, const npy_intp steps, NPY_TRIG_OP); -#endif - -static NPY_INLINE int -run_unary_@isa@_sincos_FLOAT(char **args, npy_intp const *dimensions, npy_intp const *steps, NPY_TRIG_OP my_trig_op) -{ -#if defined @CHK@ && defined NPY_HAVE_SSE2_INTRINSICS - if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_float), sizeof(npy_float), @REGISTER_SIZE@)) { - @ISA@_sincos_FLOAT((npy_float*)args[1], (npy_float*)args[0], dimensions[0], steps[0], my_trig_op); - return 1; - } - else - return 0; -#endif - return 0; -} - /**end repeat**/ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE void -AVX512F_exp_DOUBLE(npy_double *, npy_double *, const npy_intp n, const npy_intp stride); -#endif -static NPY_INLINE int -run_unary_avx512f_exp_DOUBLE(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -#if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) - if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_double), sizeof(npy_double), 64)) { - AVX512F_exp_DOUBLE((npy_double*)args[1], (npy_double*)args[0], dimensions[0], steps[0]); - return 1; - } - else - return 0; -#endif -#endif - return 0; -} - -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE void -AVX512F_log_DOUBLE(npy_double *, npy_double *, const npy_intp n, const npy_intp stride); -#endif -static NPY_INLINE int -run_unary_avx512f_log_DOUBLE(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -#if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) - if (IS_OUTPUT_BLOCKABLE_UNARY(sizeof(npy_double), sizeof(npy_double), 64)) { - AVX512F_log_DOUBLE((npy_double*)args[1], (npy_double*)args[0], dimensions[0], steps[0]); - return 1; - } - else - return 0; -#endif -#endif - return 0; -} - /**begin repeat * Float types * #type = npy_float, npy_double, npy_longdouble# @@ -507,92 +232,6 @@ run_@name@_simd_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp /**end repeat1**/ -/**begin repeat1 - * Arithmetic - * # kind = add, subtract, multiply, divide# - */ - -#if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS - -/* prototypes */ -static void -sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, - npy_intp n); -static void -sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, - npy_intp n); -static void -sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, - npy_intp n); - -#elif @VECTOR@ - -static void -simd_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, - npy_intp n); -static void -simd_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, - npy_intp n); -static void -simd_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, - npy_intp n); - -#endif - -static NPY_INLINE int -run_binary_simd_@kind@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ -#if @vector@ && defined NPY_HAVE_SSE2_INTRINSICS - @type@ * ip1 = (@type@ *)args[0]; - @type@ * ip2 = (@type@ *)args[1]; - @type@ * op = (@type@ *)args[2]; - npy_intp n = dimensions[0]; -#if defined __AVX512F__ - const npy_uintp vector_size_bytes = 64; -#elif defined __AVX2__ - const npy_uintp vector_size_bytes = 32; -#else - const npy_uintp vector_size_bytes = 32; -#endif - /* argument one scalar */ - if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), vector_size_bytes)) { - sse2_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); - return 1; - } - /* argument two scalar */ - else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), vector_size_bytes)) { - sse2_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); - return 1; - } - else if (IS_BLOCKABLE_BINARY(sizeof(@type@), vector_size_bytes)) { - sse2_binary_@kind@_@TYPE@(op, ip1, ip2, n); - return 1; - } -#elif @VECTOR@ - @type@ * ip1 = (@type@ *)args[0]; - @type@ * ip2 = (@type@ *)args[1]; - @type@ * op = (@type@ *)args[2]; - npy_intp n = dimensions[0]; - /* argument one scalar */ - if (IS_BLOCKABLE_BINARY_SCALAR1(sizeof(@type@), NPY_SIMD_WIDTH)) { - simd_binary_scalar1_@kind@_@TYPE@(op, ip1, ip2, n); - return 1; - } - /* argument two scalar */ - else if (IS_BLOCKABLE_BINARY_SCALAR2(sizeof(@type@), NPY_SIMD_WIDTH)) { - simd_binary_scalar2_@kind@_@TYPE@(op, ip1, ip2, n); - return 1; - } - else if (IS_BLOCKABLE_BINARY(sizeof(@type@), NPY_SIMD_WIDTH)) { - simd_binary_@kind@_@TYPE@(op, ip1, ip2, n); - return 1; - } -#endif - return 0; -} - -/**end repeat1**/ - /**begin repeat1 * #kind = equal, not_equal, less, less_equal, greater, greater_equal, * logical_and, logical_or# @@ -777,7 +416,6 @@ static NPY_INLINE npy_double sse2_horizontal_@VOP@___m128d(__m128d v) _mm_store_sd(&r, _mm_@VOP@_pd(tmp, v)); /* m(ab) m(bb) */ return r; } - /**end repeat**/ /**begin repeat @@ -797,372 +435,51 @@ static NPY_INLINE npy_double sse2_horizontal_@VOP@___m128d(__m128d v) * #double = 0, 1# * #cast = _mm_castps_si128, _mm_castpd_si128# */ - - -/**begin repeat1 -* Arithmetic -* # kind = add, subtract, multiply, divide# -* # OP = +, -, *, /# -* # VOP = add, sub, mul, div# -*/ +/* + * compress 4 vectors to 4/8 bytes in op with filled with 0 or 1 + * the last vector is passed as a pointer as MSVC 2010 is unable to ignore the + * calling convention leading to C2719 on 32 bit, see #4795 + */ +static NPY_INLINE void +sse2_compress4_to_byte_@TYPE@(@vtype@ r1, @vtype@ r2, @vtype@ r3, @vtype@ * r4, + npy_bool * op) +{ + const __m128i mask = @vpre@_set1_epi8(0x1); + __m128i ir1 = @vpre@_packs_epi32(@cast@(r1), @cast@(r2)); + __m128i ir2 = @vpre@_packs_epi32(@cast@(r3), @cast@(*r4)); + __m128i rr = @vpre@_packs_epi16(ir1, ir2); +#if @double@ + rr = @vpre@_packs_epi16(rr, rr); + rr = @vpre@_and_si128(rr, mask); + @vpre@_storel_epi64((__m128i*)op, rr); +#else + rr = @vpre@_and_si128(rr, mask); + @vpre@_storeu_si128((__m128i*)op, rr); +#endif +} static void -sse2_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) -{ -#ifdef __AVX512F__ - const npy_intp vector_size_bytes = 64; - LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) - op[i] = ip1[i] @OP@ ip2[i]; - /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], vector_size_bytes) && npy_is_aligned(&ip2[i], vector_size_bytes)) { - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); - @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - } - else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); - @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); - @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - else { - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, a); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); - @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } +sse2_signbit_@TYPE@(npy_bool * op, @type@ * ip1, npy_intp n) +{ + LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { + op[i] = npy_signbit(ip1[i]) != 0; } -#elif __AVX2__ - const npy_intp vector_size_bytes = 32; - LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) - op[i] = ip1[i] @OP@ ip2[i]; - /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], vector_size_bytes) && - npy_is_aligned(&ip2[i], vector_size_bytes)) { - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); - @vpre256@_store_@vsuf@(&op[i], c); - } + LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { + @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); + int r = @vpre@_movemask_@vsuf@(a); + if (sizeof(@type@) == 8) { + op[i] = r & 1; + op[i + 1] = (r >> 1); } else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); - @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } - } - else if (npy_is_aligned(&ip1[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); - @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } - else if (npy_is_aligned(&ip2[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); - @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); + op[i] = r & 1; + op[i + 1] = (r >> 1) & 1; + op[i + 2] = (r >> 2) & 1; + op[i + 3] = (r >> 3); } } - else { - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, a); - @vpre256@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); - @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } - } -#else - LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) - op[i] = ip1[i] @OP@ ip2[i]; - /* lots of specializations, to squeeze out max performance */ - if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES) && - npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); - @vpre@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); - @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } - } - else if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES)) { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); - @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } - else if (npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); - @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } - else { - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, a); - @vpre@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); - @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } - } -#endif - LOOP_BLOCKED_END { - op[i] = ip1[i] @OP@ ip2[i]; - } -} - - -static void -sse2_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) -{ -#ifdef __AVX512F__ - const npy_intp vector_size_bytes = 64; - const @vtype512@ a = @vpre512@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) - op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ b = @vpre512@_load_@vsuf@(&ip2[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ b = @vpre512@_loadu_@vsuf@(&ip2[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - - -#elif __AVX2__ - const npy_intp vector_size_bytes = 32; - const @vtype256@ a = @vpre256@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) - op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ b = @vpre256@_load_@vsuf@(&ip2[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ b = @vpre256@_loadu_@vsuf@(&ip2[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } -#else - const @vtype@ a = @vpre@_set1_@vsuf@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) - op[i] = ip1[0] @OP@ ip2[i]; - if (npy_is_aligned(&ip2[i], VECTOR_SIZE_BYTES)) { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ b = @vpre@_load_@vsuf@(&ip2[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ b = @vpre@_loadu_@vsuf@(&ip2[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } -#endif - LOOP_BLOCKED_END { - op[i] = ip1[0] @OP@ ip2[i]; - } -} - - -static void -sse2_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) -{ -#ifdef __AVX512F__ - const npy_intp vector_size_bytes = 64; - const @vtype512@ b = @vpre512@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) - op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_load_@vsuf@(&ip1[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype512@ a = @vpre512@_loadu_@vsuf@(&ip1[i]); - @vtype512@ c = @vpre512@_@VOP@_@vsuf@(a, b); - @vpre512@_store_@vsuf@(&op[i], c); - } - } - -#elif __AVX2__ - const npy_intp vector_size_bytes = 32; - const @vtype256@ b = @vpre256@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, vector_size_bytes) - op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], vector_size_bytes)) { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_load_@vsuf@(&ip1[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, vector_size_bytes) { - @vtype256@ a = @vpre256@_loadu_@vsuf@(&ip1[i]); - @vtype256@ c = @vpre256@_@VOP@_@vsuf@(a, b); - @vpre256@_store_@vsuf@(&op[i], c); - } - } -#else - const @vtype@ b = @vpre@_set1_@vsuf@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, VECTOR_SIZE_BYTES) - op[i] = ip1[i] @OP@ ip2[0]; - if (npy_is_aligned(&ip1[i], VECTOR_SIZE_BYTES)) { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_loadu_@vsuf@(&ip1[i]); - @vtype@ c = @vpre@_@VOP@_@vsuf@(a, b); - @vpre@_store_@vsuf@(&op[i], c); - } - } -#endif - LOOP_BLOCKED_END { - op[i] = ip1[i] @OP@ ip2[0]; - } -} - -/**end repeat1**/ - -/* - * compress 4 vectors to 4/8 bytes in op with filled with 0 or 1 - * the last vector is passed as a pointer as MSVC 2010 is unable to ignore the - * calling convention leading to C2719 on 32 bit, see #4795 - */ -static NPY_INLINE void -sse2_compress4_to_byte_@TYPE@(@vtype@ r1, @vtype@ r2, @vtype@ r3, @vtype@ * r4, - npy_bool * op) -{ - const __m128i mask = @vpre@_set1_epi8(0x1); - __m128i ir1 = @vpre@_packs_epi32(@cast@(r1), @cast@(r2)); - __m128i ir2 = @vpre@_packs_epi32(@cast@(r3), @cast@(*r4)); - __m128i rr = @vpre@_packs_epi16(ir1, ir2); -#if @double@ - rr = @vpre@_packs_epi16(rr, rr); - rr = @vpre@_and_si128(rr, mask); - @vpre@_storel_epi64((__m128i*)op, rr); -#else - rr = @vpre@_and_si128(rr, mask); - @vpre@_storeu_si128((__m128i*)op, rr); -#endif -} - -static void -sse2_signbit_@TYPE@(npy_bool * op, @type@ * ip1, npy_intp n) -{ - LOOP_BLOCK_ALIGN_VAR(ip1, @type@, VECTOR_SIZE_BYTES) { - op[i] = npy_signbit(ip1[i]) != 0; - } - LOOP_BLOCKED(@type@, VECTOR_SIZE_BYTES) { - @vtype@ a = @vpre@_load_@vsuf@(&ip1[i]); - int r = @vpre@_movemask_@vsuf@(a); - if (sizeof(@type@) == 8) { - op[i] = r & 1; - op[i + 1] = (r >> 1); - } - else { - op[i] = r & 1; - op[i + 1] = (r >> 1) & 1; - op[i + 2] = (r >> 2) & 1; - op[i + 3] = (r >> 3); - } - } - LOOP_BLOCKED_END { - op[i] = npy_signbit(ip1[i]) != 0; + LOOP_BLOCKED_END { + op[i] = npy_signbit(ip1[i]) != 0; } } @@ -1522,111 +839,6 @@ fma_invert_mask_pd(__m256i ymask) return _mm256_andnot_si256(ymask, _mm256_set1_epi32(0xFFFFFFFF)); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 -fma_should_calculate_sine(__m256i k, __m256i andop, __m256i cmp) -{ - return _mm256_cvtepi32_ps( - _mm256_cmpeq_epi32(_mm256_and_si256(k, andop), cmp)); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 -fma_should_negate(__m256i k, __m256i andop, __m256i cmp) -{ - return fma_should_calculate_sine(k, andop, cmp); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 -fma_get_exponent(__m256 x) -{ - /* - * Special handling of denormals: - * 1) Multiply denormal elements with 2**100 (0x71800000) - * 2) Get the 8 bits of unbiased exponent - * 3) Subtract 100 from exponent of denormals - */ - - __m256 two_power_100 = _mm256_castsi256_ps(_mm256_set1_epi32(0x71800000)); - __m256 denormal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_LT_OQ); - __m256 normal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_GE_OQ); - - __m256 temp1 = _mm256_blendv_ps(x, _mm256_set1_ps(0.0f), normal_mask); - __m256 temp = _mm256_mul_ps(temp1, two_power_100); - x = _mm256_blendv_ps(x, temp, denormal_mask); - - __m256 exp = _mm256_cvtepi32_ps( - _mm256_sub_epi32( - _mm256_srli_epi32( - _mm256_castps_si256(x), 23),_mm256_set1_epi32(0x7E))); - - __m256 denorm_exp = _mm256_sub_ps(exp, _mm256_set1_ps(100.0f)); - return _mm256_blendv_ps(exp, denorm_exp, denormal_mask); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_FMA __m256 -fma_get_mantissa(__m256 x) -{ - /* - * Special handling of denormals: - * 1) Multiply denormal elements with 2**100 (0x71800000) - * 2) Get the 23 bits of mantissa - * 3) Mantissa for denormals is not affected by the multiplication - */ - - __m256 two_power_100 = _mm256_castsi256_ps(_mm256_set1_epi32(0x71800000)); - __m256 denormal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_LT_OQ); - __m256 normal_mask = _mm256_cmp_ps(x, _mm256_set1_ps(FLT_MIN), _CMP_GE_OQ); - - __m256 temp1 = _mm256_blendv_ps(x, _mm256_set1_ps(0.0f), normal_mask); - __m256 temp = _mm256_mul_ps(temp1, two_power_100); - x = _mm256_blendv_ps(x, temp, denormal_mask); - - __m256i mantissa_bits = _mm256_set1_epi32(0x7fffff); - __m256i exp_126_bits = _mm256_set1_epi32(126 << 23); - return _mm256_castsi256_ps( - _mm256_or_si256( - _mm256_and_si256( - _mm256_castps_si256(x), mantissa_bits), exp_126_bits)); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX2 __m256 -fma_scalef_ps(__m256 poly, __m256 quadrant) -{ - /* - * Handle denormals (which occur when quadrant <= -125): - * 1) This function computes poly*(2^quad) by adding the exponent of - poly to quad - * 2) When quad <= -125, the output is a denormal and the above logic - breaks down - * 3) To handle such cases, we split quadrant: -125 + (quadrant + 125) - * 4) poly*(2^-125) is computed the usual way - * 5) 2^(quad-125) can be computed by: 2 << abs(quad-125) - * 6) The final div operation generates the denormal - */ - __m256 minquadrant = _mm256_set1_ps(-125.0f); - __m256 denormal_mask = _mm256_cmp_ps(quadrant, minquadrant, _CMP_LE_OQ); - if (_mm256_movemask_ps(denormal_mask) != 0x0000) { - __m256 quad_diff = _mm256_sub_ps(quadrant, minquadrant); - quad_diff = _mm256_sub_ps(_mm256_setzero_ps(), quad_diff); - quad_diff = _mm256_blendv_ps(_mm256_setzero_ps(), quad_diff, denormal_mask); - __m256i two_power_diff = _mm256_sllv_epi32( - _mm256_set1_epi32(1), _mm256_cvtps_epi32(quad_diff)); - quadrant = _mm256_max_ps(quadrant, minquadrant); //keep quadrant >= -126 - __m256i exponent = _mm256_slli_epi32(_mm256_cvtps_epi32(quadrant), 23); - poly = _mm256_castsi256_ps( - _mm256_add_epi32( - _mm256_castps_si256(poly), exponent)); - __m256 denorm_poly = _mm256_div_ps(poly, _mm256_cvtepi32_ps(two_power_diff)); - return _mm256_blendv_ps(poly, denorm_poly, denormal_mask); - } - else { - __m256i exponent = _mm256_slli_epi32(_mm256_cvtps_epi32(quadrant), 23); - poly = _mm256_castsi256_ps( - _mm256_add_epi32( - _mm256_castps_si256(poly), exponent)); - return poly; - } -} - /**begin repeat * #vsub = ps, pd# * #vtype = __m256, __m256d# @@ -1754,64 +966,6 @@ avx512_invert_mask_pd(__mmask8 ymask) return _mm512_knot(ymask); } -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 -avx512_should_calculate_sine(__m512i k, __m512i andop, __m512i cmp) -{ - return _mm512_cmpeq_epi32_mask(_mm512_and_epi32(k, andop), cmp); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __mmask16 -avx512_should_negate(__m512i k, __m512i andop, __m512i cmp) -{ - return avx512_should_calculate_sine(k, andop, cmp); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 -avx512_get_exponent(__m512 x) -{ - return _mm512_add_ps(_mm512_getexp_ps(x), _mm512_set1_ps(1.0f)); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 -avx512_get_mantissa(__m512 x) -{ - return _mm512_getmant_ps(x, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512 -avx512_scalef_ps(__m512 poly, __m512 quadrant) -{ - return _mm512_scalef_ps(poly, quadrant); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512d -avx512_permute_x4var_pd(__m512d t0, - __m512d t1, - __m512d t2, - __m512d t3, - __m512i index) -{ - __mmask8 lut_mask = _mm512_cmp_epi64_mask( - _mm512_and_epi64(_mm512_set1_epi64(0x10ULL), index), - _mm512_set1_epi64(0), _MM_CMPINT_GT); - __m512d res1 = _mm512_permutex2var_pd(t0, index, t1); - __m512d res2 = _mm512_permutex2var_pd(t2, index, t3); - return _mm512_mask_blend_pd(lut_mask, res1, res2); -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F __m512d -avx512_permute_x8var_pd(__m512d t0, __m512d t1, __m512d t2, __m512d t3, - __m512d t4, __m512d t5, __m512d t6, __m512d t7, - __m512i index) -{ - __mmask8 lut_mask = _mm512_cmp_epi64_mask( - _mm512_and_epi64(_mm512_set1_epi64(0x20ULL), index), - _mm512_set1_epi64(0), _MM_CMPINT_GT); - __m512d res1 = avx512_permute_x4var_pd(t0, t1, t2, t3, index); - __m512d res2 = avx512_permute_x4var_pd(t4, t5, t6, t7, index); - return _mm512_mask_blend_pd(lut_mask, res1, res2); -} - /**begin repeat * #vsub = ps, pd# * #type= npy_float, npy_double# @@ -1969,68 +1123,6 @@ avx512_csquare_@vsub@(@vtype@ x) #if defined @CHK@ -/* - * Vectorized Cody-Waite range reduction technique - * Performs the reduction step x* = x - y*C in three steps: - * 1) x* = x - y*c1 - * 2) x* = x - y*c2 - * 3) x* = x - y*c3 - * c1, c2 are exact floating points, c3 = C - c1 - c2 simulates higher precision - */ - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ -@isa@_range_reduction(@vtype@ x, @vtype@ y, @vtype@ c1, @vtype@ c2, @vtype@ c3) -{ - @vtype@ reduced_x = @fmadd@(y, c1, x); - reduced_x = @fmadd@(y, c2, reduced_x); - reduced_x = @fmadd@(y, c3, reduced_x); - return reduced_x; -} - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @mask@ -@isa@_in_range_mask(@vtype@ x, npy_float fmax, npy_float fmin) -{ - @mask@ m1 = _mm@vsize@_cmp_ps@vsub@( - x, _mm@vsize@_set1_ps(fmax), _CMP_GT_OQ); - @mask@ m2 = _mm@vsize@_cmp_ps@vsub@( - x, _mm@vsize@_set1_ps(fmin), _CMP_LT_OQ); - return _mm@vsize@_@or@(m1,m2); -} - -/* - * Approximate cosine algorithm for x \in [-PI/4, PI/4] - * Maximum ULP across all 32-bit floats = 0.875 - */ - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ -@isa@_cosine(@vtype@ x2, @vtype@ invf8, @vtype@ invf6, @vtype@ invf4, - @vtype@ invf2, @vtype@ invf0) -{ - @vtype@ cos = @fmadd@(invf8, x2, invf6); - cos = @fmadd@(cos, x2, invf4); - cos = @fmadd@(cos, x2, invf2); - cos = @fmadd@(cos, x2, invf0); - return cos; -} - -/* - * Approximate sine algorithm for x \in [-PI/4, PI/4] - * Maximum ULP across all 32-bit floats = 0.647 - */ - -static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ -@isa@_sine(@vtype@ x, @vtype@ x2, @vtype@ invf9, @vtype@ invf7, - @vtype@ invf5, @vtype@ invf3, - @vtype@ zero) -{ - @vtype@ sin = @fmadd@(invf9, x2, invf7); - sin = @fmadd@(sin, x2, invf5); - sin = @fmadd@(sin, x2, invf3); - sin = @fmadd@(sin, x2, zero); - sin = @fmadd@(sin, x, x); - return sin; -} - static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ @vtype@ @isa@_sqrt_ps(@vtype@ x) { @@ -2116,7 +1208,7 @@ AVX512_SKX_@func@_@TYPE@(npy_bool* op, @type@* ip, const npy_intp array_size, co x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip, vindex_ip, load_mask); } #if @is_signbit@ - x1 = _mm512_and_@vsuffix@(x1,signbit); + x1 = _mm512_and_@vsuffix@(x1,signbit); #endif @mask@ fpclassmask = _mm512_fpclass_@vsuffix@_mask(x1, @IMM8@); @@ -2154,18 +1246,22 @@ AVX512_SKX_@func@_@TYPE@(npy_bool* op, @type@* ip, const npy_intp array_size, co * #vtype2_scatter = _mm512_mask_i32scatter_epi32, _mm256_mask_i32scatter_epi32# * #setzero = _mm512_setzero_epi32, _mm256_setzero_si256# */ +/**begin repeat1 + * #func = maximum, minimum# + * #vectorf = max, min# + */ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512_SKX_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE NPY_GCC_TARGET_AVX512_SKX void -AVX512_SKX_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) +#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS +static NPY_INLINE NPY_GCC_TARGET_AVX512F void +AVX512F_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) { const npy_intp stride_ip1 = steps[0]/(npy_intp)sizeof(@type@); - const npy_intp stride_ip2 = steps[1]/(npy_intp)sizeof(int); + const npy_intp stride_ip2 = steps[1]/(npy_intp)sizeof(@type@); const npy_intp stride_op = steps[2]/(npy_intp)sizeof(@type@); const npy_intp array_size = dimensions[0]; npy_intp num_remaining_elements = array_size; @type@* ip1 = (@type@*) args[0]; - int* ip2 = (int*) args[1]; + @type@* ip2 = (@type@*) args[1]; @type@* op = (@type@*) args[2]; @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); @@ -2186,15 +1282,13 @@ AVX512_SKX_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const @vindextype@ vindex_ip2 = @vindexload@((@vindextype@*)&index_ip2[0]); @vindextype@ vindex_op = @vindexload@((@vindextype@*)&index_op[0]); @vtype1@ zeros_f = _mm512_setzero_@vsuffix@(); - @vtype2@ zeros = @setzero@(); while (num_remaining_elements > 0) { if (num_remaining_elements < @num_lanes@) { load_mask = avx512_get_partial_load_mask_@vsuffix@( num_remaining_elements, @num_lanes@); } - @vtype1@ x1; - @vtype2@ x2; + @vtype1@ x1, x2; if (stride_ip1 == 1) { x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); } @@ -2202,14 +1296,21 @@ AVX512_SKX_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip1, vindex_ip1, load_mask); } if (stride_ip2 == 1) { - x2 = @vtype2_load@(load_mask, ip2); + x2 = avx512_masked_load_@vsuffix@(load_mask, ip2); } else { - x2 = @vtype2_gather@(zeros, load_mask, vindex_ip2, ip2, 4); + x2 = avx512_masked_gather_@vsuffix@(zeros_f, ip2, vindex_ip2, load_mask); } - @vtype1@ out = _mm512_scalef_@vsuffix@(x1, _mm512_cvtepi32_@vsuffix@(x2)); - + /* + * when only one of the argument is a nan, the maxps/maxpd instruction + * returns the second argument. The additional blend instruction fixes + * this issue to conform with NumPy behaviour. + */ + @mask@ nan_mask = _mm512_cmp_@vsuffix@_mask(x1, x1, _CMP_NEQ_UQ); + @vtype1@ out = _mm512_@vectorf@_@vsuffix@(x1, x2); + out = _mm512_mask_blend_@vsuffix@(nan_mask, out, x1); + if (stride_op == 1) { _mm512_mask_storeu_@vsuffix@(op, load_mask, out); } @@ -2224,186 +1325,28 @@ AVX512_SKX_ldexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const num_remaining_elements -= @num_lanes@; } } +#endif +/**end repeat1**/ +/**end repeat**/ -static NPY_INLINE NPY_GCC_TARGET_AVX512_SKX void -AVX512_SKX_frexp_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ - const npy_intp stride_ip1 = steps[0]/(npy_intp)sizeof(@type@); - const npy_intp stride_op1 = steps[1]/(npy_intp)sizeof(@type@); - const npy_intp stride_op2 = steps[2]/(npy_intp)sizeof(int); - const npy_intp array_size = dimensions[0]; - npy_intp num_remaining_elements = array_size; - @type@* ip1 = (@type@*) args[0]; - @type@* op1 = (@type@*) args[1]; - int* op2 = (int*) args[2]; +/**begin repeat + * #ISA = FMA, AVX512F# + * #isa = fma, avx512# + * #vsize = 256, 512# + * #BYTES = 32, 64# + * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #mask = __m256, __mmask16# + * #vsub = , _mask# + * #vtype = __m256, __m512# + * #cvtps_epi32 = _mm256_cvtps_epi32, # + * #masked_store = _mm256_maskstore_ps, _mm512_mask_storeu_ps# + * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# + */ - @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); - - /* - * Note: while generally indices are npy_intp, we ensure that our maximum index - * will fit in an int32 as a precondition for this function via - * IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP - */ - - npy_int32 index_ip1[@num_lanes@], index_op1[@num_lanes@], index_op2[@num_lanes@]; - for (npy_int32 ii = 0; ii < @num_lanes@; ii++) { - index_ip1[ii] = ii*stride_ip1; - index_op1[ii] = ii*stride_op1; - index_op2[ii] = ii*stride_op2; - } - @vindextype@ vindex_ip1 = @vindexload@((@vindextype@*)&index_ip1[0]); - @vindextype@ vindex_op1 = @vindexload@((@vindextype@*)&index_op1[0]); - @vindextype@ vindex_op2 = @vindexload@((@vindextype@*)&index_op2[0]); - @vtype1@ zeros_f = _mm512_setzero_@vsuffix@(); - - while (num_remaining_elements > 0) { - if (num_remaining_elements < @num_lanes@) { - load_mask = avx512_get_partial_load_mask_@vsuffix@( - num_remaining_elements, @num_lanes@); - } - @vtype1@ x1; - if (stride_ip1 == 1) { - x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); - } - else { - x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip1, vindex_ip1, load_mask); - } - - /* - * The x86 instructions vpgetmant and vpgetexp do not conform - * with NumPy's output for special floating points: NAN, +/-INF, +/-0.0 - * We mask these values with spmask to avoid invalid exceptions. - */ - @mask@ spmask =_mm512_knot(_mm512_fpclass_@vsuffix@_mask( - x1, 0b10011111)); - @vtype1@ out1 = _mm512_maskz_getmant_@vsuffix@( - spmask, x1, _MM_MANT_NORM_p5_1, _MM_MANT_SIGN_src); - out1 = _mm512_mask_mov_@vsuffix@(x1, spmask, out1); - @vtype2@ out2 = _mm512_cvt@vsuffix@_epi32( - _mm512_maskz_add_@vsuffix@(spmask, _mm512_set1_@vsuffix@(1.0), - _mm512_maskz_getexp_@vsuffix@(spmask, x1))); - if (stride_op1 == 1) { - _mm512_mask_storeu_@vsuffix@(op1, load_mask, out1); - } - else { - _mm512_mask_i32scatter_@vsuffix@(op1, load_mask, vindex_op1, out1, @scale@); - } - if (stride_op2 == 1) { - @vtype2_store@(op2, load_mask, out2); - } - else { - @vtype2_scatter@(op2, load_mask, vindex_op2, out2, 4); - } - - ip1 += @num_lanes@*stride_ip1; - op1 += @num_lanes@*stride_op1; - op2 += @num_lanes@*stride_op2; - num_remaining_elements -= @num_lanes@; - } -} -#endif - -/**begin repeat1 - * #func = maximum, minimum# - * #vectorf = max, min# - */ - -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_INLINE NPY_GCC_TARGET_AVX512F void -AVX512F_@func@_@TYPE@(char **args, npy_intp const *dimensions, npy_intp const *steps) -{ - const npy_intp stride_ip1 = steps[0]/(npy_intp)sizeof(@type@); - const npy_intp stride_ip2 = steps[1]/(npy_intp)sizeof(@type@); - const npy_intp stride_op = steps[2]/(npy_intp)sizeof(@type@); - const npy_intp array_size = dimensions[0]; - npy_intp num_remaining_elements = array_size; - @type@* ip1 = (@type@*) args[0]; - @type@* ip2 = (@type@*) args[1]; - @type@* op = (@type@*) args[2]; - - @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); - - /* - * Note: while generally indices are npy_intp, we ensure that our maximum index - * will fit in an int32 as a precondition for this function via - * IS_BINARY_SMALL_STEPS_AND_NOMEMOVERLAP - */ - - npy_int32 index_ip1[@num_lanes@], index_ip2[@num_lanes@], index_op[@num_lanes@]; - for (npy_int32 ii = 0; ii < @num_lanes@; ii++) { - index_ip1[ii] = ii*stride_ip1; - index_ip2[ii] = ii*stride_ip2; - index_op[ii] = ii*stride_op; - } - @vindextype@ vindex_ip1 = @vindexload@((@vindextype@*)&index_ip1[0]); - @vindextype@ vindex_ip2 = @vindexload@((@vindextype@*)&index_ip2[0]); - @vindextype@ vindex_op = @vindexload@((@vindextype@*)&index_op[0]); - @vtype1@ zeros_f = _mm512_setzero_@vsuffix@(); - - while (num_remaining_elements > 0) { - if (num_remaining_elements < @num_lanes@) { - load_mask = avx512_get_partial_load_mask_@vsuffix@( - num_remaining_elements, @num_lanes@); - } - @vtype1@ x1, x2; - if (stride_ip1 == 1) { - x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); - } - else { - x1 = avx512_masked_gather_@vsuffix@(zeros_f, ip1, vindex_ip1, load_mask); - } - if (stride_ip2 == 1) { - x2 = avx512_masked_load_@vsuffix@(load_mask, ip2); - } - else { - x2 = avx512_masked_gather_@vsuffix@(zeros_f, ip2, vindex_ip2, load_mask); - } - - /* - * when only one of the argument is a nan, the maxps/maxpd instruction - * returns the second argument. The additional blend instruction fixes - * this issue to conform with NumPy behaviour. - */ - @mask@ nan_mask = _mm512_cmp_@vsuffix@_mask(x1, x1, _CMP_NEQ_UQ); - @vtype1@ out = _mm512_@vectorf@_@vsuffix@(x1, x2); - out = _mm512_mask_blend_@vsuffix@(nan_mask, out, x1); - - if (stride_op == 1) { - _mm512_mask_storeu_@vsuffix@(op, load_mask, out); - } - else { - /* scatter! */ - _mm512_mask_i32scatter_@vsuffix@(op, load_mask, vindex_op, out, @scale@); - } - - ip1 += @num_lanes@*stride_ip1; - ip2 += @num_lanes@*stride_ip2; - op += @num_lanes@*stride_op; - num_remaining_elements -= @num_lanes@; - } -} -#endif -/**end repeat1**/ -/**end repeat**/ - -/**begin repeat - * #ISA = FMA, AVX512F# - * #isa = fma, avx512# - * #vsize = 256, 512# - * #BYTES = 32, 64# - * #cvtps_epi32 = _mm256_cvtps_epi32, # - * #mask = __m256, __mmask16# - * #vsub = , _mask# - * #vtype = __m256, __m512# - * #cvtps_epi32 = _mm256_cvtps_epi32, # - * #masked_store = _mm256_maskstore_ps, _mm512_mask_storeu_ps# - * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# - */ - -/**begin repeat1 - * #func = rint, ceil, floor, trunc# - * #vectorf = rint, ceil, floor, trunc# - */ +/**begin repeat1 + * #func = rint, ceil, floor, trunc# + * #vectorf = rint, ceil, floor, trunc# + */ #if defined @CHK@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void @@ -2524,814 +1467,6 @@ static NPY_INLINE NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void /**end repeat1**/ /**end repeat**/ -/**begin repeat - * #ISA = FMA, AVX512F# - * #isa = fma, avx512# - * #vtype = __m256, __m512# - * #vsize = 256, 512# - * #BYTES = 32, 64# - * #NUM_LANES = 8, 16# - * #mask = __m256, __mmask16# - * #vsub = , _mask# - * #or_masks =_mm256_or_ps, _mm512_kor# - * #and_masks =_mm256_and_ps, _mm512_kand# - * #xor_masks =_mm256_xor_ps, _mm512_kxor# - * #fmadd = _mm256_fmadd_ps, _mm512_fmadd_ps# - * #mask_to_int = _mm256_movemask_ps, # - * #full_mask= 0xFF, 0xFFFF# - * #masked_store = _mm256_maskstore_ps, _mm512_mask_storeu_ps# - * #cvtps_epi32 = _mm256_cvtps_epi32, # - * #CHK = HAVE_ATTRIBUTE_TARGET_AVX2_WITH_INTRINSICS, HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS# - */ - -/* - * Vectorized approximate sine/cosine algorithms: The following code is a - * vectorized version of the algorithm presented here: - * https://stackoverflow.com/questions/30463616/payne-hanek-algorithm-implementation-in-c/30465751#30465751 - * (1) Load data in ZMM/YMM registers and generate mask for elements that are - * within range [-71476.0625f, 71476.0625f] for cosine and [-117435.992f, - * 117435.992f] for sine. - * (2) For elements within range, perform range reduction using Cody-Waite's - * method: x* = x - y*PI/2, where y = rint(x*2/PI). x* \in [-PI/4, PI/4]. - * (3) Map cos(x) to (+/-)sine or (+/-)cosine of x* based on the quadrant k = - * int(y). - * (4) For elements outside that range, Cody-Waite reduction performs poorly - * leading to catastrophic cancellation. We compute cosine by calling glibc in - * a scalar fashion. - * (5) Vectorized implementation has a max ULP of 1.49 and performs at least - * 5-7x faster than scalar implementations when magnitude of all elements in - * the array < 71476.0625f (117435.992f for sine). Worst case performance is - * when all the elements are large leading to about 1-2% reduction in - * performance. - */ - -#if defined @CHK@ -static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void -@ISA@_sincos_FLOAT(npy_float * op, - npy_float * ip, - const npy_intp array_size, - const npy_intp steps, - NPY_TRIG_OP my_trig_op) -{ - const npy_intp stride = steps/(npy_intp)sizeof(npy_float); - const npy_int num_lanes = @NUM_LANES@; - npy_float large_number = 71476.0625f; - if (my_trig_op == npy_compute_sin) { - large_number = 117435.992f; - } - - /* Load up frequently used constants */ - @vtype@i zeros = _mm@vsize@_set1_epi32(0); - @vtype@i ones = _mm@vsize@_set1_epi32(1); - @vtype@i twos = _mm@vsize@_set1_epi32(2); - @vtype@ two_over_pi = _mm@vsize@_set1_ps(NPY_TWO_O_PIf); - @vtype@ codyw_c1 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_PI_O_2_HIGHf); - @vtype@ codyw_c2 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_PI_O_2_MEDf); - @vtype@ codyw_c3 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_PI_O_2_LOWf); - @vtype@ cos_invf0 = _mm@vsize@_set1_ps(NPY_COEFF_INVF0_COSINEf); - @vtype@ cos_invf2 = _mm@vsize@_set1_ps(NPY_COEFF_INVF2_COSINEf); - @vtype@ cos_invf4 = _mm@vsize@_set1_ps(NPY_COEFF_INVF4_COSINEf); - @vtype@ cos_invf6 = _mm@vsize@_set1_ps(NPY_COEFF_INVF6_COSINEf); - @vtype@ cos_invf8 = _mm@vsize@_set1_ps(NPY_COEFF_INVF8_COSINEf); - @vtype@ sin_invf3 = _mm@vsize@_set1_ps(NPY_COEFF_INVF3_SINEf); - @vtype@ sin_invf5 = _mm@vsize@_set1_ps(NPY_COEFF_INVF5_SINEf); - @vtype@ sin_invf7 = _mm@vsize@_set1_ps(NPY_COEFF_INVF7_SINEf); - @vtype@ sin_invf9 = _mm@vsize@_set1_ps(NPY_COEFF_INVF9_SINEf); - @vtype@ cvt_magic = _mm@vsize@_set1_ps(NPY_RINT_CVT_MAGICf); - @vtype@ zero_f = _mm@vsize@_set1_ps(0.0f); - @vtype@ quadrant, reduced_x, reduced_x2, cos, sin; - @vtype@i iquadrant; - @mask@ nan_mask, glibc_mask, sine_mask, negate_mask; - @mask@ load_mask = @isa@_get_full_load_mask_ps(); - npy_intp num_remaining_elements = array_size; - - /* - * Note: while generally indices are npy_intp, we ensure that our maximum index - * will fit in an int32 as a precondition for this function via - * IS_OUTPUT_BLOCKABLE_UNARY - */ - npy_int32 indexarr[16]; - for (npy_int32 ii = 0; ii < 16; ii++) { - indexarr[ii] = ii*stride; - } - @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); - - while (num_remaining_elements > 0) { - - if (num_remaining_elements < num_lanes) { - load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, - num_lanes); - } - - @vtype@ x_in; - if (stride == 1) { - x_in = @isa@_masked_load_ps(load_mask, ip); - } - else { - x_in = @isa@_masked_gather_ps(zero_f, ip, vindex, load_mask); - } - - /* - * For elements outside of this range, Cody-Waite's range reduction - * becomes inaccurate and we will call glibc to compute cosine for - * these numbers - */ - - glibc_mask = @isa@_in_range_mask(x_in, large_number,-large_number); - glibc_mask = @and_masks@(load_mask, glibc_mask); - nan_mask = _mm@vsize@_cmp_ps@vsub@(x_in, x_in, _CMP_NEQ_UQ); - @vtype@ x = @isa@_set_masked_lanes_ps(x_in, zero_f, @or_masks@(nan_mask, glibc_mask)); - npy_int iglibc_mask = @mask_to_int@(glibc_mask); - - if (iglibc_mask != @full_mask@) { - quadrant = _mm@vsize@_mul_ps(x, two_over_pi); - - /* round to nearest */ - quadrant = _mm@vsize@_add_ps(quadrant, cvt_magic); - quadrant = _mm@vsize@_sub_ps(quadrant, cvt_magic); - - /* Cody-Waite's range reduction algorithm */ - reduced_x = @isa@_range_reduction(x, quadrant, - codyw_c1, codyw_c2, codyw_c3); - reduced_x2 = _mm@vsize@_mul_ps(reduced_x, reduced_x); - - /* compute cosine and sine */ - cos = @isa@_cosine(reduced_x2, cos_invf8, cos_invf6, cos_invf4, - cos_invf2, cos_invf0); - sin = @isa@_sine(reduced_x, reduced_x2, sin_invf9, sin_invf7, - sin_invf5, sin_invf3, zero_f); - - iquadrant = _mm@vsize@_cvtps_epi32(quadrant); - if (my_trig_op == npy_compute_cos) { - iquadrant = _mm@vsize@_add_epi32(iquadrant, ones); - } - - /* blend sin and cos based on the quadrant */ - sine_mask = @isa@_should_calculate_sine(iquadrant, ones, zeros); - cos = @isa@_blend(cos, sin, sine_mask); - - /* multiply by -1 for appropriate elements */ - negate_mask = @isa@_should_negate(iquadrant, twos, twos); - cos = @isa@_blend(cos, _mm@vsize@_sub_ps(zero_f, cos), negate_mask); - cos = @isa@_set_masked_lanes_ps(cos, _mm@vsize@_set1_ps(NPY_NANF), nan_mask); - - @masked_store@(op, @cvtps_epi32@(load_mask), cos); - } - - /* process elements using glibc for large elements */ - if (iglibc_mask != 0) { - float NPY_DECL_ALIGNED(@BYTES@) ip_fback[@NUM_LANES@]; - _mm@vsize@_store_ps(ip_fback, x_in); - - if (my_trig_op == npy_compute_cos) { - for (int ii = 0; ii < num_lanes; ++ii, iglibc_mask >>= 1) { - if (iglibc_mask & 0x01) { - op[ii] = npy_cosf(ip_fback[ii]); - } - } - } - else { - for (int ii = 0; ii < num_lanes; ++ii, iglibc_mask >>= 1) { - if (iglibc_mask & 0x01) { - op[ii] = npy_sinf(ip_fback[ii]); - } - } - } - } - ip += num_lanes*stride; - op += num_lanes; - num_remaining_elements -= num_lanes; - } -} - -/* - * Vectorized implementation of exp using AVX2 and AVX512: - * 1) if x >= xmax; return INF (overflow) - * 2) if x <= xmin; return 0.0f (underflow) - * 3) Range reduction (using Coyd-Waite): - * a) y = x - k*ln(2); k = rint(x/ln(2)); y \in [0, ln(2)] - * 4) Compute exp(y) = P/Q, ratio of 2 polynomials P and Q - * b) P = 5th order and Q = 2nd order polynomials obtained from Remez's - * algorithm (mini-max polynomial approximation) - * 5) Compute exp(x) = exp(y) * 2^k - * 6) Max ULP error measured across all 32-bit FP's = 2.52 (x = 0xc2781e37) - * 7) Max relative error measured across all 32-bit FP's= 2.1264E-07 (for the - * same x = 0xc2781e37) - */ - -static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void -@ISA@_exp_FLOAT(npy_float * op, - npy_float * ip, - const npy_intp array_size, - const npy_intp steps) -{ - const npy_intp stride = steps/(npy_intp)sizeof(npy_float); - const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); - npy_float xmax = 88.72283935546875f; - npy_float xmin = -103.97208404541015625f; - - /* - * Note: while generally indices are npy_intp, we ensure that our maximum index - * will fit in an int32 as a precondition for this function via - * IS_OUTPUT_BLOCKABLE_UNARY - */ - npy_int32 indexarr[16]; - for (npy_int32 ii = 0; ii < 16; ii++) { - indexarr[ii] = ii*stride; - } - - /* Load up frequently used constants */ - @vtype@ codyw_c1 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_LOGE_2_HIGHf); - @vtype@ codyw_c2 = _mm@vsize@_set1_ps(NPY_CODY_WAITE_LOGE_2_LOWf); - @vtype@ exp_p0 = _mm@vsize@_set1_ps(NPY_COEFF_P0_EXPf); - @vtype@ exp_p1 = _mm@vsize@_set1_ps(NPY_COEFF_P1_EXPf); - @vtype@ exp_p2 = _mm@vsize@_set1_ps(NPY_COEFF_P2_EXPf); - @vtype@ exp_p3 = _mm@vsize@_set1_ps(NPY_COEFF_P3_EXPf); - @vtype@ exp_p4 = _mm@vsize@_set1_ps(NPY_COEFF_P4_EXPf); - @vtype@ exp_p5 = _mm@vsize@_set1_ps(NPY_COEFF_P5_EXPf); - @vtype@ exp_q0 = _mm@vsize@_set1_ps(NPY_COEFF_Q0_EXPf); - @vtype@ exp_q1 = _mm@vsize@_set1_ps(NPY_COEFF_Q1_EXPf); - @vtype@ exp_q2 = _mm@vsize@_set1_ps(NPY_COEFF_Q2_EXPf); - @vtype@ cvt_magic = _mm@vsize@_set1_ps(NPY_RINT_CVT_MAGICf); - @vtype@ log2e = _mm@vsize@_set1_ps(NPY_LOG2Ef); - @vtype@ inf = _mm@vsize@_set1_ps(NPY_INFINITYF); - @vtype@ zeros_f = _mm@vsize@_set1_ps(0.0f); - @vtype@ poly, num_poly, denom_poly, quadrant; - @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)&indexarr[0]); - - @mask@ xmax_mask, xmin_mask, nan_mask, inf_mask; - @mask@ overflow_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); - @mask@ load_mask = @isa@_get_full_load_mask_ps(); - npy_intp num_remaining_elements = array_size; - - while (num_remaining_elements > 0) { - - if (num_remaining_elements < num_lanes) { - load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, - num_lanes); - } - - @vtype@ x; - if (stride == 1) { - x = @isa@_masked_load_ps(load_mask, ip); - } - else { - x = @isa@_masked_gather_ps(zeros_f, ip, vindex, load_mask); - } - - nan_mask = _mm@vsize@_cmp_ps@vsub@(x, x, _CMP_NEQ_UQ); - x = @isa@_set_masked_lanes_ps(x, zeros_f, nan_mask); - - xmax_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(xmax), _CMP_GE_OQ); - xmin_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(xmin), _CMP_LE_OQ); - inf_mask = _mm@vsize@_cmp_ps@vsub@(x, inf, _CMP_EQ_OQ); - overflow_mask = @or_masks@(overflow_mask, - @xor_masks@(xmax_mask, inf_mask)); - - x = @isa@_set_masked_lanes_ps(x, zeros_f, @or_masks@( - @or_masks@(nan_mask, xmin_mask), xmax_mask)); - - quadrant = _mm@vsize@_mul_ps(x, log2e); - - /* round to nearest */ - quadrant = _mm@vsize@_add_ps(quadrant, cvt_magic); - quadrant = _mm@vsize@_sub_ps(quadrant, cvt_magic); - - /* Cody-Waite's range reduction algorithm */ - x = @isa@_range_reduction(x, quadrant, codyw_c1, codyw_c2, zeros_f); - - num_poly = @fmadd@(exp_p5, x, exp_p4); - num_poly = @fmadd@(num_poly, x, exp_p3); - num_poly = @fmadd@(num_poly, x, exp_p2); - num_poly = @fmadd@(num_poly, x, exp_p1); - num_poly = @fmadd@(num_poly, x, exp_p0); - denom_poly = @fmadd@(exp_q2, x, exp_q1); - denom_poly = @fmadd@(denom_poly, x, exp_q0); - poly = _mm@vsize@_div_ps(num_poly, denom_poly); - - /* - * compute val = poly * 2^quadrant; which is same as adding the - * exponent of quadrant to the exponent of poly. quadrant is an int, - * so extracting exponent is simply extracting 8 bits. - */ - poly = @isa@_scalef_ps(poly, quadrant); - - /* - * elem > xmax; return inf - * elem < xmin; return 0.0f - * elem = +/- nan, return nan - */ - poly = @isa@_set_masked_lanes_ps(poly, _mm@vsize@_set1_ps(NPY_NANF), nan_mask); - poly = @isa@_set_masked_lanes_ps(poly, inf, xmax_mask); - poly = @isa@_set_masked_lanes_ps(poly, zeros_f, xmin_mask); - - @masked_store@(op, @cvtps_epi32@(load_mask), poly); - - ip += num_lanes*stride; - op += num_lanes; - num_remaining_elements -= num_lanes; - } - - if (@mask_to_int@(overflow_mask)) { - npy_set_floatstatus_overflow(); - } -} - -/* - * Vectorized implementation of log using AVX2 and AVX512 - * 1) if x < 0.0f; return -NAN (invalid input) - * 2) Range reduction: y = x/2^k; - * a) y = normalized mantissa, k is the exponent (0.5 <= y < 1) - * 3) Compute log(y) = P/Q, ratio of 2 polynomials P and Q - * b) P = 5th order and Q = 5th order polynomials obtained from Remez's - * algorithm (mini-max polynomial approximation) - * 5) Compute log(x) = log(y) + k*ln(2) - * 6) Max ULP error measured across all 32-bit FP's = 3.83 (x = 0x3f486945) - * 7) Max relative error measured across all 32-bit FP's = 2.359E-07 (for same - * x = 0x3f486945) - */ - -static NPY_GCC_OPT_3 NPY_GCC_TARGET_@ISA@ void -@ISA@_log_FLOAT(npy_float * op, - npy_float * ip, - const npy_intp array_size, - const npy_intp steps) -{ - const npy_intp stride = steps/(npy_intp)sizeof(npy_float); - const npy_int num_lanes = @BYTES@/(npy_intp)sizeof(npy_float); - - /* - * Note: while generally indices are npy_intp, we ensure that our maximum index - * will fit in an int32 as a precondition for this function via - * IS_OUTPUT_BLOCKABLE_UNARY - */ - npy_int32 indexarr[16]; - for (npy_int32 ii = 0; ii < 16; ii++) { - indexarr[ii] = ii*stride; - } - - /* Load up frequently used constants */ - @vtype@ log_p0 = _mm@vsize@_set1_ps(NPY_COEFF_P0_LOGf); - @vtype@ log_p1 = _mm@vsize@_set1_ps(NPY_COEFF_P1_LOGf); - @vtype@ log_p2 = _mm@vsize@_set1_ps(NPY_COEFF_P2_LOGf); - @vtype@ log_p3 = _mm@vsize@_set1_ps(NPY_COEFF_P3_LOGf); - @vtype@ log_p4 = _mm@vsize@_set1_ps(NPY_COEFF_P4_LOGf); - @vtype@ log_p5 = _mm@vsize@_set1_ps(NPY_COEFF_P5_LOGf); - @vtype@ log_q0 = _mm@vsize@_set1_ps(NPY_COEFF_Q0_LOGf); - @vtype@ log_q1 = _mm@vsize@_set1_ps(NPY_COEFF_Q1_LOGf); - @vtype@ log_q2 = _mm@vsize@_set1_ps(NPY_COEFF_Q2_LOGf); - @vtype@ log_q3 = _mm@vsize@_set1_ps(NPY_COEFF_Q3_LOGf); - @vtype@ log_q4 = _mm@vsize@_set1_ps(NPY_COEFF_Q4_LOGf); - @vtype@ log_q5 = _mm@vsize@_set1_ps(NPY_COEFF_Q5_LOGf); - @vtype@ loge2 = _mm@vsize@_set1_ps(NPY_LOGE2f); - @vtype@ nan = _mm@vsize@_set1_ps(NPY_NANF); - @vtype@ neg_nan = _mm@vsize@_set1_ps(-NPY_NANF); - @vtype@ neg_inf = _mm@vsize@_set1_ps(-NPY_INFINITYF); - @vtype@ inf = _mm@vsize@_set1_ps(NPY_INFINITYF); - @vtype@ zeros_f = _mm@vsize@_set1_ps(0.0f); - @vtype@ ones_f = _mm@vsize@_set1_ps(1.0f); - @vtype@i vindex = _mm@vsize@_loadu_si@vsize@((@vtype@i*)indexarr); - @vtype@ poly, num_poly, denom_poly, exponent; - - @mask@ inf_mask, nan_mask, sqrt2_mask, zero_mask, negx_mask; - @mask@ invalid_mask = @isa@_get_partial_load_mask_ps(0, num_lanes); - @mask@ divide_by_zero_mask = invalid_mask; - @mask@ load_mask = @isa@_get_full_load_mask_ps(); - npy_intp num_remaining_elements = array_size; - - while (num_remaining_elements > 0) { - - if (num_remaining_elements < num_lanes) { - load_mask = @isa@_get_partial_load_mask_ps(num_remaining_elements, - num_lanes); - } - - @vtype@ x_in; - if (stride == 1) { - x_in = @isa@_masked_load_ps(load_mask, ip); - } - else { - x_in = @isa@_masked_gather_ps(zeros_f, ip, vindex, load_mask); - } - - negx_mask = _mm@vsize@_cmp_ps@vsub@(x_in, zeros_f, _CMP_LT_OQ); - zero_mask = _mm@vsize@_cmp_ps@vsub@(x_in, zeros_f, _CMP_EQ_OQ); - inf_mask = _mm@vsize@_cmp_ps@vsub@(x_in, inf, _CMP_EQ_OQ); - nan_mask = _mm@vsize@_cmp_ps@vsub@(x_in, x_in, _CMP_NEQ_UQ); - divide_by_zero_mask = @or_masks@(divide_by_zero_mask, - @and_masks@(zero_mask, load_mask)); - invalid_mask = @or_masks@(invalid_mask, negx_mask); - - @vtype@ x = @isa@_set_masked_lanes_ps(x_in, zeros_f, negx_mask); - - /* set x = normalized mantissa */ - exponent = @isa@_get_exponent(x); - x = @isa@_get_mantissa(x); - - /* if x < sqrt(2) {exp = exp-1; x = 2*x} */ - sqrt2_mask = _mm@vsize@_cmp_ps@vsub@(x, _mm@vsize@_set1_ps(NPY_SQRT1_2f), _CMP_LE_OQ); - x = @isa@_blend(x, _mm@vsize@_add_ps(x,x), sqrt2_mask); - exponent = @isa@_blend(exponent, - _mm@vsize@_sub_ps(exponent,ones_f), sqrt2_mask); - - /* x = x - 1 */ - x = _mm@vsize@_sub_ps(x, ones_f); - - /* Polynomial approximation for log(1+x) */ - num_poly = @fmadd@(log_p5, x, log_p4); - num_poly = @fmadd@(num_poly, x, log_p3); - num_poly = @fmadd@(num_poly, x, log_p2); - num_poly = @fmadd@(num_poly, x, log_p1); - num_poly = @fmadd@(num_poly, x, log_p0); - denom_poly = @fmadd@(log_q5, x, log_q4); - denom_poly = @fmadd@(denom_poly, x, log_q3); - denom_poly = @fmadd@(denom_poly, x, log_q2); - denom_poly = @fmadd@(denom_poly, x, log_q1); - denom_poly = @fmadd@(denom_poly, x, log_q0); - poly = _mm@vsize@_div_ps(num_poly, denom_poly); - poly = @fmadd@(exponent, loge2, poly); - - /* - * x < 0.0f; return -NAN - * x = +/- NAN; return NAN - * x = 0.0f; return -INF - */ - poly = @isa@_set_masked_lanes_ps(poly, nan, nan_mask); - poly = @isa@_set_masked_lanes_ps(poly, neg_nan, negx_mask); - poly = @isa@_set_masked_lanes_ps(poly, neg_inf, zero_mask); - poly = @isa@_set_masked_lanes_ps(poly, inf, inf_mask); - - @masked_store@(op, @cvtps_epi32@(load_mask), poly); - - ip += num_lanes*stride; - op += num_lanes; - num_remaining_elements -= num_lanes; - } - - if (@mask_to_int@(invalid_mask)) { - npy_set_floatstatus_invalid(); - } - if (@mask_to_int@(divide_by_zero_mask)) { - npy_set_floatstatus_divbyzero(); - } -} -#endif -/**end repeat**/ - -/* - * Vectorized implementation of exp double using AVX512 - * Reference: Tang, P.T.P., "Table-driven implementation of the - * exponential function in IEEE floating-point - * arithmetic," ACM Transactions on Mathematical - * Software, vol. 15, pp. 144-157, 1989. - * 1) if x > mTH_max or x is INF; return INF (overflow) - * 2) if x < mTH_min; return 0.0f (underflow) - * 3) if abs(x) < mTH_nearzero; return 1.0f + x - * 4) if x is Nan; return Nan - * 5) Range reduction: - * x = (32m + j)ln2 / 32 + r; r in [-ln2/64, ln2/64] - * 6) exp(r) - 1 is approximated by a polynomial function p(r) - * exp(x) = 2^m(2^(j/32) + 2^(j/32)p(r)); - */ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS -#if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) -static NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F void -AVX512F_exp_DOUBLE(npy_double * op, - npy_double * ip, - const npy_intp array_size, - const npy_intp steps) -{ - npy_intp num_remaining_elements = array_size; - const npy_intp stride = steps / (npy_intp)sizeof(npy_double); - const npy_int num_lanes = 64 / (npy_intp)sizeof(npy_double); - npy_int32 indexarr[8]; - for (npy_int32 ii = 0; ii < 8; ii++) { - indexarr[ii] = ii*stride; - } - - __m512d InvLn2N = _mm512_set1_pd(NPY_INV_LN2_MUL_32); - __m512d mShift = _mm512_set1_pd(NPY_RINT_CVT_MAGIC); - __m512d mNegL1 = _mm512_set1_pd(NPY_TANG_NEG_L1); - __m512d mNegL2 = _mm512_set1_pd(NPY_TANG_NEG_L2); - __m512i mMod = _mm512_set1_epi64(0x1f); - __m512d mA1 = _mm512_set1_pd(NPY_TANG_A1); - __m512d mA2 = _mm512_set1_pd(NPY_TANG_A2); - __m512d mA3 = _mm512_set1_pd(NPY_TANG_A3); - __m512d mA4 = _mm512_set1_pd(NPY_TANG_A4); - __m512d mA5 = _mm512_set1_pd(NPY_TANG_A5); - __m512d mTH_nearzero = _mm512_set1_pd(0x1p-54); - __m512d mTH_max = _mm512_set1_pd(0x1.62e42fefa39efp+9); - __m512d mTH_min = _mm512_set1_pd(-0x1.74910d52d3053p+9); - __m512d mTH_inf = _mm512_set1_pd(NPY_INFINITY); - __m512d zeros_d = _mm512_set1_pd(0.0f); - __m512d ones_d = _mm512_set1_pd(1.0f); - __m256i vindex = _mm256_loadu_si256((__m256i*)&indexarr[0]); - - __m512d mTable_top_0 = _mm512_loadu_pd(&(EXP_Table_top[8*0])); - __m512d mTable_top_1 = _mm512_loadu_pd(&(EXP_Table_top[8*1])); - __m512d mTable_top_2 = _mm512_loadu_pd(&(EXP_Table_top[8*2])); - __m512d mTable_top_3 = _mm512_loadu_pd(&(EXP_Table_top[8*3])); - __m512d mTable_tail_0 = _mm512_loadu_pd(&(EXP_Table_tail[8*0])); - __m512d mTable_tail_1 = _mm512_loadu_pd(&(EXP_Table_tail[8*1])); - __m512d mTable_tail_2 = _mm512_loadu_pd(&(EXP_Table_tail[8*2])); - __m512d mTable_tail_3 = _mm512_loadu_pd(&(EXP_Table_tail[8*3])); - - __mmask8 overflow_mask = avx512_get_partial_load_mask_pd(0, num_lanes); - __mmask8 load_mask = avx512_get_full_load_mask_pd(); - __mmask8 xmin_mask, xmax_mask, inf_mask, nan_mask, nearzero_mask; - - while (num_remaining_elements > 0) { - if (num_remaining_elements < num_lanes) { - load_mask = avx512_get_partial_load_mask_pd(num_remaining_elements, - num_lanes); - } - - __m512d x; - if (1 == stride) { - x = avx512_masked_load_pd(load_mask, ip); - } - else { - x = avx512_masked_gather_pd(zeros_d, ip, vindex, load_mask); - } - - nan_mask = _mm512_cmp_pd_mask(x, x, _CMP_NEQ_UQ); - x = avx512_set_masked_lanes_pd(x, zeros_d, nan_mask); - xmax_mask = _mm512_cmp_pd_mask(x, mTH_max, _CMP_GT_OQ); - xmin_mask = _mm512_cmp_pd_mask(x, mTH_min, _CMP_LT_OQ); - inf_mask = _mm512_cmp_pd_mask(x, mTH_inf, _CMP_EQ_OQ); - __m512i x_abs = _mm512_and_epi64(_mm512_castpd_si512(x), - _mm512_set1_epi64(0x7FFFFFFFFFFFFFFF)); - nearzero_mask = _mm512_cmp_pd_mask(_mm512_castsi512_pd(x_abs), - mTH_nearzero, _CMP_LT_OQ); - nearzero_mask = _mm512_kxor(nearzero_mask, nan_mask); - overflow_mask = _mm512_kor(overflow_mask, - _mm512_kxor(xmax_mask, inf_mask)); - x = avx512_set_masked_lanes_pd(x, zeros_d, - _mm512_kor(_mm512_kor(nan_mask, xmin_mask), - _mm512_kor(xmax_mask, nearzero_mask))); - - /* z = x * 32/ln2 */ - __m512d z = _mm512_mul_pd(x, InvLn2N); - - /* round to nearest */ - __m512d kd = _mm512_add_pd(z, mShift); - __m512i ki = _mm512_castpd_si512(kd); - kd = _mm512_sub_pd(kd, mShift); - - /* r = (x + kd*mNegL1) + kd*mNegL2 */ - __m512d r1 = _mm512_fmadd_pd(kd, mNegL1, x); - __m512d r2 = _mm512_mul_pd(kd, mNegL2); - __m512d r = _mm512_add_pd(r1,r2); - - /* Polynomial approximation for exp(r) - 1 */ - __m512d q = _mm512_fmadd_pd(mA5, r, mA4); - q = _mm512_fmadd_pd(q, r, mA3); - q = _mm512_fmadd_pd(q, r, mA2); - q = _mm512_fmadd_pd(q, r, mA1); - q = _mm512_mul_pd(q, r); - __m512d p = _mm512_fmadd_pd(r, q, r2);; - p = _mm512_add_pd(r1, p); - - /* Get 2^(j/32) from lookup table */ - __m512i j = _mm512_and_epi64(ki, mMod); - __m512d top = avx512_permute_x4var_pd(mTable_top_0, mTable_top_1, - mTable_top_2, mTable_top_3, j); - __m512d tail = avx512_permute_x4var_pd(mTable_tail_0, mTable_tail_1, - mTable_tail_2, mTable_tail_3, j); - - /* - * s = top + tail; - * exp(x) = 2^m * (top + (tail + s * p)); - */ - __m512d s = _mm512_add_pd(top, tail); - __m512d res = _mm512_fmadd_pd(s, p, tail); - res = _mm512_add_pd(res, top); - res= _mm512_scalef_pd(res, _mm512_div_pd(kd, _mm512_set1_pd(32))); - - /* return special cases */ - res = avx512_set_masked_lanes_pd(res, _mm512_add_pd(x, ones_d), - nearzero_mask); - res = avx512_set_masked_lanes_pd(res, _mm512_set1_pd(NPY_NAN), - nan_mask); - res = avx512_set_masked_lanes_pd(res, mTH_inf, xmax_mask); - res = avx512_set_masked_lanes_pd(res, zeros_d, xmin_mask); - - _mm512_mask_storeu_pd(op, load_mask, res); - - ip += num_lanes * stride; - op += num_lanes; - num_remaining_elements -= num_lanes; - } - if (overflow_mask) { - npy_set_floatstatus_overflow(); - } -} -#endif -#endif - -/* - * Vectorized implementation of log double using AVX512 - * Reference: - * [1] Tang, Ping Tak Peter. Table-lookup algorithms for elementary functions - * and their error analysis. No. CONF-9106103-1. Argonne National Lab., - * IL (USA), 1991. - * [2] Tang, Ping-Tak Peter. "Table-driven implementation of the logarithm - * function in IEEE floating-point arithmetic." ACM Transactions on - * Mathematical Software (TOMS) 16.4 (1990): 378-400. - * [3] Muller, Jean-Michel. "Elementary functions: algorithms and - * implementation." (2016). - * 1) if x = 0; return -INF - * 2) if x < 0; return NAN - * 3) if x is INF; return INF - * 4) if x is NAN; return NAN - * 5) if x on (1.0 - 0x1p-4, 1.0 + 0x1.09p-4), calling npy_log() - * 6) Range reduction: - * log(x) = log(2^m * z) - * = mln2 + log(z) - * 7) log(z) = log(z / c_k) + log(c_k); - * where c_k = 1 + k/64, k = 0,1,...,64 - * s.t. |x - c_k| <= 1/128 when x on[1,2]. - * 8) r = 2(x - c_k)/(x + c_k) - * log(x/c_k) = log((1 + r/2) / (1 - r/2)) - * = p(r) - * = 2((r/2) + 1/3*(r/2)^3 + 1/5*(r/2)^5 + ...) - */ -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS -#if !(defined(__clang__) && (__clang_major__ < 10 || (__clang_major__ == 10 && __clang_minor__ < 1))) -static NPY_GCC_OPT_3 NPY_GCC_TARGET_AVX512F void -AVX512F_log_DOUBLE(npy_double * op, - npy_double * ip, - const npy_intp array_size, - const npy_intp steps) -{ - npy_intp num_remaining_elements = array_size; - const npy_intp stride = steps / (npy_intp)sizeof(npy_double); - const npy_int num_lanes = 64 / (npy_intp)sizeof(npy_double); - npy_int32 indexarr[8]; - for (npy_int32 ii = 0; ii < 8; ii++) { - indexarr[ii] = ii*stride; - } - - __m512d zeros_d = _mm512_set1_pd(0.0f); - __m512d ones_d = _mm512_set1_pd(1.0f); - __m512d mInf = _mm512_set1_pd(NPY_INFINITY); - __m512d mInv64 = (__m512d)(_mm512_set1_epi64(0x3f90000000000000)); - __m512d mNeg_nan = _mm512_set1_pd(-NPY_NAN); - __m512d mNan = _mm512_set1_pd(NPY_NAN); - __m512d mNeg_inf = _mm512_set1_pd(-NPY_INFINITY); - __m512d mA1 = _mm512_set1_pd(NPY_TANG_LOG_A1); - __m512d mA2 = _mm512_set1_pd(NPY_TANG_LOG_A2); - __m512d mA3 = _mm512_set1_pd(NPY_TANG_LOG_A3); - __m512d mA4 = _mm512_set1_pd(NPY_TANG_LOG_A4); - __m512d mLN2HI = _mm512_set1_pd(NPY_TANG_LOG_LN2HI); - __m512d mLN2LO = _mm512_set1_pd(NPY_TANG_LOG_LN2LO); - - __m512d mTo_glibc_min = _mm512_set1_pd(1.0 - 0x1p-4); - __m512d mTo_glibc_max = _mm512_set1_pd(1.0 + 0x1.09p-4); - __m256i vindex = _mm256_loadu_si256((__m256i*)&indexarr[0]); - - /* Load lookup table data */ - /**begin repeat - * #i = 0, 1, 2, 3, 4, 5, 6, 7# - */ - - __m512d mLUT_TOP_@i@ = _mm512_loadu_pd(&(LOG_TABLE_TOP[8*@i@])); - __m512d mLUT_TAIL_@i@ = _mm512_loadu_pd(&(LOG_TABLE_TAIL[8*@i@])); - - /**end repeat**/ - - __mmask8 load_mask = avx512_get_full_load_mask_pd(); - __mmask8 invalid_mask = avx512_get_partial_load_mask_pd(0, num_lanes); - __mmask8 divide_by_zero_mask = invalid_mask; - - __mmask8 inf_mask, nan_mask, zero_mask, negx_mask, denormal_mask, - glibc_mask; - - __m512d x_in; - while (num_remaining_elements > 0) { - if (num_remaining_elements < num_lanes) { - load_mask = avx512_get_partial_load_mask_pd(num_remaining_elements, - num_lanes); - } - - if (1 == stride) { - x_in = avx512_masked_load_pd(load_mask, ip); - } - else { - x_in = avx512_masked_gather_pd(zeros_d, ip, vindex, load_mask); - } - - /* call glibc when x on [1.0 - 0x1p-4, 1.0 + 0x1.09p-4] */ - __mmask8 m1 = _mm512_cmp_pd_mask(x_in, mTo_glibc_max, _CMP_LT_OQ); - __mmask8 m2 = _mm512_cmp_pd_mask(x_in, mTo_glibc_min, _CMP_GT_OQ); - glibc_mask = m1 & m2; - - if (glibc_mask != 0xFF) { - zero_mask = _mm512_cmp_pd_mask(x_in, zeros_d, _CMP_EQ_OQ); - inf_mask = _mm512_cmp_pd_mask(x_in, mInf, _CMP_EQ_OQ); - negx_mask = _mm512_cmp_pd_mask(x_in, zeros_d, _CMP_LT_OQ); - nan_mask = _mm512_cmp_pd_mask(x_in, x_in, _CMP_NEQ_UQ); - - divide_by_zero_mask = divide_by_zero_mask | (zero_mask & load_mask); - invalid_mask = invalid_mask | negx_mask; - - __m512d x = avx512_set_masked_lanes_pd(x_in, zeros_d, negx_mask); - __m512i ix = (__m512i)x; - - /* Normalize x when it is denormal */ - __m512i top12 = _mm512_and_epi64(ix, - _mm512_set1_epi64(0xfff0000000000000)); - denormal_mask = _mm512_cmp_epi64_mask(top12, _mm512_set1_epi64(0), - _CMP_EQ_OQ); - denormal_mask = (~zero_mask) & denormal_mask; - ix = (__m512i)_mm512_mask_mul_pd(x, denormal_mask, - x, _mm512_set1_pd(0x1p52)); - ix = _mm512_mask_sub_epi64(ix, denormal_mask, - ix, _mm512_set1_epi64(52ULL << 52)); - - /* - * x = 2^k * z; where z in range [1,2] - */ - __m512i tmp = _mm512_sub_epi64(ix, - _mm512_set1_epi64(0x3ff0000000000000)); - __m512i i = _mm512_and_epi64(_mm512_srai_epi64(tmp, 52 - 6), - _mm512_set1_epi64(0x3fULL)); - __m512i ik = _mm512_srai_epi64(tmp, 52); - __m512d z = (__m512d)(_mm512_sub_epi64(ix, _mm512_and_epi64(tmp, - _mm512_set1_epi64(0xfff0000000000000)))); - /* c = i/64 + 1 */ - __m256i i_32 = _mm512_cvtepi64_epi32(i); - __m512d c = _mm512_fmadd_pd(_mm512_cvtepi32_pd(i_32), mInv64, ones_d); - - /* u = 2 * (z - c) / (z + c) */ - __m512d u = _mm512_div_pd(_mm512_sub_pd(z, c), _mm512_add_pd(z, c)); - u = _mm512_mul_pd(_mm512_set1_pd(2.0), u); - - /* v = u * u */ - __m512d v = _mm512_mul_pd(u,u); - - /* log(z/c) = u + u*v*(A1 + v*(A2 + v*(A3 + v*A4))) */ - __m512d res = _mm512_fmadd_pd(v, mA4, mA3); - res = _mm512_fmadd_pd(v, res, mA2); - res = _mm512_fmadd_pd(v, res, mA1); - res = _mm512_mul_pd(v, res); - res = _mm512_fmadd_pd(u, res, u); - - /* Load lookup table data */ - __m512d c_hi = avx512_permute_x8var_pd(mLUT_TOP_0, mLUT_TOP_1, - mLUT_TOP_2, mLUT_TOP_3, mLUT_TOP_4, mLUT_TOP_5, - mLUT_TOP_6, mLUT_TOP_7, i); - __m512d c_lo = avx512_permute_x8var_pd(mLUT_TAIL_0, mLUT_TAIL_1, - mLUT_TAIL_2, mLUT_TAIL_3, mLUT_TAIL_4, mLUT_TAIL_5, - mLUT_TAIL_6, mLUT_TAIL_7, i); - - /* - * log(x) = k * ln2_hi + c_hi + - * k * ln2_lo + c_lo + - * log(z/c) - */ - __m256i ik_32 = _mm512_cvtepi64_epi32(ik); - __m512d k = _mm512_cvtepi32_pd(ik_32); - __m512d tt = _mm512_fmadd_pd(k, mLN2HI, c_hi); - __m512d tt2 = _mm512_fmadd_pd(k, mLN2LO, c_lo); - tt = _mm512_add_pd(tt, tt2); - res = _mm512_add_pd(tt, res); - - /* return special cases */ - res = avx512_set_masked_lanes_pd(res, mNan, nan_mask); - res = avx512_set_masked_lanes_pd(res, mNeg_nan, negx_mask); - res = avx512_set_masked_lanes_pd(res, mNeg_inf, zero_mask); - res = avx512_set_masked_lanes_pd(res, mInf, inf_mask); - - _mm512_mask_storeu_pd(op, load_mask, res); - } - - /* call glibc's log func when x around 1.0f */ - if (glibc_mask != 0) { - double NPY_DECL_ALIGNED(64) ip_fback[8]; - _mm512_store_pd(ip_fback, x_in); - - for (int ii = 0; ii < 8; ++ii, glibc_mask >>= 1) { - if (glibc_mask & 0x01) { - op[ii] = npy_log(ip_fback[ii]); - } - } - } - ip += num_lanes * stride; - op += num_lanes; - num_remaining_elements -= num_lanes; - } - - if (invalid_mask) { - npy_set_floatstatus_invalid(); - } - if (divide_by_zero_mask) { - npy_set_floatstatus_divbyzero(); - } -} -#endif -#endif - /**begin repeat * #TYPE = CFLOAT, CDOUBLE# * #type = npy_float, npy_double# @@ -3347,45 +1482,6 @@ AVX512F_log_DOUBLE(npy_double * op, * #IS_FLOAT = 1, 0# */ -/**begin repeat1 - * #func = add, subtract, multiply# - * #vectorf = _mm512_add, _mm512_sub, avx512_cmul# - */ - -#if defined HAVE_ATTRIBUTE_TARGET_AVX512F_WITH_INTRINSICS && defined NPY_HAVE_SSE2_INTRINSICS -static NPY_GCC_OPT_3 NPY_INLINE NPY_GCC_TARGET_AVX512F void -AVX512F_@func@_@TYPE@(char **args, const npy_intp *dimensions, const npy_intp *steps) -{ - const npy_intp array_size = dimensions[0]; - npy_intp num_remaining_elements = 2*array_size; - @type@* ip1 = (@type@*) args[0]; - @type@* ip2 = (@type@*) args[1]; - @type@* op = (@type@*) args[2]; - - @mask@ load_mask = avx512_get_full_load_mask_@vsuffix@(); - - while (num_remaining_elements > 0) { - if (num_remaining_elements < @num_lanes@) { - load_mask = avx512_get_partial_load_mask_@vsuffix@( - num_remaining_elements, @num_lanes@); - } - @vtype@ x1, x2; - x1 = avx512_masked_load_@vsuffix@(load_mask, ip1); - x2 = avx512_masked_load_@vsuffix@(load_mask, ip2); - - @vtype@ out = @vectorf@_@vsuffix@(x1, x2); - - _mm512_mask_storeu_@vsuffix@(op, load_mask, out); - - ip1 += @num_lanes@; - ip2 += @num_lanes@; - op += @num_lanes@; - num_remaining_elements -= @num_lanes@; - } -} -#endif -/**end repeat1**/ - /**begin repeat1 * #func = square, conjugate# * #vectorf = avx512_csquare, avx512_conjugate# @@ -3648,86 +1744,5 @@ sse2_@kind@_BOOL(@type@ * op, @type@ * ip, const npy_intp n) /**end repeat**/ #undef VECTOR_SIZE_BYTES -#else /* NPY_HAVE_SSE2_INTRINSICS */ - -/**begin repeat - * #type = npy_float, npy_double# - * #TYPE = FLOAT, DOUBLE# - * #sfx = f32, f64# - * #CHK = , _F64# - */ - -#if NPY_SIMD@CHK@ - -/**begin repeat1 -* Arithmetic -* # kind = add, subtract, multiply, divide# -* # OP = +, -, *, /# -* # VOP = add, sub, mul, div# -*/ - -static void -simd_binary_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) -{ - LOOP_BLOCK_ALIGN_VAR(op, @type@, NPY_SIMD_WIDTH) { - op[i] = ip1[i] @OP@ ip2[i]; - } - /* lots of specializations, to squeeze out max performance */ - if (ip1 == ip2) { - LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { - npyv_@sfx@ a = npyv_load_@sfx@(&ip1[i]); - npyv_@sfx@ c = npyv_@VOP@_@sfx@(a, a); - npyv_store_@sfx@(&op[i], c); - } - } - else { - LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { - npyv_@sfx@ a = npyv_load_@sfx@(&ip1[i]); - npyv_@sfx@ b = npyv_load_@sfx@(&ip2[i]); - npyv_@sfx@ c = npyv_@VOP@_@sfx@(a, b); - npyv_store_@sfx@(&op[i], c); - } - } - LOOP_BLOCKED_END { - op[i] = ip1[i] @OP@ ip2[i]; - } -} - -static void -simd_binary_scalar1_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) -{ - const npyv_@sfx@ v1 = npyv_setall_@sfx@(ip1[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, NPY_SIMD_WIDTH) { - op[i] = ip1[0] @OP@ ip2[i]; - } - LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { - npyv_@sfx@ v2 = npyv_load_@sfx@(&ip2[i]); - npyv_@sfx@ v3 = npyv_@VOP@_@sfx@(v1, v2); - npyv_store_@sfx@(&op[i], v3); - } - LOOP_BLOCKED_END { - op[i] = ip1[0] @OP@ ip2[i]; - } -} - -static void -simd_binary_scalar2_@kind@_@TYPE@(@type@ * op, @type@ * ip1, @type@ * ip2, npy_intp n) -{ - const npyv_@sfx@ v2 = npyv_setall_@sfx@(ip2[0]); - LOOP_BLOCK_ALIGN_VAR(op, @type@, NPY_SIMD_WIDTH) { - op[i] = ip1[i] @OP@ ip2[0]; - } - LOOP_BLOCKED(@type@, NPY_SIMD_WIDTH) { - npyv_@sfx@ v1 = npyv_load_@sfx@(&ip1[i]); - npyv_@sfx@ v3 = npyv_@VOP@_@sfx@(v1, v2); - npyv_store_@sfx@(&op[i], v3); - } - LOOP_BLOCKED_END { - op[i] = ip1[i] @OP@ ip2[0]; - } -} -/**end repeat1**/ -#endif /* NPY_SIMD@CHK@ */ -/**end repeat**/ -#endif +#endif /* NPY_HAVE_SSE2_INTRINSICS */ #endif diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 1a035eb619e2..0644a28c011b 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -28,10 +28,11 @@ #define NPY_NO_DEPRECATED_API NPY_API_VERSION #include "Python.h" +#include "stddef.h" #include "npy_config.h" - #include "npy_pycompat.h" +#include "npy_argparse.h" #include "numpy/arrayobject.h" #include "numpy/ufuncobject.h" @@ -46,6 +47,7 @@ #include "npy_import.h" #include "extobj.h" #include "common.h" +#include "dtypemeta.h" #include "numpyos.h" /********** PRINTF DEBUG TRACING **************/ @@ -95,8 +97,9 @@ _get_wrap_prepare_args(ufunc_full_args full_args) { /* ---------------------------------------------------------------- */ -static int -_does_loop_use_arrays(void *data); +static PyObject * +prepare_input_arguments_for_outer(PyObject *args, PyUFuncObject *ufunc); + /*UFUNC_API*/ NPY_NO_EXPORT int @@ -270,7 +273,7 @@ _get_output_array_method(PyObject *obj, PyObject *method, */ static void _find_array_prepare(ufunc_full_args args, - PyObject **output_prep, int nin, int nout) + PyObject **output_prep, int nout) { int i; PyObject *prep; @@ -397,27 +400,19 @@ _ufunc_setup_flags(PyUFuncObject *ufunc, npy_uint32 op_in_flags, * A NULL is placed in output_wrap for outputs that * should just have PyArray_Return called. */ -static int -_find_array_wrap(ufunc_full_args args, PyObject *kwds, - PyObject **output_wrap, int nin, int nout) +static void +_find_array_wrap(ufunc_full_args args, npy_bool subok, + PyObject **output_wrap, int nin, int nout) { int i; - PyObject *obj; PyObject *wrap = NULL; /* * If a 'subok' parameter is passed and isn't True, don't wrap but put None * into slots with out arguments which means return the out argument */ - if (kwds != NULL) { - obj = PyDict_GetItemWithError(kwds, npy_um_str_subok); - if (obj == NULL && PyErr_Occurred()) { - return -1; - } - else if (obj != NULL && obj != Py_True) { - /* skip search for wrap members */ - goto handle_out; - } + if (!subok) { + goto handle_out; } /* @@ -453,7 +448,6 @@ _find_array_wrap(ufunc_full_args args, PyObject *kwds, } Py_XDECREF(wrap); - return 0; } @@ -849,103 +843,12 @@ ufunc_get_name_cstr(PyUFuncObject *ufunc) { return ufunc->name ? ufunc->name : ""; } -/* - * Helpers for keyword parsing - */ - -/* - * Find key in a list of pointers to keyword names. - * The list should end with NULL. - * - * Returns either the index into the list (pointing to the final key with NULL - * if no match was found), or -1 on failure. - */ -static npy_intp -locate_key(PyObject **kwnames, PyObject *key) -{ - PyObject **kwname = kwnames; - while (*kwname != NULL && *kwname != key) { - kwname++; - } - /* Slow fallback, just in case */ - if (*kwname == NULL) { - int cmp = 0; - kwname = kwnames; - while (*kwname != NULL && - (cmp = PyObject_RichCompareBool(key, *kwname, - Py_EQ)) == 0) { - kwname++; - } - if (cmp < 0) { - return -1; - } - } - return kwname - kwnames; -} - -/* - * Parse keyword arguments, matching against kwnames - * - * Arguments beyond kwnames (the va_list) should contain converters and outputs - * for each keyword name (where an output can be NULL to indicate the particular - * keyword should be ignored). - * - * Returns 0 on success, -1 on failure with an error set. - * - * Note that the parser does not clean up on failure, i.e., already parsed keyword - * values may hold new references, which the caller has to remove. - * - * TODO: ufunc is only used for the name in error messages; passing on the - * name instead might be an option. - * - * TODO: instead of having this function ignore of keywords for which the - * corresponding output is NULL, the calling routine should prepare the - * correct list. - */ -static int -parse_ufunc_keywords(PyUFuncObject *ufunc, PyObject *kwds, PyObject **kwnames, ...) -{ - va_list va; - PyObject *key, *value; - Py_ssize_t pos = 0; - typedef int converter(PyObject *, void *); - - while (PyDict_Next(kwds, &pos, &key, &value)) { - npy_intp i; - converter *convert; - void *output = NULL; - npy_intp index = locate_key(kwnames, key); - if (index < 0) { - return -1; - } - if (kwnames[index]) { - va_start(va, kwnames); - for (i = 0; i <= index; i++) { - convert = va_arg(va, converter *); - output = va_arg(va, void *); - } - va_end(va); - } - if (output) { - if (!convert(value, output)) { - return -1; - } - } - else { - PyErr_Format(PyExc_TypeError, - "'%S' is an invalid keyword to ufunc '%s'", - key, ufunc_get_name_cstr(ufunc)); - return -1; - } - } - return 0; -} /* * Converters for use in parsing of keywords arguments. */ -NPY_NO_EXPORT int -_subok_converter(PyObject *obj, int *subok) +static int +_subok_converter(PyObject *obj, npy_bool *subok) { if (PyBool_Check(obj)) { *subok = (obj == Py_True); @@ -958,7 +861,7 @@ _subok_converter(PyObject *obj, int *subok) } } -NPY_NO_EXPORT int +static int _keepdims_converter(PyObject *obj, int *keepdims) { if (PyBool_Check(obj)) { @@ -972,7 +875,7 @@ _keepdims_converter(PyObject *obj, int *keepdims) } } -NPY_NO_EXPORT int +static int _wheremask_converter(PyObject *obj, PyArrayObject **wheremask) { /* @@ -996,75 +899,32 @@ _wheremask_converter(PyObject *obj, PyArrayObject **wheremask) } } -NPY_NO_EXPORT int -_new_reference(PyObject *obj, PyObject **out) -{ - Py_INCREF(obj); - *out = obj; - return NPY_SUCCEED; -} - -NPY_NO_EXPORT int -_borrowed_reference(PyObject *obj, PyObject **out) -{ - *out = obj; - return NPY_SUCCEED; -} /* - * Parses the positional and keyword arguments for a generic ufunc call. - * All returned arguments are new references (with optional ones NULL - * if not present) + * Due to the array override, do the actual parameter conversion + * only in this step. This function takes the reference objects and + * parses them into the desired values. + * This function cleans up after itself and NULLs references on error, + * however, the caller has to ensure that `out_op[0:nargs]` and `out_whermeask` + * are NULL initialized. */ static int -get_ufunc_arguments(PyUFuncObject *ufunc, - PyObject *args, PyObject *kwds, - PyArrayObject **out_op, - NPY_ORDER *out_order, - NPY_CASTING *out_casting, - PyObject **out_extobj, - PyObject **out_typetup, /* type: Tuple[np.dtype] */ - int *out_subok, /* bool */ - PyArrayObject **out_wheremask, /* PyArray of bool */ - PyObject **out_axes, /* type: List[Tuple[T]] */ - PyObject **out_axis, /* type: T */ - int *out_keepdims) /* bool */ +convert_ufunc_arguments(PyUFuncObject *ufunc, + ufunc_full_args full_args, PyArrayObject **out_op, + PyObject *order_obj, NPY_ORDER *out_order, + PyObject *casting_obj, NPY_CASTING *out_casting, + PyObject *subok_obj, npy_bool *out_subok, + PyObject *where_obj, PyArrayObject **out_wheremask, /* PyArray of bool */ + PyObject *keepdims_obj, int *out_keepdims) { - int i, nargs; int nin = ufunc->nin; int nout = ufunc->nout; int nop = ufunc->nargs; PyObject *obj; - PyArray_Descr *dtype = NULL; - /* - * Initialize output objects so caller knows when outputs and optional - * arguments are set (also means we can safely XDECREF on failure). - */ - for (i = 0; i < nop; i++) { - out_op[i] = NULL; - } - *out_extobj = NULL; - *out_typetup = NULL; - if (out_axes != NULL) { - *out_axes = NULL; - } - if (out_axis != NULL) { - *out_axis = NULL; - } - if (out_wheremask != NULL) { - *out_wheremask = NULL; - } - /* Check number of arguments */ - nargs = PyTuple_Size(args); - if ((nargs < nin) || (nargs > nop)) { - PyErr_SetString(PyExc_ValueError, "invalid number of arguments"); - return -1; - } - - /* Get input arguments */ - for (i = 0; i < nin; ++i) { - obj = PyTuple_GET_ITEM(args, i); + /* Convert and fill in input arguments */ + for (int i = 0; i < nin; i++) { + obj = PyTuple_GET_ITEM(full_args.in, i); if (PyArray_Check(obj)) { PyArrayObject *obj_a = (PyArrayObject *)obj; @@ -1080,148 +940,43 @@ get_ufunc_arguments(PyUFuncObject *ufunc, } } - /* Get positional output arguments */ - for (i = nin; i < nargs; ++i) { - obj = PyTuple_GET_ITEM(args, i); - if (_set_out_array(obj, out_op + i) < 0) { - goto fail; + /* Convert and fill in output arguments */ + if (full_args.out != NULL) { + for (int i = 0; i < nout; i++) { + obj = PyTuple_GET_ITEM(full_args.out, i); + if (_set_out_array(obj, out_op + i + nin) < 0) { + goto fail; + } } } /* - * If keywords are present, get keyword output and other arguments. - * Raise an error if anything else is present in the keyword dictionary. + * Convert most arguments manually here, since it is easier to handle + * the ufunc override if we first parse only to objects. */ - if (kwds) { - PyObject *out_kwd = NULL; - PyObject *sig = NULL; - static PyObject *kwnames[13] = {NULL}; - if (kwnames[0] == NULL) { - kwnames[0] = npy_um_str_out; - kwnames[1] = npy_um_str_where; - kwnames[2] = npy_um_str_axes; - kwnames[3] = npy_um_str_axis; - kwnames[4] = npy_um_str_keepdims; - kwnames[5] = npy_um_str_casting; - kwnames[6] = npy_um_str_order; - kwnames[7] = npy_um_str_dtype; - kwnames[8] = npy_um_str_subok; - kwnames[9] = npy_um_str_signature; - kwnames[10] = npy_um_str_sig; - kwnames[11] = npy_um_str_extobj; - kwnames[12] = NULL; /* sentinel */ - } - /* - * Parse using converters to calculate outputs - * (NULL outputs are treated as indicating a keyword is not allowed). - */ - if (parse_ufunc_keywords( - ufunc, kwds, kwnames, - _borrowed_reference, &out_kwd, - _wheremask_converter, out_wheremask, /* new reference */ - _new_reference, out_axes, - _new_reference, out_axis, - _keepdims_converter, out_keepdims, - PyArray_CastingConverter, out_casting, - PyArray_OrderConverter, out_order, - PyArray_DescrConverter2, &dtype, /* new reference */ - _subok_converter, out_subok, - _new_reference, out_typetup, - _borrowed_reference, &sig, - _new_reference, out_extobj) < 0) { - goto fail; - } - /* - * Check that outputs were not passed as positional as well, - * and that they are either None or an array. - */ - if (out_kwd) { /* borrowed reference */ - /* - * Output arrays are generally specified as a tuple of arrays - * and None, but may be a single array or None for ufuncs - * with a single output. - */ - if (nargs > nin) { - PyErr_SetString(PyExc_ValueError, - "cannot specify 'out' as both a " - "positional and keyword argument"); - goto fail; - } - if (PyTuple_CheckExact(out_kwd)) { - if (PyTuple_GET_SIZE(out_kwd) != nout) { - PyErr_SetString(PyExc_ValueError, - "The 'out' tuple must have exactly " - "one entry per ufunc output"); - goto fail; - } - /* 'out' must be a tuple of arrays and Nones */ - for(i = 0; i < nout; ++i) { - PyObject *val = PyTuple_GET_ITEM(out_kwd, i); - if (_set_out_array(val, out_op+nin+i) < 0) { - goto fail; - } - } - } - else if (nout == 1) { - /* Can be an array if it only has one output */ - if (_set_out_array(out_kwd, out_op + nin) < 0) { - goto fail; - } - } - else { - PyErr_SetString(PyExc_TypeError, - nout > 1 ? "'out' must be a tuple of arrays" : - "'out' must be an array or a tuple with " - "a single array"); - goto fail; - } - } - /* - * Check we did not get both axis and axes, or multiple ways - * to define a signature. - */ - if (out_axes != NULL && out_axis != NULL && - *out_axes != NULL && *out_axis != NULL) { - PyErr_SetString(PyExc_TypeError, - "cannot specify both 'axis' and 'axes'"); - goto fail; - } - if (sig) { /* borrowed reference */ - if (*out_typetup != NULL) { - PyErr_SetString(PyExc_ValueError, - "cannot specify both 'sig' and 'signature'"); - goto fail; - } - Py_INCREF(sig); - *out_typetup = sig; - } - if (dtype) { /* new reference */ - if (*out_typetup != NULL) { - PyErr_SetString(PyExc_RuntimeError, - "cannot specify both 'signature' and 'dtype'"); - goto fail; - } - /* Note: "N" uses the reference */ - *out_typetup = Py_BuildValue("(N)", dtype); - } + if (where_obj && !_wheremask_converter(where_obj, out_wheremask)) { + goto fail; + } + if (keepdims_obj && !_keepdims_converter(keepdims_obj, out_keepdims)) { + goto fail; + } + if (casting_obj && !PyArray_CastingConverter(casting_obj, out_casting)) { + goto fail; + } + if (order_obj && !PyArray_OrderConverter(order_obj, out_order)) { + goto fail; + } + if (subok_obj && !_subok_converter(subok_obj, out_subok)) { + goto fail; } return 0; fail: - Py_XDECREF(dtype); - Py_XDECREF(*out_typetup); - Py_XDECREF(*out_extobj); if (out_wheremask != NULL) { - Py_XDECREF(*out_wheremask); - } - if (out_axes != NULL) { - Py_XDECREF(*out_axes); + Py_XSETREF(*out_wheremask, NULL); } - if (out_axis != NULL) { - Py_XDECREF(*out_axis); - } - for (i = 0; i < nop; i++) { - Py_XDECREF(out_op[i]); + for (int i = 0; i < nop; i++) { + Py_XSETREF(out_op[i], NULL); } return -1; } @@ -1278,65 +1033,6 @@ check_for_trivial_loop(PyUFuncObject *ufunc, return 1; } -static void -trivial_two_operand_loop(PyArrayObject **op, - PyUFuncGenericFunction innerloop, - void *innerloopdata) -{ - char *data[2]; - npy_intp count[2], stride[2]; - int needs_api; - NPY_BEGIN_THREADS_DEF; - - needs_api = PyDataType_REFCHK(PyArray_DESCR(op[0])) || - PyDataType_REFCHK(PyArray_DESCR(op[1])); - - PyArray_PREPARE_TRIVIAL_PAIR_ITERATION(op[0], op[1], - count[0], - data[0], data[1], - stride[0], stride[1]); - count[1] = count[0]; - NPY_UF_DBG_PRINT1("two operand loop count %d\n", (int)count[0]); - - if (!needs_api) { - NPY_BEGIN_THREADS_THRESHOLDED(count[0]); - } - - innerloop(data, count, stride, innerloopdata); - - NPY_END_THREADS; -} - -static void -trivial_three_operand_loop(PyArrayObject **op, - PyUFuncGenericFunction innerloop, - void *innerloopdata) -{ - char *data[3]; - npy_intp count[3], stride[3]; - int needs_api; - NPY_BEGIN_THREADS_DEF; - - needs_api = PyDataType_REFCHK(PyArray_DESCR(op[0])) || - PyDataType_REFCHK(PyArray_DESCR(op[1])) || - PyDataType_REFCHK(PyArray_DESCR(op[2])); - - PyArray_PREPARE_TRIVIAL_TRIPLE_ITERATION(op[0], op[1], op[2], - count[0], - data[0], data[1], data[2], - stride[0], stride[1], stride[2]); - count[1] = count[0]; - count[2] = count[0]; - NPY_UF_DBG_PRINT1("three operand loop count %d\n", (int)count[0]); - - if (!needs_api) { - NPY_BEGIN_THREADS_THRESHOLDED(count[0]); - } - - innerloop(data, count, stride, innerloopdata); - - NPY_END_THREADS; -} /* * Calls the given __array_prepare__ function on the operand *op, @@ -1410,6 +1106,151 @@ prepare_ufunc_output(PyUFuncObject *ufunc, return 0; } + +/* + * Check whether a trivial loop is possible and call the innerloop if it is. + * A trivial loop is defined as one where a single strided inner-loop call + * is possible. + * + * This function only supports a single output (due to the overlap check). + * It always accepts 0-D arrays and will broadcast them. The function + * cannot broadcast any other array (as it requires a single stride). + * The function accepts all 1-D arrays, and N-D arrays that are either all + * C- or all F-contiguous. + * + * Returns -2 if a trivial loop is not possible, 0 on success and -1 on error. + */ +static NPY_INLINE int +try_trivial_single_output_loop(PyUFuncObject *ufunc, + PyArrayObject *op[], PyArray_Descr *dtypes[], + NPY_ORDER order, PyObject *arr_prep[], ufunc_full_args full_args, + PyUFuncGenericFunction innerloop, void *innerloopdata) +{ + int nin = ufunc->nin; + int nop = nin + 1; + assert(ufunc->nout == 1); + + /* The order of all N-D contiguous operands, can be fixed by `order` */ + int operation_order = 0; + if (order == NPY_CORDER) { + operation_order = NPY_ARRAY_C_CONTIGUOUS; + } + else if (order == NPY_FORTRANORDER) { + operation_order = NPY_ARRAY_F_CONTIGUOUS; + } + + int operation_ndim = 0; + npy_intp *operation_shape = NULL; + npy_intp fixed_strides[NPY_MAXARGS]; + + for (int iop = 0; iop < nop; iop++) { + if (op[iop] == NULL) { + /* The out argument may be NULL (and only that one); fill later */ + assert(iop == nin); + continue; + } + + int op_ndim = PyArray_NDIM(op[iop]); + + /* Special case 0-D since we can handle broadcasting using a 0-stride */ + if (op_ndim == 0) { + fixed_strides[iop] = 0; + continue; + } + + /* First non 0-D op: fix dimensions, shape (order is fixed later) */ + if (operation_ndim == 0) { + operation_ndim = op_ndim; + operation_shape = PyArray_SHAPE(op[iop]); + } + else if (op_ndim != operation_ndim) { + return -2; /* dimension mismatch (except 0-d ops) */ + } + else if (!PyArray_CompareLists( + operation_shape, PyArray_DIMS(op[iop]), op_ndim)) { + return -2; /* shape mismatch */ + } + + if (op_ndim == 1) { + fixed_strides[iop] = PyArray_STRIDES(op[iop])[0]; + } + else { + fixed_strides[iop] = PyArray_ITEMSIZE(op[iop]); /* contiguous */ + + /* This op must match the operation order (and be contiguous) */ + int op_order = (PyArray_FLAGS(op[iop]) & + (NPY_ARRAY_C_CONTIGUOUS|NPY_ARRAY_F_CONTIGUOUS)); + if (op_order == 0) { + return -2; /* N-dimensional op must be contiguous */ + } + else if (operation_order == 0) { + operation_order = op_order; /* op fixes order */ + } + else if (operation_order != op_order) { + return -2; + } + } + } + + if (op[nin] == NULL) { + Py_INCREF(dtypes[nin]); + op[nin] = (PyArrayObject *) PyArray_NewFromDescr(&PyArray_Type, + dtypes[nin], operation_ndim, operation_shape, + NULL, NULL, operation_order==NPY_ARRAY_F_CONTIGUOUS, NULL); + if (op[nin] == NULL) { + return -1; + } + fixed_strides[nin] = dtypes[nin]->elsize; + } + else { + /* If any input overlaps with the output, we use the full path. */ + for (int iop = 0; iop < nin; iop++) { + if (!PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK( + op[iop], op[nin], + PyArray_TRIVIALLY_ITERABLE_OP_READ, + PyArray_TRIVIALLY_ITERABLE_OP_NOREAD)) { + return -2; + } + } + /* Check self-overlap (non 1-D are contiguous, perfect overlap is OK) */ + if (operation_ndim == 1 && + PyArray_STRIDES(op[nin])[0] < PyArray_ITEMSIZE(op[nin]) && + PyArray_STRIDES(op[nin])[0] != 0) { + return -2; + } + } + + /* Call the __prepare_array__ if necessary */ + if (prepare_ufunc_output(ufunc, &op[nin], + arr_prep[0], full_args, 0) < 0) { + return -1; + } + + /* + * We can use the trivial (single inner-loop call) optimization + * and `fixed_strides` holds the strides for that call. + */ + char *data[NPY_MAXARGS]; + npy_intp count = PyArray_MultiplyList(operation_shape, operation_ndim); + int needs_api = 0; + NPY_BEGIN_THREADS_DEF; + + for (int iop = 0; iop < nop; iop++) { + data[iop] = PyArray_BYTES(op[iop]); + needs_api |= PyDataType_REFCHK(dtypes[iop]); + } + + if (!needs_api) { + NPY_BEGIN_THREADS_THRESHOLDED(count); + } + + innerloop(data, &count, fixed_strides, innerloopdata); + + NPY_END_THREADS; + return 0; +} + + static int iterator_loop(PyUFuncObject *ufunc, PyArrayObject **op, @@ -1431,6 +1272,7 @@ iterator_loop(PyUFuncObject *ufunc, char **dataptr; npy_intp *stride; npy_intp *count_ptr; + int needs_api; PyArrayObject **op_it; npy_uint32 iter_flags; @@ -1525,6 +1367,7 @@ iterator_loop(PyUFuncObject *ufunc, dataptr = NpyIter_GetDataPtrArray(iter); stride = NpyIter_GetInnerStrideArray(iter); count_ptr = NpyIter_GetInnerLoopSizePtr(iter); + needs_api = NpyIter_IterationNeedsAPI(iter); NPY_BEGIN_THREADS_NDITER(iter); @@ -1532,7 +1375,7 @@ iterator_loop(PyUFuncObject *ufunc, do { NPY_UF_DBG_PRINT1("iterator loop count %d\n", (int)*count_ptr); innerloop(dataptr, count_ptr, stride, innerloopdata); - } while (iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); NPY_END_THREADS; } @@ -1568,7 +1411,6 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, ufunc_full_args full_args, npy_uint32 *op_flags) { - npy_intp nin = ufunc->nin, nout = ufunc->nout; PyUFuncGenericFunction innerloop; void *innerloopdata; int needs_api = 0; @@ -1577,119 +1419,14 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, &innerloop, &innerloopdata, &needs_api) < 0) { return -1; } - /* If the loop wants the arrays, provide them. */ - if (_does_loop_use_arrays(innerloopdata)) { - innerloopdata = (void*)op; - } /* First check for the trivial cases that don't need an iterator */ - if (trivial_loop_ok) { - if (nin == 1 && nout == 1) { - if (op[1] == NULL && - (order == NPY_ANYORDER || order == NPY_KEEPORDER) && - PyArray_TRIVIALLY_ITERABLE(op[0])) { - Py_INCREF(dtypes[1]); - op[1] = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, - dtypes[1], - PyArray_NDIM(op[0]), - PyArray_DIMS(op[0]), - NULL, NULL, - PyArray_ISFORTRAN(op[0]) ? - NPY_ARRAY_F_CONTIGUOUS : 0, - NULL); - if (op[1] == NULL) { - return -1; - } - - /* Call the __prepare_array__ if necessary */ - if (prepare_ufunc_output(ufunc, &op[1], - arr_prep[0], full_args, 0) < 0) { - return -1; - } - - NPY_UF_DBG_PRINT("trivial 1 input with allocated output\n"); - trivial_two_operand_loop(op, innerloop, innerloopdata); - - return 0; - } - else if (op[1] != NULL && - PyArray_NDIM(op[1]) >= PyArray_NDIM(op[0]) && - PyArray_TRIVIALLY_ITERABLE_PAIR(op[0], op[1], - PyArray_TRIVIALLY_ITERABLE_OP_READ, - PyArray_TRIVIALLY_ITERABLE_OP_NOREAD)) { - - /* Call the __prepare_array__ if necessary */ - if (prepare_ufunc_output(ufunc, &op[1], - arr_prep[0], full_args, 0) < 0) { - return -1; - } - - NPY_UF_DBG_PRINT("trivial 1 input\n"); - trivial_two_operand_loop(op, innerloop, innerloopdata); - - return 0; - } - } - else if (nin == 2 && nout == 1) { - if (op[2] == NULL && - (order == NPY_ANYORDER || order == NPY_KEEPORDER) && - PyArray_TRIVIALLY_ITERABLE_PAIR(op[0], op[1], - PyArray_TRIVIALLY_ITERABLE_OP_READ, - PyArray_TRIVIALLY_ITERABLE_OP_READ)) { - PyArrayObject *tmp; - /* - * Have to choose the input with more dimensions to clone, as - * one of them could be a scalar. - */ - if (PyArray_NDIM(op[0]) >= PyArray_NDIM(op[1])) { - tmp = op[0]; - } - else { - tmp = op[1]; - } - Py_INCREF(dtypes[2]); - op[2] = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, - dtypes[2], - PyArray_NDIM(tmp), - PyArray_DIMS(tmp), - NULL, NULL, - PyArray_ISFORTRAN(tmp) ? - NPY_ARRAY_F_CONTIGUOUS : 0, - NULL); - if (op[2] == NULL) { - return -1; - } - - /* Call the __prepare_array__ if necessary */ - if (prepare_ufunc_output(ufunc, &op[2], - arr_prep[0], full_args, 0) < 0) { - return -1; - } - - NPY_UF_DBG_PRINT("trivial 2 input with allocated output\n"); - trivial_three_operand_loop(op, innerloop, innerloopdata); - - return 0; - } - else if (op[2] != NULL && - PyArray_NDIM(op[2]) >= PyArray_NDIM(op[0]) && - PyArray_NDIM(op[2]) >= PyArray_NDIM(op[1]) && - PyArray_TRIVIALLY_ITERABLE_TRIPLE(op[0], op[1], op[2], - PyArray_TRIVIALLY_ITERABLE_OP_READ, - PyArray_TRIVIALLY_ITERABLE_OP_READ, - PyArray_TRIVIALLY_ITERABLE_OP_NOREAD)) { - - /* Call the __prepare_array__ if necessary */ - if (prepare_ufunc_output(ufunc, &op[2], - arr_prep[0], full_args, 0) < 0) { - return -1; - } - - NPY_UF_DBG_PRINT("trivial 2 input\n"); - trivial_three_operand_loop(op, innerloop, innerloopdata); - - return 0; - } + if (trivial_loop_ok && ufunc->nout == 1) { + int fast_path_result = try_trivial_single_output_loop(ufunc, + op, dtypes, order, arr_prep, full_args, + innerloop, innerloopdata); + if (fast_path_result != -2) { + return fast_path_result; } } @@ -1697,7 +1434,6 @@ execute_legacy_ufunc_loop(PyUFuncObject *ufunc, * If no trivial loop matched, an iterator is required to * resolve broadcasting, etc */ - NPY_UF_DBG_PRINT("iterator loop\n"); if (iterator_loop(ufunc, op, dtypes, order, buffersize, arr_prep, full_args, @@ -1859,6 +1595,7 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, dataptr = NpyIter_GetDataPtrArray(iter); strides = NpyIter_GetInnerStrideArray(iter); countptr = NpyIter_GetInnerLoopSizePtr(iter); + needs_api = NpyIter_IterationNeedsAPI(iter); NPY_BEGIN_THREADS_NDITER(iter); @@ -1869,7 +1606,7 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, innerloop(dataptr, strides, dataptr[nop], strides[nop], *countptr, innerloopdata); - } while (iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); NPY_END_THREADS; @@ -1879,122 +1616,6 @@ execute_fancy_ufunc_loop(PyUFuncObject *ufunc, return NpyIter_Deallocate(iter); } -static npy_bool -tuple_all_none(PyObject *tup) { - npy_intp i; - for (i = 0; i < PyTuple_GET_SIZE(tup); ++i) { - if (PyTuple_GET_ITEM(tup, i) != Py_None) { - return NPY_FALSE; - } - } - return NPY_TRUE; -} - -/* - * Convert positional args and the out kwarg into an input and output tuple. - * - * If the output tuple would be all None, return NULL instead. - * - * This duplicates logic in many places, so further refactoring is needed: - * - get_ufunc_arguments - * - PyUFunc_WithOverride - * - normalize___call___args - */ -static int -make_full_arg_tuple( - ufunc_full_args *full_args, - npy_intp nin, npy_intp nout, - PyObject *args, PyObject *kwds) -{ - PyObject *out_kwd = NULL; - npy_intp nargs = PyTuple_GET_SIZE(args); - npy_intp i; - - /* This should have been checked by the caller */ - assert(nin <= nargs && nargs <= nin + nout); - - /* Initialize so we can XDECREF safely */ - full_args->in = NULL; - full_args->out = NULL; - - /* Get the input arguments*/ - full_args->in = PyTuple_GetSlice(args, 0, nin); - if (full_args->in == NULL) { - goto fail; - } - - /* Look for output keyword arguments */ - if (kwds) { - out_kwd = PyDict_GetItemWithError(kwds, npy_um_str_out); - if (out_kwd == NULL && PyErr_Occurred()) { - goto fail; - } - } - else { - out_kwd = NULL; - } - - if (out_kwd != NULL) { - assert(nargs == nin); - if (out_kwd == Py_None) { - return 0; - } - else if (PyTuple_Check(out_kwd)) { - assert(PyTuple_GET_SIZE(out_kwd) == nout); - if (tuple_all_none(out_kwd)) { - return 0; - } - Py_INCREF(out_kwd); - full_args->out = out_kwd; - return 0; - } - else { - /* A single argument x is promoted to (x, None, None ...) */ - full_args->out = PyTuple_New(nout); - if (full_args->out == NULL) { - goto fail; - } - Py_INCREF(out_kwd); - PyTuple_SET_ITEM(full_args->out, 0, out_kwd); - for (i = 1; i < nout; ++i) { - Py_INCREF(Py_None); - PyTuple_SET_ITEM(full_args->out, i, Py_None); - } - return 0; - } - } - - /* No outputs in kwargs; if also none in args, we're done */ - if (nargs == nin) { - return 0; - } - /* copy across positional output arguments, adding trailing Nones */ - full_args->out = PyTuple_New(nout); - if (full_args->out == NULL) { - goto fail; - } - for (i = nin; i < nargs; ++i) { - PyObject *item = PyTuple_GET_ITEM(args, i); - Py_INCREF(item); - PyTuple_SET_ITEM(full_args->out, i - nin, item); - } - for (i = nargs; i < nin + nout; ++i) { - Py_INCREF(Py_None); - PyTuple_SET_ITEM(full_args->out, i - nin, Py_None); - } - - /* don't return a tuple full of None */ - if (tuple_all_none(full_args->out)) { - Py_DECREF(full_args->out); - full_args->out = NULL; - } - return 0; - -fail: - Py_XDECREF(full_args->in); - Py_XDECREF(full_args->out); - return -1; -} /* * Validate that operands have enough dimensions, accounting for @@ -2484,14 +2105,15 @@ _initialize_variable_parts(PyUFuncObject *ufunc, } static int -PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, - PyObject *args, PyObject *kwds, - PyArrayObject **op) +PyUFunc_GeneralizedFunctionInternal(PyUFuncObject *ufunc, PyArrayObject **op, + ufunc_full_args full_args, PyObject *type_tup, PyObject *extobj, + NPY_CASTING casting, NPY_ORDER order, npy_bool subok, + PyObject *axis, PyObject *axes, int keepdims) { int nin, nout; int i, j, idim, nop; const char *ufunc_name; - int retval, subok = 1; + int retval; int needs_api = 0; PyArray_Descr *dtypes[NPY_MAXARGS]; @@ -2528,20 +2150,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, int **remap_axis = NULL; /* The __array_prepare__ function to call for each output */ PyObject *arr_prep[NPY_MAXARGS]; - /* The separated input and output arguments, parsed from args and kwds */ - ufunc_full_args full_args = {NULL, NULL}; - - NPY_ORDER order = NPY_KEEPORDER; - /* Use the default assignment casting rule */ - NPY_CASTING casting = NPY_DEFAULT_ASSIGN_CASTING; - /* other possible keyword arguments */ - PyObject *extobj, *type_tup, *axes, *axis; - int keepdims = -1; - - if (ufunc == NULL) { - PyErr_SetString(PyExc_ValueError, "function not supported"); - return -1; - } nin = ufunc->nin; nout = ufunc->nout; @@ -2563,18 +2171,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, goto fail; } - NPY_UF_DBG_PRINT("Getting arguments\n"); - - /* - * Get all the arguments. - */ - retval = get_ufunc_arguments(ufunc, args, kwds, - op, &order, &casting, &extobj, - &type_tup, &subok, NULL, &axes, &axis, &keepdims); - if (retval < 0) { - NPY_UF_DBG_PRINT("Failure in getting arguments\n"); - return retval; - } /* * If keepdims was passed in (and thus changed from the initial value * on top), check the gufunc is suitable, i.e., that its inputs share @@ -2634,6 +2230,8 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, /* Possibly remap axes. */ if (axes != NULL || axis != NULL) { + assert(!(axes != NULL && axis != NULL)); + remap_axis = PyArray_malloc(sizeof(remap_axis[0]) * nop); remap_axis_memory = PyArray_malloc(sizeof(remap_axis_memory[0]) * nop * NPY_MAXDIMS); @@ -2811,20 +2409,11 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, #endif if (subok) { - if (make_full_arg_tuple(&full_args, nin, nout, args, kwds) < 0) { - goto fail; - } - /* * Get the appropriate __array_prepare__ function to call * for each output */ - _find_array_prepare(full_args, arr_prep, nin, nout); - } - - /* If the loop wants the arrays, provide them */ - if (_does_loop_use_arrays(innerloopdata)) { - innerloopdata = (void*)op; + _find_array_prepare(full_args, arr_prep, nout); } /* @@ -2851,7 +2440,7 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, /* Fill in any allocated outputs */ { PyArrayObject **operands = NpyIter_GetOperandArray(iter); - for (i = 0; i < nop; ++i) { + for (i = nin; i < nop; ++i) { if (op[i] == NULL) { op[i] = operands[i]; Py_INCREF(op[i]); @@ -2973,6 +2562,7 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, } dataptr = NpyIter_GetDataPtrArray(iter); count_ptr = NpyIter_GetInnerLoopSizePtr(iter); + needs_api = NpyIter_IterationNeedsAPI(iter); if (!needs_api && !NpyIter_IterationNeedsAPI(iter)) { NPY_BEGIN_THREADS_THRESHOLDED(total_problem_size); @@ -2980,39 +2570,11 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, do { inner_dimensions[0] = *count_ptr; innerloop(dataptr, inner_dimensions, inner_strides, innerloopdata); - } while (iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); if (!needs_api && !NpyIter_IterationNeedsAPI(iter)) { NPY_END_THREADS; } - } else { - /** - * For each output operand, check if it has non-zero size, - * and assign the identity if it does. For example, a dot - * product of two zero-length arrays will be a scalar, - * which has size one. - */ - npy_bool reorderable; - PyObject *identity = _get_identity(ufunc, &reorderable); - if (identity == NULL) { - retval = -1; - goto fail; - } - - for (i = nin; i < nop; ++i) { - if (PyArray_SIZE(op[i]) != 0) { - if (identity == Py_None) { - PyErr_Format(PyExc_ValueError, - "ufunc %s ", - ufunc_name); - Py_DECREF(identity); - retval = -1; - goto fail; - } - PyArray_FillWithScalar(op[i], identity); - } - } - Py_DECREF(identity); } /* Check whether any errors occurred during the loop */ @@ -3032,12 +2594,6 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, Py_XDECREF(dtypes[i]); Py_XDECREF(arr_prep[i]); } - Py_XDECREF(type_tup); - Py_XDECREF(extobj); - Py_XDECREF(axes); - Py_XDECREF(axis); - Py_XDECREF(full_args.in); - Py_XDECREF(full_args.out); PyArray_free(remap_axis_memory); PyArray_free(remap_axis); @@ -3050,36 +2606,25 @@ PyUFunc_GeneralizedFunction(PyUFuncObject *ufunc, PyArray_free(inner_strides); NpyIter_Deallocate(iter); for (i = 0; i < nop; ++i) { - Py_XDECREF(op[i]); - op[i] = NULL; Py_XDECREF(dtypes[i]); Py_XDECREF(arr_prep[i]); } - Py_XDECREF(type_tup); - Py_XDECREF(extobj); - Py_XDECREF(axes); - Py_XDECREF(axis); - Py_XDECREF(full_args.in); - Py_XDECREF(full_args.out); PyArray_free(remap_axis_memory); PyArray_free(remap_axis); return retval; } -/* - * This generic function is called with the ufunc object, the arguments to it, - * and an array of (pointers to) PyArrayObjects which are NULL. - * - * 'op' is an array of at least NPY_MAXARGS PyArrayObject *. - */ + static int -PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, - PyObject *args, PyObject *kwds, PyArrayObject **op) +PyUFunc_GenericFunctionInternal(PyUFuncObject *ufunc, PyArrayObject **op, + ufunc_full_args full_args, PyObject *type_tup, PyObject *extobj, + NPY_CASTING casting, NPY_ORDER order, npy_bool subok, + PyArrayObject *wheremask) { int nin, nout; int i, nop; const char *ufunc_name; - int retval = -1, subok = 1; + int retval = -1; npy_uint32 op_flags[NPY_MAXARGS]; npy_intp default_op_out_flags; @@ -3088,33 +2633,11 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, /* These parameters come from extobj= or from a TLS global */ int buffersize = 0, errormask = 0; - /* The mask provided in the 'where=' parameter */ - PyArrayObject *wheremask = NULL; - /* The __array_prepare__ function to call for each output */ PyObject *arr_prep[NPY_MAXARGS]; - /* - * This is either args, or args with the out= parameter from - * kwds added appropriately. - */ - ufunc_full_args full_args = {NULL, NULL}; int trivial_loop_ok = 0; - NPY_ORDER order = NPY_KEEPORDER; - /* Use the default assignment casting rule */ - NPY_CASTING casting = NPY_DEFAULT_ASSIGN_CASTING; - PyObject *extobj, *type_tup; - - if (ufunc == NULL) { - PyErr_SetString(PyExc_ValueError, "function not supported"); - return -1; - } - - if (ufunc->core_enabled) { - return PyUFunc_GeneralizedFunction(ufunc, args, kwds, op); - } - nin = ufunc->nin; nout = ufunc->nout; nop = nin + nout; @@ -3126,18 +2649,7 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, /* Initialize all the dtypes and __array_prepare__ callbacks to NULL */ for (i = 0; i < nop; ++i) { dtypes[i] = NULL; - arr_prep[i] = NULL; - } - - NPY_UF_DBG_PRINT("Getting arguments\n"); - - /* Get all the arguments */ - retval = get_ufunc_arguments(ufunc, args, kwds, - op, &order, &casting, &extobj, - &type_tup, &subok, &wheremask, NULL, NULL, NULL); - if (retval < 0) { - NPY_UF_DBG_PRINT("Failure in getting arguments\n"); - return retval; + arr_prep[i] = NULL; } /* Get the buffersize and errormask */ @@ -3185,17 +2697,13 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, #endif if (subok) { - if (make_full_arg_tuple(&full_args, nin, nout, args, kwds) < 0) { - goto fail; - } /* * Get the appropriate __array_prepare__ function to call * for each output */ - _find_array_prepare(full_args, arr_prep, nin, nout); + _find_array_prepare(full_args, arr_prep, nout); } - /* Do the ufunc loop */ if (wheremask != NULL) { NPY_UF_DBG_PRINT("Executing fancy inner loop\n"); @@ -3257,11 +2765,6 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, Py_XDECREF(dtypes[i]); Py_XDECREF(arr_prep[i]); } - Py_XDECREF(type_tup); - Py_XDECREF(extobj); - Py_XDECREF(full_args.in); - Py_XDECREF(full_args.out); - Py_XDECREF(wheremask); NPY_UF_DBG_PRINT("Returning success code 0\n"); @@ -3270,16 +2773,9 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, fail: NPY_UF_DBG_PRINT1("Returning failure code %d\n", retval); for (i = 0; i < nop; ++i) { - Py_XDECREF(op[i]); - op[i] = NULL; Py_XDECREF(dtypes[i]); Py_XDECREF(arr_prep[i]); } - Py_XDECREF(type_tup); - Py_XDECREF(extobj); - Py_XDECREF(full_args.in); - Py_XDECREF(full_args.out); - Py_XDECREF(wheremask); return retval; } @@ -3287,21 +2783,17 @@ PyUFunc_GenericFunction_int(PyUFuncObject *ufunc, /*UFUNC_API*/ NPY_NO_EXPORT int -PyUFunc_GenericFunction(PyUFuncObject *ufunc, - PyObject *args, PyObject *kwds, PyArrayObject **op) +PyUFunc_GenericFunction(PyUFuncObject *NPY_UNUSED(ufunc), + PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds), + PyArrayObject **NPY_UNUSED(op)) { - /* NumPy 1.19, 2020-01-24 */ - if (DEPRECATE( - "PyUFunc_GenericFunction() C-API function is deprecated " - "and expected to be removed rapidly. If you are using it (i.e. see " - "this warning/error), please notify the NumPy developers. " - "As of now it is expected that any use case is served better by " - "the direct use of `PyObject_Call(ufunc, args, kwargs)`. " - "PyUFunc_GenericFunction function has slightly different " - "untested behaviour.") < 0) { - return -1; - } - return PyUFunc_GenericFunction_int(ufunc, args, kwds, op); + /* NumPy 1.21, 2020-03-29 */ + PyErr_SetString(PyExc_RuntimeError, + "The `PyUFunc_GenericFunction()` C-API function has been disabled. " + "Please use `PyObject_Call(ufunc, args, kwargs)`, which has " + "identical behaviour but allows subclass and `__array_ufunc__` " + "override handling and only returns the normal ufunc result."); + return -1; } @@ -3520,6 +3012,10 @@ reduce_loop(NpyIter *iter, char **dataptrs, npy_intp const *strides, innerloop(dataptrs_copy, &count, strides_copy, innerloopdata); + if (needs_api && PyErr_Occurred()) { + goto finish_loop; + } + /* Jump to the faster loop when skipping is done */ if (skip_first_count == 0) { if (iternext(iter)) { @@ -3531,6 +3027,11 @@ reduce_loop(NpyIter *iter, char **dataptrs, npy_intp const *strides, } } while (iternext(iter)); } + + if (needs_api && PyErr_Occurred()) { + goto finish_loop; + } + do { /* Turn the two items into three for the inner loop */ dataptrs_copy[0] = dataptrs[0]; @@ -3569,7 +3070,7 @@ reduce_loop(NpyIter *iter, char **dataptrs, npy_intp const *strides, n = 1; } } - } while (iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); finish_loop: NPY_END_THREADS; @@ -3608,13 +3109,9 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, const char *ufunc_name = ufunc_get_name_cstr(ufunc); /* These parameters come from a TLS global */ int buffersize = 0, errormask = 0; - static PyObject *NoValue = NULL; NPY_UF_DBG_PRINT1("\nEvaluating ufunc %s.reduce\n", ufunc_name); - npy_cache_import("numpy", "_NoValue", &NoValue); - if (NoValue == NULL) return NULL; - ndim = PyArray_NDIM(arr); /* Create an array of flags for reduction */ @@ -3640,7 +3137,7 @@ PyUFunc_Reduce(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, } /* Get the initial value */ - if (initial == NULL || initial == NoValue) { + if (initial == NULL) { initial = identity; /* @@ -3882,6 +3379,7 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, goto fail; } dataptr = NpyIter_GetDataPtrArray(iter); + needs_api = NpyIter_IterationNeedsAPI(iter); /* Execute the loop with just the outer iterator */ @@ -3932,7 +3430,7 @@ PyUFunc_Accumulate(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *out, innerloop(dataptr_copy, &count_m1, stride_copy, innerloopdata); } - } while (iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); NPY_END_THREADS; } @@ -4263,6 +3761,7 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, npy_intp stride0_ind = PyArray_STRIDE(op[0], axis); int itemsize = op_dtypes[0]->elsize; + int needs_api = NpyIter_IterationNeedsAPI(iter); /* Get the variables needed for the loop */ iternext = NpyIter_GetIterNext(iter, NULL); @@ -4327,7 +3826,7 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, stride_copy, innerloopdata); } } - } while (iternext(iter)); + } while (!(needs_api && PyErr_Occurred()) && iternext(iter)); NPY_END_THREADS; } @@ -4417,32 +3916,105 @@ PyUFunc_Reduceat(PyUFuncObject *ufunc, PyArrayObject *arr, PyArrayObject *ind, } +static npy_bool +tuple_all_none(PyObject *tup) { + npy_intp i; + for (i = 0; i < PyTuple_GET_SIZE(tup); ++i) { + if (PyTuple_GET_ITEM(tup, i) != Py_None) { + return NPY_FALSE; + } + } + return NPY_TRUE; +} + + +static int +_set_full_args_out(int nout, PyObject *out_obj, ufunc_full_args *full_args) +{ + if (PyTuple_CheckExact(out_obj)) { + if (PyTuple_GET_SIZE(out_obj) != nout) { + PyErr_SetString(PyExc_ValueError, + "The 'out' tuple must have exactly " + "one entry per ufunc output"); + return -1; + } + if (tuple_all_none(out_obj)) { + return 0; + } + else { + Py_INCREF(out_obj); + full_args->out = out_obj; + } + } + else if (nout == 1) { + if (out_obj == Py_None) { + return 0; + } + /* Can be an array if it only has one output */ + full_args->out = PyTuple_Pack(1, out_obj); + if (full_args->out == NULL) { + return -1; + } + } + else { + PyErr_SetString(PyExc_TypeError, + nout > 1 ? "'out' must be a tuple of arrays" : + "'out' must be an array or a tuple with " + "a single array"); + return -1; + } + return 0; +} + + +/* + * Convert function which replaces np._NoValue with NULL. + * As a converter returns 0 on error and 1 on success. + */ +static int +_not_NoValue(PyObject *obj, PyObject **out) +{ + static PyObject *NoValue = NULL; + npy_cache_import("numpy", "_NoValue", &NoValue); + if (NoValue == NULL) { + return 0; + } + if (obj == NoValue) { + *out = NULL; + } + else { + *out = obj; + } + return 1; +} + + +/* forward declaration */ +static PyArray_DTypeMeta * _get_dtype(PyObject *dtype_obj); + /* * This code handles reduce, reduceat, and accumulate * (accumulate and reduce are special cases of the more general reduceat * but they are handled separately for speed) */ static PyObject * -PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, - PyObject *kwds, int operation) +PyUFunc_GenericReduction(PyUFuncObject *ufunc, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, int operation) { int i, naxes=0, ndim; int axes[NPY_MAXDIMS]; - PyObject *axes_in = NULL; + + ufunc_full_args full_args = {NULL, NULL}; + PyObject *axes_obj = NULL; PyArrayObject *mp = NULL, *wheremask = NULL, *ret = NULL; - PyObject *op; - PyObject *obj_ind; + PyObject *op = NULL; PyArrayObject *indices = NULL; PyArray_Descr *otype = NULL; PyArrayObject *out = NULL; int keepdims = 0; PyObject *initial = NULL; - static char *reduce_kwlist[] = { - "array", "axis", "dtype", "out", "keepdims", "initial", "where", NULL}; - static char *accumulate_kwlist[] = { - "array", "axis", "dtype", "out", NULL}; - static char *reduceat_kwlist[] = { - "array", "indices", "axis", "dtype", "out", NULL}; + npy_bool out_is_passed_by_position; + static char *_reduce_type[] = {"reduce", "accumulate", "reduceat", NULL}; @@ -4468,62 +4040,136 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, _reduce_type[operation]); return NULL; } - /* if there is a tuple of 1 for `out` in kwds, unpack it */ - if (kwds != NULL) { - PyObject *out_obj = PyDict_GetItemWithError(kwds, npy_um_str_out); - if (out_obj == NULL && PyErr_Occurred()){ - return NULL; - } - else if (out_obj != NULL && PyTuple_CheckExact(out_obj)) { - if (PyTuple_GET_SIZE(out_obj) != 1) { - PyErr_SetString(PyExc_ValueError, - "The 'out' tuple must have exactly one entry"); - return NULL; - } - out_obj = PyTuple_GET_ITEM(out_obj, 0); - PyDict_SetItem(kwds, npy_um_str_out, out_obj); - } - } + /* + * Perform argument parsing, but start by only extracting. This is + * just to preserve the behaviour that __array_ufunc__ did not perform + * any checks on arguments, and we could change this or change it for + * certain parameters. + */ + PyObject *otype_obj = NULL, *out_obj = NULL, *indices_obj = NULL; + PyObject *keepdims_obj = NULL, *wheremask_obj = NULL; if (operation == UFUNC_REDUCEAT) { - PyArray_Descr *indtype; - indtype = PyArray_DescrFromType(NPY_INTP); - if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO&O&:reduceat", reduceat_kwlist, - &op, - &obj_ind, - &axes_in, - PyArray_DescrConverter2, &otype, - PyArray_OutputConverter, &out)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("reduceat", args, len_args, kwnames, + "array", NULL, &op, + "indices", NULL, &indices_obj, + "|axis", NULL, &axes_obj, + "|dtype", NULL, &otype_obj, + "|out", NULL, &out_obj, + NULL, NULL, NULL) < 0) { goto fail; } - indices = (PyArrayObject *)PyArray_FromAny(obj_ind, indtype, - 1, 1, NPY_ARRAY_CARRAY, NULL); - if (indices == NULL) { + /* Prepare inputs for PyUfunc_CheckOverride */ + full_args.in = PyTuple_Pack(2, op, indices_obj); + if (full_args.in == NULL) { goto fail; } + out_is_passed_by_position = len_args >= 5; } else if (operation == UFUNC_ACCUMULATE) { - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&:accumulate", - accumulate_kwlist, - &op, - &axes_in, - PyArray_DescrConverter2, &otype, - PyArray_OutputConverter, &out)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("accumulate", args, len_args, kwnames, + "array", NULL, &op, + "|axis", NULL, &axes_obj, + "|dtype", NULL, &otype_obj, + "|out", NULL, &out_obj, + NULL, NULL, NULL) < 0) { goto fail; } + /* Prepare input for PyUfunc_CheckOverride */ + full_args.in = PyTuple_Pack(1, op); + if (full_args.in == NULL) { + goto fail; + } + out_is_passed_by_position = len_args >= 4; } else { - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO&O&iOO&:reduce", - reduce_kwlist, - &op, - &axes_in, - PyArray_DescrConverter2, &otype, - PyArray_OutputConverter, &out, - &keepdims, &initial, - _wheremask_converter, &wheremask)) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments("reduce", args, len_args, kwnames, + "array", NULL, &op, + "|axis", NULL, &axes_obj, + "|dtype", NULL, &otype_obj, + "|out", NULL, &out_obj, + "|keepdims", NULL, &keepdims_obj, + "|initial", &_not_NoValue, &initial, + "|where", NULL, &wheremask_obj, + NULL, NULL, NULL) < 0) { + goto fail; + } + /* Prepare input for PyUfunc_CheckOverride */ + full_args.in = PyTuple_Pack(1, op); + if (full_args.in == NULL) { + goto fail; + } + out_is_passed_by_position = len_args >= 4; + } + + /* Normalize output for PyUFunc_CheckOverride and conversion. */ + if (out_is_passed_by_position) { + /* in this branch, out is always wrapped in a tuple. */ + if (out_obj != Py_None) { + full_args.out = PyTuple_Pack(1, out_obj); + if (full_args.out == NULL) { + goto fail; + } + } + } + else if (out_obj) { + if (_set_full_args_out(1, out_obj, &full_args) < 0) { + goto fail; + } + /* Ensure that out_obj is the array, not the tuple: */ + if (full_args.out != NULL) { + out_obj = PyTuple_GET_ITEM(full_args.out, 0); + } + } + + /* We now have all the information required to check for Overrides */ + PyObject *override = NULL; + int errval = PyUFunc_CheckOverride(ufunc, _reduce_type[operation], + full_args.in, full_args.out, args, len_args, kwnames, &override); + if (errval) { + return NULL; + } + else if (override) { + Py_XDECREF(full_args.in); + Py_XDECREF(full_args.out); + return override; + } + + /* Finish parsing of all parameters (no matter which reduce-like) */ + if (indices_obj) { + PyArray_Descr *indtype = PyArray_DescrFromType(NPY_INTP); + + indices = (PyArrayObject *)PyArray_FromAny(indices_obj, + indtype, 1, 1, NPY_ARRAY_CARRAY, NULL); + if (indices == NULL) { + goto fail; + } + } + if (otype_obj && otype_obj != Py_None) { + /* Use `_get_dtype` because `dtype` is a DType and not the instance */ + PyArray_DTypeMeta *dtype = _get_dtype(otype_obj); + if (dtype == NULL) { goto fail; } + Py_INCREF(dtype->singleton); + otype = dtype->singleton; + } + if (out_obj && !PyArray_OutputConverter(out_obj, &out)) { + goto fail; + } + if (keepdims_obj && !PyArray_PythonPyIntFromInt(keepdims_obj, &keepdims)) { + goto fail; + } + if (wheremask_obj && !_wheremask_converter(wheremask_obj, &wheremask)) { + goto fail; } + /* Ensure input is an array */ mp = (PyArrayObject *)PyArray_FromAny(op, NULL, 0, 0, 0, NULL); if (mp == NULL) { @@ -4542,7 +4188,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, } /* Convert the 'axis' parameter into a list of axes */ - if (axes_in == NULL) { + if (axes_obj == NULL) { /* apply defaults */ if (ndim == 0) { naxes = 0; @@ -4552,22 +4198,22 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, axes[0] = 0; } } - else if (axes_in == Py_None) { + else if (axes_obj == Py_None) { /* Convert 'None' into all the axes */ naxes = ndim; for (i = 0; i < naxes; ++i) { axes[i] = i; } } - else if (PyTuple_Check(axes_in)) { - naxes = PyTuple_Size(axes_in); + else if (PyTuple_Check(axes_obj)) { + naxes = PyTuple_Size(axes_obj); if (naxes < 0 || naxes > NPY_MAXDIMS) { PyErr_SetString(PyExc_ValueError, "too many values for 'axis'"); goto fail; } for (i = 0; i < naxes; ++i) { - PyObject *tmp = PyTuple_GET_ITEM(axes_in, i); + PyObject *tmp = PyTuple_GET_ITEM(axes_obj, i); int axis = PyArray_PyIntAsInt(tmp); if (error_converting(axis)) { goto fail; @@ -4580,7 +4226,7 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, } else { /* Try to interpret axis as an integer */ - int axis = PyArray_PyIntAsInt(axes_in); + int axis = PyArray_PyIntAsInt(axes_obj); /* TODO: PyNumber_Index would be good to use here */ if (error_converting(axis)) { goto fail; @@ -4671,72 +4317,589 @@ PyUFunc_GenericReduction(PyUFuncObject *ufunc, PyObject *args, } Py_DECREF(mp); Py_DECREF(otype); + Py_XDECREF(full_args.in); + Py_XDECREF(full_args.out); if (ret == NULL) { return NULL; } - /* Wrap and return the output */ - { - /* Find __array_wrap__ - note that these rules are different to the - * normal ufunc path - */ - PyObject *wrap; - if (out != NULL) { - wrap = Py_None; - Py_INCREF(wrap); - } - else if (Py_TYPE(op) != Py_TYPE(ret)) { - wrap = PyObject_GetAttr(op, npy_um_str_array_wrap); - if (wrap == NULL) { - PyErr_Clear(); + /* Wrap and return the output */ + { + /* Find __array_wrap__ - note that these rules are different to the + * normal ufunc path + */ + PyObject *wrap; + if (out != NULL) { + wrap = Py_None; + Py_INCREF(wrap); + } + else if (Py_TYPE(op) != Py_TYPE(ret)) { + wrap = PyObject_GetAttr(op, npy_um_str_array_wrap); + if (wrap == NULL) { + PyErr_Clear(); + } + else if (!PyCallable_Check(wrap)) { + Py_DECREF(wrap); + wrap = NULL; + } + } + else { + wrap = NULL; + } + return _apply_array_wrap(wrap, ret, NULL); + } + +fail: + Py_XDECREF(otype); + Py_XDECREF(mp); + Py_XDECREF(wheremask); + Py_XDECREF(full_args.in); + Py_XDECREF(full_args.out); + return NULL; +} + + +/* + * Perform a basic check on `dtype`, `sig`, and `signature` since only one + * may be set. If `sig` is used, writes it into `out_signature` (which should + * be set to `signature_obj` so that following code only requires to handle + * `signature_obj`). + * + * Does NOT incref the output! This only copies the borrowed references + * gotten during the argument parsing. + * + * This function does not do any normalization of the input dtype tuples, + * this happens after the array-ufunc override check currently. + */ +static int +_check_and_copy_sig_to_signature( + PyObject *sig_obj, PyObject *signature_obj, PyObject *dtype, + PyObject **out_signature) +{ + *out_signature = NULL; + if (signature_obj != NULL) { + *out_signature = signature_obj; + } + + if (sig_obj != NULL) { + if (*out_signature != NULL) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'sig' and 'signature'"); + *out_signature = NULL; + return -1; + } + *out_signature = sig_obj; + } + + if (dtype != NULL) { + if (*out_signature != NULL) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'signature' and 'dtype'"); + return -1; + } + /* dtype needs to be converted, delay after the override check */ + } + return 0; +} + + +/* + * Note: This function currently lets DType classes pass, but in general + * the class (not the descriptor instance) is the preferred input, so the + * parsing should eventually be adapted to prefer classes and possible + * deprecated instances. (Users should not notice that much, since `np.float64` + * or "float64" usually denotes the DType class rather than the instance.) + */ +static PyArray_DTypeMeta * +_get_dtype(PyObject *dtype_obj) { + if (PyObject_TypeCheck(dtype_obj, &PyArrayDTypeMeta_Type)) { + Py_INCREF(dtype_obj); + return (PyArray_DTypeMeta *)dtype_obj; + } + else { + PyArray_Descr *descr = NULL; + if (!PyArray_DescrConverter(dtype_obj, &descr)) { + return NULL; + } + PyArray_DTypeMeta *out = NPY_DTYPE(descr); + if (NPY_UNLIKELY(!out->legacy)) { + /* TODO: this path was unreachable when added. */ + PyErr_SetString(PyExc_TypeError, + "Cannot pass a new user DType instance to the `dtype` or " + "`signature` arguments of ufuncs. Pass the DType class " + "instead."); + Py_DECREF(descr); + return NULL; + } + else if (NPY_UNLIKELY(out->singleton != descr)) { + /* This does not warn about `metadata`, but units is important. */ + if (!PyArray_EquivTypes(out->singleton, descr)) { + PyErr_Format(PyExc_TypeError, + "The `dtype` and `signature` arguments to " + "ufuncs only select the general DType and not details " + "such as the byte order or time unit (with rare " + "exceptions see release notes). To avoid this warning " + "please use the scalar types `np.float64`, or string " + "notation.\n" + "In rare cases where the time unit was preserved, " + "either cast the inputs or provide an output array. " + "In the future NumPy may transition to allow providing " + "`dtype=` to denote the outputs `dtype` as well"); + Py_DECREF(descr); + return NULL; + } + } + Py_INCREF(out); + Py_DECREF(descr); + return out; + } +} + + +static int +_make_new_typetup( + int nop, PyArray_DTypeMeta *signature[], PyObject **out_typetup) { + *out_typetup = PyTuple_New(nop); + if (*out_typetup == NULL) { + return -1; + } + + int noncount = 0; + for (int i = 0; i < nop; i++) { + PyObject *item; + if (signature[i] == NULL) { + item = Py_None; + noncount++; + } + else { + if (!signature[i]->legacy || signature[i]->abstract) { + /* + * The legacy type resolution can't deal with these. + * This path will return `None` or so in the future to + * set an error later if the legacy type resolution is used. + */ + PyErr_SetString(PyExc_RuntimeError, + "Internal NumPy error: new DType in signature not yet " + "supported. (This should be unreachable code!)"); + Py_SETREF(*out_typetup, NULL); + return -1; + } + item = (PyObject *)signature[i]->singleton; + } + Py_INCREF(item); + PyTuple_SET_ITEM(*out_typetup, i, item); + } + if (noncount == nop) { + /* The whole signature was None, simply ignore type tuple */ + Py_DECREF(*out_typetup); + *out_typetup = NULL; + } + return 0; +} + + +/* + * Finish conversion parsing of the type tuple. NumPy always only honored + * the type number for passed in descriptors/dtypes. + * The `dtype` argument is interpreted as the first output DType (not + * descriptor). + * Unlike the dtype of an `out` array, it influences loop selection! + * + * NOTE: This function replaces the type tuple if passed in (it steals + * the original reference and returns a new object and reference)! + * The caller must XDECREF the type tuple both on error or success. + * + * The function returns a new, normalized type-tuple. + */ +static int +_get_normalized_typetup(PyUFuncObject *ufunc, + PyObject *dtype_obj, PyObject *signature_obj, PyObject **out_typetup) +{ + if (dtype_obj == NULL && signature_obj == NULL) { + return 0; + } + + int res = -1; + int nin = ufunc->nin, nout = ufunc->nout, nop = nin + nout; + /* + * TODO: `signature` will be the main result in the future and + * not the typetup. (Type tuple construction can be deffered to when + * the legacy fallback is used). + */ + PyArray_DTypeMeta *signature[NPY_MAXARGS]; + memset(signature, '\0', sizeof(*signature) * nop); + + if (dtype_obj != NULL) { + if (dtype_obj == Py_None) { + /* If `dtype=None` is passed, no need to do anything */ + assert(*out_typetup == NULL); + return 0; + } + if (nout == 0) { + /* This may be allowed (NumPy does not do this)? */ + PyErr_SetString(PyExc_TypeError, + "Cannot provide `dtype` when a ufunc has no outputs"); + return -1; + } + PyArray_DTypeMeta *dtype = _get_dtype(dtype_obj); + if (dtype == NULL) { + return -1; + } + for (int i = nin; i < nop; i++) { + Py_INCREF(dtype); + signature[i] = dtype; + } + Py_DECREF(dtype); + res = _make_new_typetup(nop, signature, out_typetup); + goto finish; + } + + assert(signature_obj != NULL); + /* Fill in specified_types from the tuple or string (signature_obj) */ + if (PyTuple_Check(signature_obj)) { + Py_ssize_t n = PyTuple_GET_SIZE(signature_obj); + if (n == 1 && nop != 1) { + /* + * Special handling, because we deprecate this path. The path + * probably mainly existed since the `dtype=obj` was passed through + * as `(obj,)` and parsed later. + */ + if (PyTuple_GET_ITEM(signature_obj, 0) == Py_None) { + PyErr_SetString(PyExc_TypeError, + "a single item type tuple cannot contain None."); + goto finish; + } + if (DEPRECATE("The use of a length 1 tuple for the ufunc " + "`signature` is deprecated. Use `dtype` or fill the" + "tuple with `None`s.") < 0) { + goto finish; + } + /* Use the same logic as for `dtype=` */ + res = _get_normalized_typetup(ufunc, + PyTuple_GET_ITEM(signature_obj, 0), NULL, out_typetup); + goto finish; + } + if (n != nop) { + PyErr_Format(PyExc_ValueError, + "a type-tuple must be specified of length %d for ufunc '%s'", + nop, ufunc_get_name_cstr(ufunc)); + goto finish; + } + for (int i = 0; i < nop; ++i) { + PyObject *item = PyTuple_GET_ITEM(signature_obj, i); + if (item == Py_None) { + continue; + } + signature[i] = _get_dtype(item); + if (signature[i] == NULL) { + goto finish; + } + } + } + else if (PyBytes_Check(signature_obj) || PyUnicode_Check(signature_obj)) { + PyObject *str_object = NULL; + + if (PyBytes_Check(signature_obj)) { + str_object = PyUnicode_FromEncodedObject(signature_obj, NULL, NULL); + if (str_object == NULL) { + goto finish; } - else if (!PyCallable_Check(wrap)) { - Py_DECREF(wrap); - wrap = NULL; + } + else { + Py_INCREF(signature_obj); + str_object = signature_obj; + } + + Py_ssize_t length; + const char *str = PyUnicode_AsUTF8AndSize(str_object, &length); + if (str == NULL) { + Py_DECREF(str_object); + goto finish; + } + + if (length != 1 && (length != nin+nout + 2 || + str[nin] != '-' || str[nin+1] != '>')) { + PyErr_Format(PyExc_ValueError, + "a type-string for %s, %d typecode(s) before and %d after " + "the -> sign", ufunc_get_name_cstr(ufunc), nin, nout); + Py_DECREF(str_object); + goto finish; + } + if (length == 1 && nin+nout != 1) { + Py_DECREF(str_object); + if (DEPRECATE("The use of a length 1 string for the ufunc " + "`signature` is deprecated. Use `dtype` attribute or " + "pass a tuple with `None`s.") < 0) { + goto finish; } + /* `signature="l"` is the same as `dtype="l"` */ + res = _get_normalized_typetup(ufunc, str_object, NULL, out_typetup); + goto finish; } else { - wrap = NULL; + for (int i = 0; i < nin+nout; ++i) { + npy_intp istr = i < nin ? i : i+2; + PyArray_Descr *descr = PyArray_DescrFromType(str[istr]); + if (descr == NULL) { + Py_DECREF(str_object); + goto finish; + } + signature[i] = NPY_DTYPE(descr); + Py_INCREF(signature[i]); + Py_DECREF(descr); + } + Py_DECREF(str_object); } - return _apply_array_wrap(wrap, ret, NULL); } + else { + PyErr_SetString(PyExc_TypeError, + "the signature object to ufunc must be a string or a tuple."); + goto finish; + } + res = _make_new_typetup(nop, signature, out_typetup); -fail: - Py_XDECREF(otype); - Py_XDECREF(mp); - Py_XDECREF(wheremask); - return NULL; + finish: + for (int i =0; i < nop; i++) { + Py_XDECREF(signature[i]); + } + return res; } +/* + * Main ufunc call implementation. + * + * This implementation makes use of the "fastcall" way of passing keyword + * arguments and is called directly from `ufunc_generic_vectorcall` when + * Python has `tp_vectorcall` (Python 3.8+). + * If `tp_vectorcall` is not available, the dictionary `kwargs` are unpacked in + * `ufunc_generic_call` with fairly little overhead. + */ static PyObject * -ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) +ufunc_generic_fastcall(PyUFuncObject *ufunc, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames, + npy_bool outer) { - int i; - PyArrayObject *mps[NPY_MAXARGS]; + PyArrayObject *operands[NPY_MAXARGS] = {NULL}; PyObject *retobj[NPY_MAXARGS]; PyObject *wraparr[NPY_MAXARGS]; PyObject *override = NULL; ufunc_full_args full_args = {NULL, NULL}; + PyObject *typetup = NULL; + int errval; + int nin = ufunc->nin, nout = ufunc->nout, nop = ufunc->nargs; - errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override); - if (errval) { + /* + * Note that the input (and possibly output) arguments are passed in as + * positional arguments. We extract these first and check for `out` + * passed by keyword later. + * Outputs and inputs are stored in `full_args.in` and `full_args.out` + * as tuples (or NULL when no outputs are passed). + */ + + /* Check number of arguments */ + if ((len_args < nin) || (len_args > nop)) { + PyErr_Format(PyExc_TypeError, + "%s() takes from %d to %d positional arguments but " + "%zd were given", + ufunc_get_name_cstr(ufunc) , nin, nop, len_args); return NULL; } + + /* Fetch input arguments. */ + full_args.in = PyTuple_New(ufunc->nin); + if (full_args.in == NULL) { + return NULL; + } + for (int i = 0; i < ufunc->nin; i++) { + PyObject *tmp = args[i]; + Py_INCREF(tmp); + PyTuple_SET_ITEM(full_args.in, i, tmp); + } + + /* + * If there are more arguments, they define the out args. Otherwise + * full_args.out is NULL for now, and the `out` kwarg may still be passed. + */ + npy_bool out_is_passed_by_position = len_args > nin; + if (out_is_passed_by_position) { + npy_bool all_none = NPY_TRUE; + + full_args.out = PyTuple_New(nout); + if (full_args.out == NULL) { + goto fail; + } + for (int i = nin; i < nop; i++) { + PyObject *tmp; + if (i < (int)len_args) { + tmp = args[i]; + if (tmp != Py_None) { + all_none = NPY_FALSE; + } + } + else { + tmp = Py_None; + } + Py_INCREF(tmp); + PyTuple_SET_ITEM(full_args.out, i-nin, tmp); + } + if (all_none) { + Py_SETREF(full_args.out, NULL); + } + } + else { + full_args.out = NULL; + } + + /* + * We have now extracted (but not converted) the input arguments. + * To simplify overrides, extract all other arguments (as objects only) + */ + PyObject *out_obj = NULL, *where_obj = NULL; + PyObject *axes_obj = NULL, *axis_obj = NULL; + PyObject *keepdims_obj = NULL, *casting_obj = NULL, *order_obj = NULL; + PyObject *subok_obj = NULL, *signature_obj = NULL, *sig_obj = NULL; + PyObject *dtype_obj = NULL, *extobj = NULL; + + /* Skip parsing if there are no keyword arguments, nothing left to do */ + if (kwnames != NULL) { + if (!ufunc->core_enabled) { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments(ufunc->name, args + len_args, 0, kwnames, + "$out", NULL, &out_obj, + "$where", NULL, &where_obj, + "$casting", NULL, &casting_obj, + "$order", NULL, &order_obj, + "$subok", NULL, &subok_obj, + "$dtype", NULL, &dtype_obj, + "$signature", NULL, &signature_obj, + "$sig", NULL, &sig_obj, + "$extobj", NULL, &extobj, + NULL, NULL, NULL) < 0) { + goto fail; + } + } + else { + NPY_PREPARE_ARGPARSER; + + if (npy_parse_arguments(ufunc->name, args + len_args, 0, kwnames, + "$out", NULL, &out_obj, + "$axes", NULL, &axes_obj, + "$axis", NULL, &axis_obj, + "$keepdims", NULL, &keepdims_obj, + "$casting", NULL, &casting_obj, + "$order", NULL, &order_obj, + "$subok", NULL, &subok_obj, + "$dtype", NULL, &dtype_obj, + "$signature", NULL, &signature_obj, + "$sig", NULL, &sig_obj, + "$extobj", NULL, &extobj, + NULL, NULL, NULL) < 0) { + goto fail; + } + if (NPY_UNLIKELY((axes_obj != NULL) && (axis_obj != NULL))) { + PyErr_SetString(PyExc_TypeError, + "cannot specify both 'axis' and 'axes'"); + goto fail; + } + } + + /* Handle `out` arguments passed by keyword */ + if (out_obj != NULL) { + if (out_is_passed_by_position) { + PyErr_SetString(PyExc_TypeError, + "cannot specify 'out' as both a " + "positional and keyword argument"); + goto fail; + } + if (_set_full_args_out(nout, out_obj, &full_args) < 0) { + goto fail; + } + } + /* + * Only one of signature, sig, and dtype should be passed. If `sig` + * was passed, this puts it into `signature_obj` instead (these + * are borrowed references). + */ + if (_check_and_copy_sig_to_signature( + sig_obj, signature_obj, dtype_obj, &signature_obj) < 0) { + goto fail; + } + } + + char *method; + if (!outer) { + method = "__call__"; + } + else { + method = "outer"; + } + /* We now have all the information required to check for Overrides */ + errval = PyUFunc_CheckOverride(ufunc, method, + full_args.in, full_args.out, + args, len_args, kwnames, &override); + if (errval) { + goto fail; + } else if (override) { + Py_DECREF(full_args.in); + Py_XDECREF(full_args.out); return override; } - errval = PyUFunc_GenericFunction_int(ufunc, args, kwds, mps); + if (outer) { + /* Outer uses special preparation of inputs (expand dims) */ + PyObject *new_in = prepare_input_arguments_for_outer(full_args.in, ufunc); + if (new_in == NULL) { + goto fail; + } + Py_SETREF(full_args.in, new_in); + } + + /* + * Parse the passed `dtype` or `signature` into an array containing + * PyArray_DTypeMeta and/or None. + */ + if (_get_normalized_typetup(ufunc, dtype_obj, signature_obj, &typetup) < 0) { + goto fail; + } + + NPY_ORDER order = NPY_KEEPORDER; + NPY_CASTING casting = NPY_DEFAULT_ASSIGN_CASTING; + npy_bool subok = NPY_TRUE; + int keepdims = -1; /* We need to know if it was passed */ + PyArrayObject *wheremask = NULL; + if (convert_ufunc_arguments(ufunc, full_args, operands, + order_obj, &order, + casting_obj, &casting, + subok_obj, &subok, + where_obj, &wheremask, + keepdims_obj, &keepdims) < 0) { + goto fail; + } + + if (!ufunc->core_enabled) { + errval = PyUFunc_GenericFunctionInternal(ufunc, operands, + full_args, typetup, extobj, casting, order, subok, + wheremask); + Py_XDECREF(wheremask); + } + else { + errval = PyUFunc_GeneralizedFunctionInternal(ufunc, operands, + full_args, typetup, extobj, casting, order, subok, + axis_obj, axes_obj, keepdims); + } + if (errval < 0) { - return NULL; + goto fail; } /* Free the input references */ - for (i = 0; i < ufunc->nin; i++) { - Py_XDECREF(mps[i]); + for (int i = 0; i < ufunc->nin; i++) { + Py_XSETREF(operands[i], NULL); } /* @@ -4756,15 +4919,10 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) * None --- array-object passed in don't call PyArray_Return * method --- the __array_wrap__ method to call. */ - if (make_full_arg_tuple(&full_args, ufunc->nin, ufunc->nout, args, kwds) < 0) { - goto fail; - } - if (_find_array_wrap(full_args, kwds, wraparr, ufunc->nin, ufunc->nout) < 0) { - goto fail; - } + _find_array_wrap(full_args, subok, wraparr, ufunc->nin, ufunc->nout); /* wrap outputs */ - for (i = 0; i < ufunc->nout; i++) { + for (int i = 0; i < ufunc->nout; i++) { int j = ufunc->nin+i; _ufunc_context context; PyObject *wrapped; @@ -4773,10 +4931,10 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) context.args = full_args; context.out_i = i; - wrapped = _apply_array_wrap(wraparr[i], mps[j], &context); - mps[j] = NULL; /* Prevent fail double-freeing this */ + wrapped = _apply_array_wrap(wraparr[i], operands[j], &context); + operands[j] = NULL; /* Prevent fail double-freeing this */ if (wrapped == NULL) { - for (j = 0; j < i; j++) { + for (int j = 0; j < i; j++) { Py_DECREF(retobj[j]); } goto fail; @@ -4785,9 +4943,9 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) retobj[i] = wrapped; } + Py_XDECREF(typetup); Py_XDECREF(full_args.in); Py_XDECREF(full_args.out); - if (ufunc->nout == 1) { return retobj[0]; } @@ -4795,21 +4953,102 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyTupleObject *ret; ret = (PyTupleObject *)PyTuple_New(ufunc->nout); - for (i = 0; i < ufunc->nout; i++) { + for (int i = 0; i < ufunc->nout; i++) { PyTuple_SET_ITEM(ret, i, retobj[i]); } return (PyObject *)ret; } fail: + Py_XDECREF(typetup); Py_XDECREF(full_args.in); Py_XDECREF(full_args.out); - for (i = ufunc->nin; i < ufunc->nargs; i++) { - Py_XDECREF(mps[i]); + for (int i = 0; i < ufunc->nargs; i++) { + Py_XDECREF(operands[i]); } return NULL; } + +/* + * TODO: The implementation below can be replaced with PyVectorcall_Call + * when available (should be Python 3.8+). + */ +static PyObject * +ufunc_generic_call( + PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) +{ + Py_ssize_t len_args = PyTuple_GET_SIZE(args); + /* + * Wrapper for tp_call to tp_fastcall, to support both on older versions + * of Python. (and generally simplifying support of both versions in the + * same codebase. + */ + if (kwds == NULL) { + return ufunc_generic_fastcall(ufunc, + PySequence_Fast_ITEMS(args), len_args, NULL, NPY_FALSE); + } + + PyObject *new_args[NPY_MAXARGS]; + Py_ssize_t len_kwds = PyDict_Size(kwds); + + if (NPY_UNLIKELY(len_args + len_kwds > NPY_MAXARGS)) { + /* + * We do not have enough scratch-space, so we have to abort; + * In practice this error should not be seen by users. + */ + PyErr_Format(PyExc_ValueError, + "%s() takes from %d to %d positional arguments but " + "%zd were given", + ufunc_get_name_cstr(ufunc) , ufunc->nin, ufunc->nargs, len_args); + return NULL; + } + + /* Copy args into the scratch space */ + for (Py_ssize_t i = 0; i < len_args; i++) { + new_args[i] = PyTuple_GET_ITEM(args, i); + } + + PyObject *kwnames = PyTuple_New(len_kwds); + + PyObject *key, *value; + Py_ssize_t pos = 0; + Py_ssize_t i = 0; + while (PyDict_Next(kwds, &pos, &key, &value)) { + Py_INCREF(key); + PyTuple_SET_ITEM(kwnames, i, key); + new_args[i + len_args] = value; + i++; + } + + PyObject *res = ufunc_generic_fastcall(ufunc, + new_args, len_args, kwnames, NPY_FALSE); + Py_DECREF(kwnames); + return res; +} + + +#if PY_VERSION_HEX >= 0x03080000 +/* + * Implement vectorcallfunc which should be defined with Python 3.8+. + * In principle this could be backported, but the speed gain seems moderate + * since ufunc calls often do not have keyword arguments and always have + * a large overhead. The only user would potentially be cython probably. + */ +static PyObject * +ufunc_generic_vectorcall(PyObject *ufunc, + PyObject *const *args, size_t len_args, PyObject *kwnames) +{ + /* + * Unlike METH_FASTCALL, `len_args` may have a flag to signal that + * args[-1] may be (temporarily) used. So normalize it here. + */ + return ufunc_generic_fastcall((PyUFuncObject *)ufunc, + args, PyVectorcall_NARGS(len_args), kwnames, NPY_FALSE); +} +#endif /* PY_VERSION_HEX >= 0x03080000 */ + + NPY_NO_EXPORT PyObject * ufunc_geterr(PyObject *NPY_UNUSED(dummy), PyObject *args) { @@ -4842,6 +5081,7 @@ ufunc_geterr(PyObject *NPY_UNUSED(dummy), PyObject *args) return res; } + NPY_NO_EXPORT PyObject * ufunc_seterr(PyObject *NPY_UNUSED(dummy), PyObject *args) { @@ -4983,7 +5223,11 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi ufunc->core_dim_flags = NULL; ufunc->userloops = NULL; ufunc->ptr = NULL; +#if PY_VERSION_HEX >= 0x03080000 + ufunc->vectorcall = &ufunc_generic_vectorcall; +#else ufunc->reserved2 = NULL; +#endif ufunc->reserved1 = 0; ufunc->iter_flags = 0; @@ -5016,44 +5260,16 @@ PyUFunc_FromFuncAndDataAndSignatureAndIdentity(PyUFuncGenericFunction *func, voi return (PyObject *)ufunc; } -/* Specify that the loop specified by the given index should use the array of - * input and arrays as the data pointer to the loop. - */ + /*UFUNC_API*/ NPY_NO_EXPORT int -PyUFunc_SetUsesArraysAsData(void **data, size_t i) -{ - /* NumPy 1.19, 2020-01-24 */ - if (DEPRECATE( - "PyUFunc_SetUsesArraysAsData() C-API function is deprecated " - "and expected to be removed rapidly. If you are using it (i.e. see " - "this warning/error), please notify the NumPy developers. " - "It is currently assumed that this function is simply unused and " - "its removal will facilitate the implementation of better " - "approaches.") < 0) { - return -1; - } - data[i] = (void*)PyUFunc_SetUsesArraysAsData; - return 0; -} - -/* - * Return 1 if the given data pointer for the loop specifies that it needs the - * arrays as the data pointer. - * - * NOTE: This is easier to specify with the type_resolver - * in the ufunc object. - * - * TODO: Remove this, since this is already basically broken - * with the addition of the masked inner loops and - * not worth fixing since the new loop selection functions - * have access to the full dtypes and can dynamically allocate - * arbitrary auxiliary data. - */ -static int -_does_loop_use_arrays(void *data) +PyUFunc_SetUsesArraysAsData(void **NPY_UNUSED(data), size_t NPY_UNUSED(i)) { - return (data == PyUFunc_SetUsesArraysAsData); + /* NumPy 1.21, 201-03-29 */ + PyErr_SetString(PyExc_RuntimeError, + "PyUFunc_SetUsesArraysAsData() C-API function has been " + "disabled. It was initially deprecated in NumPy 1.19."); + return -1; } @@ -5209,7 +5425,12 @@ PyUFunc_RegisterLoopForDescr(PyUFuncObject *ufunc, if (cmp == 0 && current != NULL && current->arg_dtypes == NULL) { current->arg_dtypes = PyArray_malloc(ufunc->nargs * sizeof(PyArray_Descr*)); - if (arg_dtypes != NULL) { + if (current->arg_dtypes == NULL) { + PyErr_NoMemory(); + result = -1; + goto done; + } + else if (arg_dtypes != NULL) { for (i = 0; i < ufunc->nargs; i++) { current->arg_dtypes[i] = arg_dtypes[i]; Py_INCREF(current->arg_dtypes[i]); @@ -5417,24 +5638,9 @@ ufunc_traverse(PyUFuncObject *self, visitproc visit, void *arg) * The result has dimensions a.ndim + b.ndim */ static PyObject * -ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) +ufunc_outer(PyUFuncObject *ufunc, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int errval; - PyObject *override = NULL; - PyObject *ret; - PyArrayObject *ap1 = NULL, *ap2 = NULL, *ap_new = NULL; - PyObject *new_args, *tmp; - static PyObject *_numpy_matrix; - - - errval = PyUFunc_CheckOverride(ufunc, "outer", args, kwds, &override); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - if (ufunc->core_enabled) { PyErr_Format(PyExc_TypeError, "method outer is not allowed in ufunc with non-trivial"\ @@ -5449,20 +5655,22 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) return NULL; } - if (PySequence_Length(args) != 2) { + if (len_args != 2) { PyErr_SetString(PyExc_TypeError, "exactly two arguments expected"); return NULL; } - tmp = PySequence_GetItem(args, 0); - if (tmp == NULL) { - return NULL; - } + return ufunc_generic_fastcall(ufunc, args, len_args, kwnames, NPY_TRUE); +} - npy_cache_import( - "numpy", - "matrix", - &_numpy_matrix); + +static PyObject * +prepare_input_arguments_for_outer(PyObject *args, PyUFuncObject *ufunc) +{ + PyArrayObject *ap1 = NULL; + PyObject *tmp; + static PyObject *_numpy_matrix; + npy_cache_import("numpy", "matrix", &_numpy_matrix); const char *matrix_deprecation_msg = ( "%s.outer() was passed a numpy matrix as %s argument. " @@ -5471,6 +5679,8 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) "array to retain the old behaviour. You can use `matrix.A` " "to achieve this."); + tmp = PyTuple_GET_ITEM(args, 0); + if (PyObject_IsInstance(tmp, _numpy_matrix)) { /* DEPRECATED 2020-05-13, NumPy 1.20 */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, @@ -5482,14 +5692,12 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) else { ap1 = (PyArrayObject *) PyArray_FROM_O(tmp); } - Py_DECREF(tmp); if (ap1 == NULL) { return NULL; } - tmp = PySequence_GetItem(args, 1); - if (tmp == NULL) { - return NULL; - } + + PyArrayObject *ap2 = NULL; + tmp = PyTuple_GET_ITEM(args, 1); if (PyObject_IsInstance(tmp, _numpy_matrix)) { /* DEPRECATED 2020-05-13, NumPy 1.20 */ if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, @@ -5502,7 +5710,6 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) else { ap2 = (PyArrayObject *) PyArray_FROM_O(tmp); } - Py_DECREF(tmp); if (ap2 == NULL) { Py_DECREF(ap1); return NULL; @@ -5518,7 +5725,7 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) "maximum supported dimension for an ndarray is %d, but " "`%s.outer()` result would have %d.", NPY_MAXDIMS, ufunc->name, newdims.len); - return NPY_FAIL; + goto fail; } if (newdims.ptr == NULL) { goto fail; @@ -5528,6 +5735,7 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) newshape[i] = 1; } + PyArrayObject *ap_new; ap_new = (PyArrayObject *)PyArray_Newshape(ap1, &newdims, NPY_CORDER); if (ap_new == NULL) { goto fail; @@ -5543,71 +5751,42 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) "To work around this issue, please convert the inputs to " "numpy arrays.", ufunc->name, Py_TYPE(ap_new)->tp_name); + Py_DECREF(ap_new); goto fail; } - new_args = Py_BuildValue("(OO)", ap_new, ap2); Py_DECREF(ap1); - Py_DECREF(ap2); - Py_DECREF(ap_new); - ret = ufunc_generic_call(ufunc, new_args, kwds); - Py_DECREF(new_args); - return ret; + return Py_BuildValue("(NN)", ap_new, ap2); fail: Py_XDECREF(ap1); Py_XDECREF(ap2); - Py_XDECREF(ap_new); return NULL; } static PyObject * -ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) +ufunc_reduce(PyUFuncObject *ufunc, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int errval; - PyObject *override = NULL; - - errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - return PyUFunc_GenericReduction(ufunc, args, kwds, UFUNC_REDUCE); + return PyUFunc_GenericReduction( + ufunc, args, len_args, kwnames, UFUNC_REDUCE); } static PyObject * -ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) +ufunc_accumulate(PyUFuncObject *ufunc, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int errval; - PyObject *override = NULL; - - errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - return PyUFunc_GenericReduction(ufunc, args, kwds, UFUNC_ACCUMULATE); + return PyUFunc_GenericReduction( + ufunc, args, len_args, kwnames, UFUNC_ACCUMULATE); } static PyObject * -ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) +ufunc_reduceat(PyUFuncObject *ufunc, + PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int errval; - PyObject *override = NULL; - - errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - return PyUFunc_GenericReduction(ufunc, args, kwds, UFUNC_REDUCEAT); + return PyUFunc_GenericReduction( + ufunc, args, len_args, kwnames, UFUNC_REDUCEAT); } /* Helper for ufunc_at, below */ @@ -5664,14 +5843,6 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) char * err_msg = NULL; NPY_BEGIN_THREADS_DEF; - errval = PyUFunc_CheckOverride(ufunc, "at", args, NULL, &override); - if (errval) { - return NULL; - } - else if (override) { - return override; - } - if (ufunc->nin > 2) { PyErr_SetString(PyExc_ValueError, "Only unary and binary ufuncs supported at this time"); @@ -5693,6 +5864,15 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) "second operand needed for ufunc"); return NULL; } + errval = PyUFunc_CheckOverride(ufunc, "at", + args, NULL, NULL, 0, NULL, &override); + + if (errval) { + return NULL; + } + else if (override) { + return override; + } if (!PyArray_Check(op1)) { PyErr_SetString(PyExc_TypeError, @@ -5945,16 +6125,16 @@ ufunc_at(PyUFuncObject *ufunc, PyObject *args) static struct PyMethodDef ufunc_methods[] = { {"reduce", (PyCFunction)ufunc_reduce, - METH_VARARGS | METH_KEYWORDS, NULL }, + METH_FASTCALL | METH_KEYWORDS, NULL }, {"accumulate", (PyCFunction)ufunc_accumulate, - METH_VARARGS | METH_KEYWORDS, NULL }, + METH_FASTCALL | METH_KEYWORDS, NULL }, {"reduceat", (PyCFunction)ufunc_reduceat, - METH_VARARGS | METH_KEYWORDS, NULL }, + METH_FASTCALL | METH_KEYWORDS, NULL }, {"outer", (PyCFunction)ufunc_outer, - METH_VARARGS | METH_KEYWORDS, NULL}, + METH_FASTCALL | METH_KEYWORDS, NULL}, {"at", (PyCFunction)ufunc_at, METH_VARARGS, NULL}, @@ -6141,10 +6321,17 @@ NPY_NO_EXPORT PyTypeObject PyUFunc_Type = { .tp_repr = (reprfunc)ufunc_repr, .tp_call = (ternaryfunc)ufunc_generic_call, .tp_str = (reprfunc)ufunc_repr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_flags = Py_TPFLAGS_DEFAULT | +#if PY_VERSION_HEX >= 0x03080000 + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_HAVE_GC, .tp_traverse = (traverseproc)ufunc_traverse, .tp_methods = ufunc_methods, .tp_getset = ufunc_getset, +#if PY_VERSION_HEX >= 0x03080000 + .tp_vectorcall_offset = offsetof(PyUFuncObject, vectorcall), +#endif }; /* End of code for ufunc objects */ diff --git a/numpy/core/src/umath/ufunc_object.h b/numpy/core/src/umath/ufunc_object.h index f5de9f9b79d1..6d4fed7c02d2 100644 --- a/numpy/core/src/umath/ufunc_object.h +++ b/numpy/core/src/umath/ufunc_object.h @@ -13,22 +13,8 @@ NPY_NO_EXPORT const char* ufunc_get_name_cstr(PyUFuncObject *ufunc); /* strings from umathmodule.c that are interned on umath import */ -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_out; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_where; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_axes; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_axis; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_keepdims; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_casting; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_order; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_dtype; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_subok; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_signature; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_sig; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_extobj; NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_array_prepare; NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_array_wrap; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_array_finalize; -NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_ufunc; NPY_VISIBILITY_HIDDEN extern PyObject *npy_um_str_pyvals_name; #endif diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 3abeb2c5a202..2834235e409f 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -12,6 +12,11 @@ #define _MULTIARRAYMODULE #define NPY_NO_DEPRECATED_API NPY_API_VERSION +// printif debug tracing +#ifndef NPY_UF_DBG_TRACING + #define NPY_UF_DBG_TRACING 0 +#endif + #include #include "Python.h" @@ -106,14 +111,18 @@ raise_no_loop_found_error( return -1; } for (i = 0; i < ufunc->nargs; ++i) { - Py_INCREF(dtypes[i]); - PyTuple_SET_ITEM(dtypes_tup, i, (PyObject *)dtypes[i]); + PyObject *tmp = Py_None; + if (dtypes[i] != NULL) { + tmp = (PyObject *)dtypes[i]; + } + Py_INCREF(tmp); + PyTuple_SET_ITEM(dtypes_tup, i, tmp); } /* produce an error object */ exc_value = PyTuple_Pack(2, ufunc, dtypes_tup); Py_DECREF(dtypes_tup); - if (exc_value == NULL){ + if (exc_value == NULL) { return -1; } PyErr_SetObject(exc_type, exc_value); @@ -279,7 +288,7 @@ PyUFunc_DefaultTypeResolver(PyUFuncObject *ufunc, } else { /* Find the specified ufunc inner loop, and fill in the dtypes */ retval = type_tuple_type_resolver(ufunc, type_tup, - operands, casting, any_object, out_dtypes); + operands, input_casting, casting, any_object, out_dtypes); } return retval; @@ -324,39 +333,65 @@ PyUFunc_SimpleBinaryComparisonTypeResolver(PyUFuncObject *ufunc, } if (type_tup == NULL) { - /* Input types are the result type */ - out_dtypes[0] = PyArray_ResultType(2, operands, 0, NULL); - if (out_dtypes[0] == NULL) { - return -1; + /* + * DEPRECATED NumPy 1.20, 2020-12. + * This check is required to avoid the FutureWarning that + * ResultType will give for number->string promotions. + * (We never supported flexible dtypes here.) + */ + if (!PyArray_ISFLEXIBLE(operands[0]) && + !PyArray_ISFLEXIBLE(operands[1])) { + out_dtypes[0] = PyArray_ResultType(2, operands, 0, NULL); + if (out_dtypes[0] == NULL) { + return -1; + } + } + else { + /* Not doing anything will lead to a loop no found error. */ + out_dtypes[0] = PyArray_DESCR(operands[0]); + Py_INCREF(out_dtypes[0]); } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); } else { - PyObject *item; - PyArray_Descr *dtype = NULL; - + PyArray_Descr *descr; /* - * If the type tuple isn't a single-element tuple, let the - * default type resolution handle this one. + * If the type tuple was originally a single element (probably), + * issue a deprecation warning, but otherwise accept it. Since the + * result dtype is always boolean, this is not actually valid unless it + * is `object` (but if there is an object input we already deferred). */ - if (!PyTuple_Check(type_tup) || PyTuple_GET_SIZE(type_tup) != 1) { + if (PyTuple_Check(type_tup) && PyTuple_GET_SIZE(type_tup) == 3 && + PyTuple_GET_ITEM(type_tup, 0) == Py_None && + PyTuple_GET_ITEM(type_tup, 1) == Py_None && + PyArray_DescrCheck(PyTuple_GET_ITEM(type_tup, 2))) { + descr = (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 2); + if (descr->type_num == NPY_OBJECT) { + if (DEPRECATE_FUTUREWARNING( + "using `dtype=object` (or equivalent signature) will " + "return object arrays in the future also when the " + "inputs do not already have `object` dtype.") < 0) { + return -1; + } + } + else if (descr->type_num != NPY_BOOL) { + if (DEPRECATE( + "using `dtype=` in comparisons is only useful for " + "`dtype=object` (and will do nothing for bool). " + "This operation will fail in the future.") < 0) { + return -1; + } + } + } + else { + /* Usually a failure, but let the the default version handle it */ return PyUFunc_DefaultTypeResolver(ufunc, casting, operands, type_tup, out_dtypes); } - item = PyTuple_GET_ITEM(type_tup, 0); - - if (item == Py_None) { - PyErr_SetString(PyExc_ValueError, - "require data type in the type tuple"); - return -1; - } - else if (!PyArray_DescrConverter(item, &dtype)) { - return -1; - } - - out_dtypes[0] = ensure_dtype_nbo(dtype); + Py_INCREF(descr); + out_dtypes[0] = ensure_dtype_nbo(descr); if (out_dtypes[0] == NULL) { return -1; } @@ -483,6 +518,35 @@ PyUFunc_SimpleUniformOperationTypeResolver( out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); } else { + int iop; + npy_bool has_flexible = 0; + npy_bool has_object = 0; + for (iop = 0; iop < ufunc->nin; iop++) { + if (PyArray_ISOBJECT(operands[iop])) { + has_object = 1; + } + if (PyArray_ISFLEXIBLE(operands[iop])) { + has_flexible = 1; + } + } + if (NPY_UNLIKELY(has_flexible && !has_object)) { + /* + * DEPRECATED NumPy 1.20, 2020-12. + * This check is required to avoid the FutureWarning that + * ResultType will give for number->string promotions. + * (We never supported flexible dtypes here.) + */ + for (iop = 0; iop < ufunc->nin; iop++) { + out_dtypes[iop] = PyArray_DESCR(operands[iop]); + Py_INCREF(out_dtypes[iop]); + } + raise_no_loop_found_error(ufunc, out_dtypes); + for (iop = 0; iop < ufunc->nin; iop++) { + Py_DECREF(out_dtypes[iop]); + out_dtypes[iop] = NULL; + } + return -1; + } out_dtypes[0] = PyArray_ResultType(ufunc->nin, operands, 0, NULL); } if (out_dtypes[0] == NULL) { @@ -490,34 +554,52 @@ PyUFunc_SimpleUniformOperationTypeResolver( } } else { - PyObject *item; - PyArray_Descr *dtype = NULL; - /* - * If the type tuple isn't a single-element tuple, let the - * default type resolution handle this one. + * This is a fast-path, since all descriptors will be identical, mainly + * when only a single descriptor was passed (which would set the out + * one in the tuple), there is no need to check all loops. + * Note that this also allows (None, None, float64) to resolve to + * (float64, float64, float64), even when the inputs do not match, + * i.e. fixing the output part of the signature can fix all of them. + * This is necessary to support `nextafter(1., inf, dtype=float32)`, + * where it is "clear" we want to cast 1. and inf to float32. */ - if (!PyTuple_Check(type_tup) || PyTuple_GET_SIZE(type_tup) != 1) { + PyArray_Descr *descr = NULL; + if (PyTuple_CheckExact(type_tup) && + PyTuple_GET_SIZE(type_tup) == nop) { + for (int i = 0; i < nop; i++) { + PyObject *item = PyTuple_GET_ITEM(type_tup, i); + if (item == Py_None) { + if (i < ufunc->nin) { + continue; + } + /* All outputs must be set (this could be relaxed) */ + descr = NULL; + break; + } + if (!PyArray_DescrCheck(item)) { + /* Defer to default resolver (will raise an error there) */ + descr = NULL; + break; + } + if (descr != NULL && descr != (PyArray_Descr *)item) { + /* Descriptor mismatch: try with default (probable error) */ + descr = NULL; + break; + } + descr = (PyArray_Descr *)item; + } + } + if (descr == NULL) { + /* in all bad/unlikely cases, use the default type resolver: */ return PyUFunc_DefaultTypeResolver(ufunc, casting, operands, type_tup, out_dtypes); } - - item = PyTuple_GET_ITEM(type_tup, 0); - - if (item == Py_None) { - PyErr_SetString(PyExc_ValueError, - "require data type in the type tuple"); - return -1; - } - else if (!PyArray_DescrConverter(item, &dtype)) { - return -1; - } - - out_dtypes[0] = ensure_dtype_nbo(dtype); - Py_DECREF(dtype); - if (out_dtypes[0] == NULL) { - return -1; + else if (descr->type_num == PyArray_DESCR(operands[0])->type_num) { + /* Prefer the input descriptor if it matches (preserve metadata) */ + descr = PyArray_DESCR(operands[0]); } + out_dtypes[0] = ensure_dtype_nbo(descr); } /* All types are the same - copy the first one to the rest */ @@ -1589,6 +1671,9 @@ ufunc_loop_matches(PyUFuncObject *self, if (types[i] == NPY_OBJECT && !any_object && self->ntypes > 1) { return 0; } + if (types[i] == NPY_NOTYPE) { + continue; /* Matched by being explicitly specified. */ + } /* * If type num is NPY_VOID and struct dtypes have been passed in, @@ -1638,6 +1723,9 @@ ufunc_loop_matches(PyUFuncObject *self, * outputs. */ for (i = nin; i < nop; ++i) { + if (types[i] == NPY_NOTYPE) { + continue; /* Matched by being explicitly specified. */ + } if (op[i] != NULL) { PyArray_Descr *tmp = PyArray_DescrFromType(types[i]); if (tmp == NULL) { @@ -1656,7 +1744,6 @@ ufunc_loop_matches(PyUFuncObject *self, Py_DECREF(tmp); } } - return 1; } @@ -1797,12 +1884,15 @@ type_tuple_userloop_type_resolver(PyUFuncObject *self, int n_specified, int *specified_types, PyArrayObject **op, + NPY_CASTING input_casting, NPY_CASTING casting, int any_object, int use_min_scalar, PyArray_Descr **out_dtype) { int i, j, nin = self->nin, nop = nin + self->nout; + assert(n_specified == nop); + int types[NPY_MAXARGS]; /* Use this to try to avoid repeating the same userdef loop search */ int last_userdef = -1; @@ -1835,28 +1925,31 @@ type_tuple_userloop_type_resolver(PyUFuncObject *self, return -1; } for (; funcdata != NULL; funcdata = funcdata->next) { - int *types = funcdata->arg_types; - int matched = 1; - - if (n_specified == nop) { - for (j = 0; j < nop; ++j) { - if (types[j] != specified_types[j] && - specified_types[j] != NPY_NOTYPE) { - matched = 0; - break; - } + int *orig_types = funcdata->arg_types; + + /* + * Copy the types into an int array for matching + * (Mostly duplicated in `type_tuple_type_resolver`) + */ + for (j = 0; j < nop; ++j) { + if (specified_types[j] == NPY_NOTYPE) { + types[j] = orig_types[j]; + continue; } - } else { - if (types[nin] != specified_types[0]) { - matched = 0; + if (orig_types[j] != specified_types[j]) { + break; } + /* indicate that we do not have to check this type anymore. */ + types[j] = NPY_NOTYPE; } - if (!matched) { + + if (j != nop) { + /* no match */ continue; } switch (ufunc_loop_matches(self, op, - casting, casting, + input_casting, casting, any_object, use_min_scalar, types, NULL, &no_castable_output, &err_src_typecode, @@ -1864,7 +1957,19 @@ type_tuple_userloop_type_resolver(PyUFuncObject *self, /* It works */ case 1: set_ufunc_loop_data_types(self, op, - out_dtype, types, NULL); + out_dtype, orig_types, NULL); + /* + * In principle, we only need to validate the + * NPY_NOTYPE ones + */ + if (PyUFunc_ValidateCasting(self, + casting, op, out_dtype) < 0) { + for (j = 0; j < self->nargs; j++) { + Py_DECREF(out_dtype[j]); + out_dtype[j] = NULL; + } + return -1; + } return 1; /* Didn't match */ case 0: @@ -1997,6 +2102,94 @@ linear_search_type_resolver(PyUFuncObject *self, return -1; } + +static int +type_tuple_type_resolver_core(PyUFuncObject *self, + PyArrayObject **op, + NPY_CASTING input_casting, NPY_CASTING casting, + int specified_types[], + int any_object, + int no_castable_output, int use_min_scalar, + PyArray_Descr **out_dtype) +{ + int i, j; + int nop = self->nargs; + int types[NPY_MAXARGS]; + + /* For making a better error message on coercion error */ + char err_dst_typecode = '-', err_src_typecode = '-'; + + /* If the ufunc has userloops, search for them. */ + if (self->userloops) { + switch (type_tuple_userloop_type_resolver(self, + nop, specified_types, + op, input_casting, casting, + any_object, use_min_scalar, + out_dtype)) { + /* Error */ + case -1: + return -1; + /* Found matching loop */ + case 1: + return 0; + } + } + + for (i = 0; i < self->ntypes; ++i) { + char *orig_types = self->types + i*self->nargs; + + /* + * Check specified types and copy into an int array for matching + * (Mostly duplicated in `type_tuple_userloop_type_resolver`) + */ + for (j = 0; j < nop; ++j) { + if (specified_types[j] == NPY_NOTYPE) { + types[j] = orig_types[j]; + continue; + } + if (orig_types[j] != specified_types[j]) { + break; + } + /* indicate that we do not have to check this type anymore. */ + types[j] = NPY_NOTYPE; + } + if (j < nop) { + /* no match */ + continue; + } + + switch (ufunc_loop_matches(self, op, + input_casting, casting, + any_object, use_min_scalar, + types, NULL, + &no_castable_output, &err_src_typecode, + &err_dst_typecode)) { + case -1: + /* Error */ + return -1; + case 0: + /* Cannot cast inputs */ + continue; + case 1: + /* Success, fill also the NPY_NOTYPE (cast from char to int) */ + for (j = 0; j < nop; j++) { + types[j] = orig_types[j]; + } + set_ufunc_loop_data_types(self, op, out_dtype, types, NULL); + /* In principle, we only need to validate the NPY_NOTYPE ones */ + if (PyUFunc_ValidateCasting(self, casting, op, out_dtype) < 0) { + for (j = 0; j < self->nargs; j++) { + Py_DECREF(out_dtype[j]); + out_dtype[j] = NULL; + } + return -1; + } + return 0; + } + } + return -2; +} + /* * Does a linear search for the inner loop of the ufunc specified by type_tup. * @@ -2007,176 +2200,101 @@ NPY_NO_EXPORT int type_tuple_type_resolver(PyUFuncObject *self, PyObject *type_tup, PyArrayObject **op, + NPY_CASTING input_casting, NPY_CASTING casting, int any_object, PyArray_Descr **out_dtype) { - npy_intp i, j, n, nin = self->nin, nop = nin + self->nout; - int n_specified = 0; - int specified_types[NPY_MAXARGS], types[NPY_MAXARGS]; + int nin = self->nin, nop = nin + self->nout; + int specified_types[NPY_MAXARGS]; const char *ufunc_name; - int no_castable_output, use_min_scalar; - - /* For making a better error message on coercion error */ - char err_dst_typecode = '-', err_src_typecode = '-'; + int no_castable_output = 0, use_min_scalar; ufunc_name = ufunc_get_name_cstr(self); use_min_scalar = should_use_min_scalar(nin, op, 0, NULL); /* Fill in specified_types from the tuple or string */ - if (PyTuple_Check(type_tup)) { - int nonecount = 0; - n = PyTuple_GET_SIZE(type_tup); - if (n != 1 && n != nop) { - PyErr_Format(PyExc_ValueError, - "a type-tuple must be specified " - "of length 1 or %d for ufunc '%s'", (int)nop, - ufunc_get_name_cstr(self)); + const char *bad_type_tup_msg = ( + "Only NumPy must call `ufunc->type_resolver()` explicitly. " + "NumPy ensures that a type-tuple is normalized now to be a tuple " + "only containing None or descriptors. If anything else is passed " + "(you are seeing this message), the `type_resolver()` was called " + "directly by a third party. " + "This is unexpected, please inform the NumPy developers about it. " + "Also note that `type_resolver` will be phased out, since it must " + "be replaced."); + + if (PyTuple_CheckExact(type_tup)) { + Py_ssize_t n = PyTuple_GET_SIZE(type_tup); + if (n != nop) { + PyErr_SetString(PyExc_RuntimeError, bad_type_tup_msg); return -1; } - - for (i = 0; i < n; ++i) { + for (int i = 0; i < nop; ++i) { PyObject *item = PyTuple_GET_ITEM(type_tup, i); if (item == Py_None) { specified_types[i] = NPY_NOTYPE; - ++nonecount; } else { - PyArray_Descr *dtype = NULL; - if (!PyArray_DescrConverter(item, &dtype)) { + if (!PyArray_DescrCheck(item)) { + PyErr_SetString(PyExc_RuntimeError, bad_type_tup_msg); return -1; } - specified_types[i] = dtype->type_num; - Py_DECREF(dtype); + specified_types[i] = ((PyArray_Descr *)item)->type_num; } } - - if (nonecount == n) { - PyErr_SetString(PyExc_ValueError, - "the type-tuple provided to the ufunc " - "must specify at least one none-None dtype"); - return -1; - } - - n_specified = n; } - else if (PyBytes_Check(type_tup) || PyUnicode_Check(type_tup)) { - Py_ssize_t length; - char *str; - PyObject *str_obj = NULL; - - if (PyUnicode_Check(type_tup)) { - str_obj = PyUnicode_AsASCIIString(type_tup); - if (str_obj == NULL) { - return -1; - } - type_tup = str_obj; - } - - if (PyBytes_AsStringAndSize(type_tup, &str, &length) < 0) { - Py_XDECREF(str_obj); - return -1; - } - if (length != 1 && (length != nop + 2 || - str[nin] != '-' || str[nin+1] != '>')) { - PyErr_Format(PyExc_ValueError, - "a type-string for %s, " \ - "requires 1 typecode, or " - "%d typecode(s) before " \ - "and %d after the -> sign", - ufunc_get_name_cstr(self), - self->nin, self->nout); - Py_XDECREF(str_obj); - return -1; - } - if (length == 1) { - PyArray_Descr *dtype; - n_specified = 1; - dtype = PyArray_DescrFromType(str[0]); - if (dtype == NULL) { - Py_XDECREF(str_obj); - return -1; - } - specified_types[0] = dtype->type_num; - Py_DECREF(dtype); - } - else { - PyArray_Descr *dtype; - n_specified = (int)nop; + else { + PyErr_SetString(PyExc_RuntimeError, bad_type_tup_msg); + return -1; + } - for (i = 0; i < nop; ++i) { - npy_intp istr = i < nin ? i : i+2; + int res = type_tuple_type_resolver_core(self, + op, input_casting, casting, specified_types, any_object, + no_castable_output, use_min_scalar, out_dtype); - dtype = PyArray_DescrFromType(str[istr]); - if (dtype == NULL) { - Py_XDECREF(str_obj); - return -1; - } - specified_types[i] = dtype->type_num; - Py_DECREF(dtype); - } - } - Py_XDECREF(str_obj); + if (res != -2) { + return res; } - /* If the ufunc has userloops, search for them. */ - if (self->userloops) { - switch (type_tuple_userloop_type_resolver(self, - n_specified, specified_types, - op, casting, - any_object, use_min_scalar, - out_dtype)) { - /* Error */ - case -1: - return -1; - /* Found matching loop */ - case 1: - return 0; + /* + * When the user passes `dtype=dtype`, it gets translated to + * `signature=(None,)*nin + (dtype,)*nout`. If the signature matches that + * exactly (could be relaxed but that is not necessary for backcompat), + * we also try `signature=(dtype,)*(nin+nout)`. + * This used to be the main meaning for `dtype=dtype`, but some calls broke + * the expectation, and changing it allows for `dtype=dtype` to be useful + * for ufuncs like `np.ldexp` in the future while also normalizing it to + * a `signature` early on. + */ + int homogeneous_type = NPY_NOTYPE; + if (self->nout > 0) { + homogeneous_type = specified_types[nin]; + for (int i = nin+1; i < nop; i++) { + if (specified_types[i] != homogeneous_type) { + homogeneous_type = NPY_NOTYPE; + break; + } } } - - for (i = 0; i < self->ntypes; ++i) { - char *orig_types = self->types + i*self->nargs; - - /* Copy the types into an int array for matching */ - for (j = 0; j < nop; ++j) { - types[j] = orig_types[j]; - } - - if (n_specified == nop) { - for (j = 0; j < nop; ++j) { - if (types[j] != specified_types[j] && - specified_types[j] != NPY_NOTYPE) { - break; - } - } - if (j < nop) { - /* no match */ - continue; + if (homogeneous_type != NPY_NOTYPE) { + for (int i = 0; i < nin; i++) { + if (specified_types[i] != NPY_NOTYPE) { + homogeneous_type = NPY_NOTYPE; + break; } + specified_types[i] = homogeneous_type; } - else if (types[nin] != specified_types[0]) { - /* no match */ - continue; - } + } + if (homogeneous_type != NPY_NOTYPE) { + /* Try again with the homogeneous specified types. */ + res = type_tuple_type_resolver_core(self, + op, input_casting, casting, specified_types, any_object, + no_castable_output, use_min_scalar, out_dtype); - switch (ufunc_loop_matches(self, op, - casting, casting, - any_object, use_min_scalar, - types, NULL, - &no_castable_output, &err_src_typecode, - &err_dst_typecode)) { - case -1: - /* Error */ - return -1; - case 0: - /* Cannot cast inputs */ - continue; - case 1: - /* Success */ - set_ufunc_loop_data_types(self, op, out_dtype, types, NULL); - return 0; + if (res != -2) { + return res; } } diff --git a/numpy/core/src/umath/ufunc_type_resolution.h b/numpy/core/src/umath/ufunc_type_resolution.h index 1d6ad3358773..b11c69852889 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.h +++ b/numpy/core/src/umath/ufunc_type_resolution.h @@ -123,6 +123,7 @@ NPY_NO_EXPORT int type_tuple_type_resolver(PyUFuncObject *self, PyObject *type_tup, PyArrayObject **op, + NPY_CASTING input_casting, NPY_CASTING casting, int any_object, PyArray_Descr **out_dtype); diff --git a/numpy/core/src/umath/umathmodule.c b/numpy/core/src/umath/umathmodule.c index 474db02453b5..b4b7db760c64 100644 --- a/numpy/core/src/umath/umathmodule.c +++ b/numpy/core/src/umath/umathmodule.c @@ -216,44 +216,16 @@ add_newdoc_ufunc(PyObject *NPY_UNUSED(dummy), PyObject *args) ***************************************************************************** */ -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_out = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_where = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_axes = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_axis = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_keepdims = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_casting = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_order = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_dtype = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_subok = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_signature = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_sig = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_extobj = NULL; NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_array_prepare = NULL; NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_array_wrap = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_array_finalize = NULL; -NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_ufunc = NULL; NPY_VISIBILITY_HIDDEN PyObject *npy_um_str_pyvals_name = NULL; /* intern some strings used in ufuncs, returns 0 on success */ static int intern_strings(void) { - if (!(npy_um_str_out = PyUnicode_InternFromString("out"))) return -1; - if (!(npy_um_str_where = PyUnicode_InternFromString("where"))) return -1; - if (!(npy_um_str_axes = PyUnicode_InternFromString("axes"))) return -1; - if (!(npy_um_str_axis = PyUnicode_InternFromString("axis"))) return -1; - if (!(npy_um_str_keepdims = PyUnicode_InternFromString("keepdims"))) return -1; - if (!(npy_um_str_casting = PyUnicode_InternFromString("casting"))) return -1; - if (!(npy_um_str_order = PyUnicode_InternFromString("order"))) return -1; - if (!(npy_um_str_dtype = PyUnicode_InternFromString("dtype"))) return -1; - if (!(npy_um_str_subok = PyUnicode_InternFromString("subok"))) return -1; - if (!(npy_um_str_signature = PyUnicode_InternFromString("signature"))) return -1; - if (!(npy_um_str_sig = PyUnicode_InternFromString("sig"))) return -1; - if (!(npy_um_str_extobj = PyUnicode_InternFromString("extobj"))) return -1; if (!(npy_um_str_array_prepare = PyUnicode_InternFromString("__array_prepare__"))) return -1; if (!(npy_um_str_array_wrap = PyUnicode_InternFromString("__array_wrap__"))) return -1; - if (!(npy_um_str_array_finalize = PyUnicode_InternFromString("__array_finalize__"))) return -1; - if (!(npy_um_str_ufunc = PyUnicode_InternFromString("__array_ufunc__"))) return -1; if (!(npy_um_str_pyvals_name = PyUnicode_InternFromString(UFUNC_PYVALS_NAME))) return -1; return 0; } diff --git a/numpy/core/tests/data/umath-validation-set-README b/numpy/core/tests/data/umath-validation-set-README.txt similarity index 82% rename from numpy/core/tests/data/umath-validation-set-README rename to numpy/core/tests/data/umath-validation-set-README.txt index 6561ca3b5191..cfc9e4145d10 100644 --- a/numpy/core/tests/data/umath-validation-set-README +++ b/numpy/core/tests/data/umath-validation-set-README.txt @@ -1,5 +1,5 @@ Steps to validate transcendental functions: -1) Add a file 'umath-validation-set-', where ufuncname is name of +1) Add a file 'umath-validation-set-.txt', where ufuncname is name of the function in NumPy you want to validate 2) The file should contain 4 columns: dtype,input,expected output,ulperror a. dtype: one of np.float16, np.float32, np.float64 @@ -11,5 +11,5 @@ Steps to validate transcendental functions: d. ulperror: expected maximum ulp error of the function. This should be same across all rows of the same dtype. Otherwise, the function is tested for the maximum ulp error among all entries of that dtype. -3) Add file umath-validation-set- to the test file test_umath_accuracy.py +3) Add file umath-validation-set-.txt to the test file test_umath_accuracy.py which will then validate your ufunc. diff --git a/numpy/core/tests/data/umath-validation-set-cos b/numpy/core/tests/data/umath-validation-set-cos.csv similarity index 100% rename from numpy/core/tests/data/umath-validation-set-cos rename to numpy/core/tests/data/umath-validation-set-cos.csv diff --git a/numpy/core/tests/data/umath-validation-set-exp b/numpy/core/tests/data/umath-validation-set-exp.csv similarity index 100% rename from numpy/core/tests/data/umath-validation-set-exp rename to numpy/core/tests/data/umath-validation-set-exp.csv diff --git a/numpy/core/tests/data/umath-validation-set-log b/numpy/core/tests/data/umath-validation-set-log.csv similarity index 100% rename from numpy/core/tests/data/umath-validation-set-log rename to numpy/core/tests/data/umath-validation-set-log.csv diff --git a/numpy/core/tests/data/umath-validation-set-sin b/numpy/core/tests/data/umath-validation-set-sin.csv similarity index 100% rename from numpy/core/tests/data/umath-validation-set-sin rename to numpy/core/tests/data/umath-validation-set-sin.csv diff --git a/numpy/core/tests/test_api.py b/numpy/core/tests/test_api.py index 0f42f7076b02..291cdae8908f 100644 --- a/numpy/core/tests/test_api.py +++ b/numpy/core/tests/test_api.py @@ -149,7 +149,7 @@ def test_array_impossible_casts(array): rt = rational(1, 2) if array: rt = np.array(rt) - with assert_raises(ValueError): + with assert_raises(TypeError): np.array(rt, dtype="M8") @@ -281,6 +281,19 @@ class MyNDArray(np.ndarray): a = np.array(1000, dtype='i4') assert_raises(TypeError, a.astype, 'U1', casting='safe') +@pytest.mark.parametrize("dt", ["S", "U"]) +def test_array_astype_to_string_discovery_empty(dt): + # See also gh-19085 + arr = np.array([""], dtype=object) + # Note, the itemsize is the `0 -> 1` logic, which should change. + # The important part the test is rather that it does not error. + assert arr.astype(dt).dtype.itemsize == np.dtype(f"{dt}1").itemsize + + # check the same thing for `np.can_cast` (since it accepts arrays) + assert np.can_cast(arr, dt, casting="unsafe") + assert not np.can_cast(arr, dt, casting="same_kind") + # as well as for the object as a descriptor: + assert np.can_cast("O", dt, casting="unsafe") @pytest.mark.parametrize("dt", ["d", "f", "S13", "U32"]) def test_array_astype_to_void(dt): diff --git a/numpy/core/tests/test_argparse.py b/numpy/core/tests/test_argparse.py new file mode 100644 index 000000000000..63a01dee404f --- /dev/null +++ b/numpy/core/tests/test_argparse.py @@ -0,0 +1,62 @@ +""" +Tests for the private NumPy argument parsing functionality. +They mainly exists to ensure good test coverage without having to try the +weirder cases on actual numpy functions but test them in one place. + +The test function is defined in C to be equivalent to (errors may not always +match exactly, and could be adjusted): + + def func(arg1, /, arg2, *, arg3): + i = integer(arg1) # reproducing the 'i' parsing in Python. + return None +""" + +import pytest + +import numpy as np +from numpy.core._multiarray_tests import argparse_example_function as func + + +def test_invalid_integers(): + with pytest.raises(TypeError, + match="integer argument expected, got float"): + func(1.) + with pytest.raises(OverflowError): + func(2**100) + + +def test_missing_arguments(): + with pytest.raises(TypeError, + match="missing required positional argument 0"): + func() + with pytest.raises(TypeError, + match="missing required positional argument 0"): + func(arg2=1, arg3=4) + with pytest.raises(TypeError, + match=r"missing required argument \'arg2\' \(pos 1\)"): + func(1, arg3=5) + + +def test_too_many_positional(): + # the second argument is positional but can be passed as keyword. + with pytest.raises(TypeError, + match="takes from 2 to 3 positional arguments but 4 were given"): + func(1, 2, 3, 4) + + +def test_multiple_values(): + with pytest.raises(TypeError, + match=r"given by name \('arg2'\) and position \(position 1\)"): + func(1, 2, arg2=3) + + +def test_string_fallbacks(): + # We can (currently?) use numpy strings to test the "slow" fallbacks + # that should normally not be taken due to string interning. + arg2 = np.unicode_("arg2") + missing_arg = np.unicode_("missing_arg") + func(1, **{arg2: 3}) + with pytest.raises(TypeError, + match="got an unexpected keyword argument 'missing_arg'"): + func(2, **{missing_arg: 3}) + diff --git a/numpy/core/tests/test_array_coercion.py b/numpy/core/tests/test_array_coercion.py index 78def936022d..45c792ad2392 100644 --- a/numpy/core/tests/test_array_coercion.py +++ b/numpy/core/tests/test_array_coercion.py @@ -38,8 +38,18 @@ def subclass(a): yield subclass + class _SequenceLike(): + # We are giving a warning that array-like's were also expected to be + # sequence-like in `np.array([array_like])`, this can be removed + # when the deprecation exired (started NumPy 1.20) + def __len__(self): + raise TypeError + + def __getitem__(self): + raise TypeError + # Array-interface - class ArrayDunder: + class ArrayDunder(_SequenceLike): def __init__(self, a): self.a = a @@ -52,7 +62,7 @@ def __array__(self, dtype=None): yield param(memoryview, id="memoryview") # Array-interface - class ArrayInterface: + class ArrayInterface(_SequenceLike): def __init__(self, a): self.a = a # need to hold on to keep interface valid self.__array_interface__ = a.__array_interface__ @@ -60,7 +70,7 @@ def __init__(self, a): yield param(ArrayInterface, id="__array_interface__") # Array-Struct - class ArrayStruct: + class ArrayStruct(_SequenceLike): def __init__(self, a): self.a = a # need to hold on to keep struct valid self.__array_struct__ = a.__array_struct__ @@ -224,6 +234,7 @@ def test_scalar(self, scalar): # Additionally to string this test also runs into a corner case # with datetime promotion (the difference is the promotion order). + @pytest.mark.filterwarnings("ignore:Promotion of numbers:FutureWarning") def test_scalar_promotion(self): for sc1, sc2 in product(scalar_instances(), scalar_instances()): sc1, sc2 = sc1.values[0], sc2.values[0] @@ -689,3 +700,35 @@ def test_too_large_array_error_paths(self): np.array(arr) with pytest.raises(MemoryError): np.array([arr]) + + @pytest.mark.parametrize("attribute", + ["__array_interface__", "__array__", "__array_struct__"]) + @pytest.mark.parametrize("error", [RecursionError, MemoryError]) + def test_bad_array_like_attributes(self, attribute, error): + # RecursionError and MemoryError are considered fatal. All errors + # (except AttributeError) should probably be raised in the future, + # but shapely made use of it, so it will require a deprecation. + + class BadInterface: + def __getattr__(self, attr): + if attr == attribute: + raise error + super().__getattr__(attr) + + with pytest.raises(error): + np.array(BadInterface()) + + @pytest.mark.parametrize("error", [RecursionError, MemoryError]) + def test_bad_array_like_bad_length(self, error): + # RecursionError and MemoryError are considered "critical" in + # sequences. We could expand this more generally though. (NumPy 1.20) + class BadSequence: + def __len__(self): + raise error + def __getitem__(self): + # must have getitem to be a Sequence + return 1 + + with pytest.raises(error): + np.array(BadSequence()) + diff --git a/numpy/core/tests/test_arraymethod.py b/numpy/core/tests/test_arraymethod.py new file mode 100644 index 000000000000..b1bc79b80a63 --- /dev/null +++ b/numpy/core/tests/test_arraymethod.py @@ -0,0 +1,58 @@ +""" +This file tests the generic aspects of ArrayMethod. At the time of writing +this is private API, but when added, public API may be added here. +""" + +import pytest + +import numpy as np +from numpy.core._multiarray_umath import _get_castingimpl as get_castingimpl + + +class TestResolveDescriptors: + # Test mainly error paths of the resolve_descriptors function, + # note that the `casting_unittests` tests exercise this non-error paths. + + # Casting implementations are the main/only current user: + method = get_castingimpl(type(np.dtype("d")), type(np.dtype("f"))) + + @pytest.mark.parametrize("args", [ + (True,), # Not a tuple. + ((None,)), # Too few elements + ((None, None, None),), # Too many + ((None, None),), # Input dtype is None, which is invalid. + ((np.dtype("d"), True),), # Output dtype is not a dtype + ((np.dtype("f"), None),), # Input dtype does not match method + ]) + def test_invalid_arguments(self, args): + with pytest.raises(TypeError): + self.method._resolve_descriptors(*args) + + +class TestSimpleStridedCall: + # Test mainly error paths of the resolve_descriptors function, + # note that the `casting_unittests` tests exercise this non-error paths. + + # Casting implementations are the main/only current user: + method = get_castingimpl(type(np.dtype("d")), type(np.dtype("f"))) + + @pytest.mark.parametrize(["args", "error"], [ + ((True,), TypeError), # Not a tuple + (((None,),), TypeError), # Too few elements + ((None, None), TypeError), # Inputs are not arrays. + (((None, None, None),), TypeError), # Too many + (((np.arange(3), np.arange(3)),), TypeError), # Incorrect dtypes + (((np.ones(3, dtype=">d"), np.ones(3, dtype="= dtype2.itemsize: + length = self.size // dtype1.itemsize + else: + length = self.size // dtype2.itemsize + + # Assume that the base array is well enough aligned for all inputs. + arr1 = np.empty(length, dtype=dtype1) + assert arr1.flags.c_contiguous + assert arr1.flags.aligned + + values = [random.randrange(-128, 128) for _ in range(length)] + + for i, value in enumerate(values): + # Use item assignment to ensure this is not using casting: + arr1[i] = value + + if dtype2 is None: + if dtype1.char == "?": + values = [bool(v) for v in values] + return arr1, values + + if dtype2.char == "?": + values = [bool(v) for v in values] + + arr2 = np.empty(length, dtype=dtype2) + assert arr2.flags.c_contiguous + assert arr2.flags.aligned + + for i, value in enumerate(values): + # Use item assignment to ensure this is not using casting: + arr2[i] = value + + return arr1, arr2, values + + def get_data_variation(self, arr1, arr2, aligned=True, contig=True): + """ + Returns a copy of arr1 that may be non-contiguous or unaligned, and a + matching array for arr2 (although not a copy). + """ + if contig: + stride1 = arr1.dtype.itemsize + stride2 = arr2.dtype.itemsize + elif aligned: + stride1 = 2 * arr1.dtype.itemsize + stride2 = 2 * arr2.dtype.itemsize + else: + stride1 = arr1.dtype.itemsize + 1 + stride2 = arr2.dtype.itemsize + 1 + + max_size1 = len(arr1) * 3 * arr1.dtype.itemsize + 1 + max_size2 = len(arr2) * 3 * arr2.dtype.itemsize + 1 + from_bytes = np.zeros(max_size1, dtype=np.uint8) + to_bytes = np.zeros(max_size2, dtype=np.uint8) + + # Sanity check that the above is large enough: + assert stride1 * len(arr1) <= from_bytes.nbytes + assert stride2 * len(arr2) <= to_bytes.nbytes + + if aligned: + new1 = as_strided(from_bytes[:-1].view(arr1.dtype), + arr1.shape, (stride1,)) + new2 = as_strided(to_bytes[:-1].view(arr2.dtype), + arr2.shape, (stride2,)) + else: + new1 = as_strided(from_bytes[1:].view(arr1.dtype), + arr1.shape, (stride1,)) + new2 = as_strided(to_bytes[1:].view(arr2.dtype), + arr2.shape, (stride2,)) + + new1[...] = arr1 + + if not contig: + # Ensure we did not overwrite bytes that should not be written: + offset = arr1.dtype.itemsize if aligned else 0 + buf = from_bytes[offset::stride1].tobytes() + assert buf.count(b"\0") == len(buf) + + if contig: + assert new1.flags.c_contiguous + assert new2.flags.c_contiguous + else: + assert not new1.flags.c_contiguous + assert not new2.flags.c_contiguous + + if aligned: + assert new1.flags.aligned + assert new2.flags.aligned + else: + assert not new1.flags.aligned or new1.dtype.alignment == 1 + assert not new2.flags.aligned or new2.dtype.alignment == 1 + + return new1, new2 + @pytest.mark.parametrize("from_Dt", simple_dtypes) def test_simple_cancast(self, from_Dt): for to_Dt in simple_dtypes: @@ -193,6 +280,183 @@ def test_simple_cancast(self, from_Dt): assert(to_dt is to_res) + @pytest.mark.filterwarnings("ignore::numpy.ComplexWarning") + @pytest.mark.parametrize("from_dt", simple_dtype_instances()) + def test_simple_direct_casts(self, from_dt): + """ + This test checks numeric direct casts for dtypes supported also by the + struct module (plus complex). It tries to be test a wide range of + inputs, but skips over possibly undefined behaviour (e.g. int rollover). + Longdouble and CLongdouble are tested, but only using double precision. + + If this test creates issues, it should possibly just be simplified + or even removed (checking whether unaligned/non-contiguous casts give + the same results is useful, though). + """ + for to_dt in simple_dtype_instances(): + to_dt = to_dt.values[0] + cast = get_castingimpl(type(from_dt), type(to_dt)) + + casting, (from_res, to_res) = cast._resolve_descriptors( + (from_dt, to_dt)) + + if from_res is not from_dt or to_res is not to_dt: + # Do not test this case, it is handled in multiple steps, + # each of which should is tested individually. + return + + safe = (casting & ~Casting.cast_is_view) <= Casting.safe + del from_res, to_res, casting + + arr1, arr2, values = self.get_data(from_dt, to_dt) + + cast._simple_strided_call((arr1, arr2)) + + # Check via python list + assert arr2.tolist() == values + + # Check that the same results are achieved for strided loops + arr1_o, arr2_o = self.get_data_variation(arr1, arr2, True, False) + cast._simple_strided_call((arr1_o, arr2_o)) + + assert_array_equal(arr2_o, arr2) + assert arr2_o.tobytes() == arr2.tobytes() + + # Check if alignment makes a difference, but only if supported + # and only if the alignment can be wrong + if ((from_dt.alignment == 1 and to_dt.alignment == 1) or + not cast._supports_unaligned): + return + + arr1_o, arr2_o = self.get_data_variation(arr1, arr2, False, True) + cast._simple_strided_call((arr1_o, arr2_o)) + + assert_array_equal(arr2_o, arr2) + assert arr2_o.tobytes() == arr2.tobytes() + + arr1_o, arr2_o = self.get_data_variation(arr1, arr2, False, False) + cast._simple_strided_call((arr1_o, arr2_o)) + + assert_array_equal(arr2_o, arr2) + assert arr2_o.tobytes() == arr2.tobytes() + + del arr1_o, arr2_o, cast + + @pytest.mark.parametrize("from_Dt", simple_dtypes) + def test_numeric_to_times(self, from_Dt): + # We currently only implement contiguous loops, so only need to + # test those. + from_dt = from_Dt() + + time_dtypes = [np.dtype("M8"), np.dtype("M8[ms]"), np.dtype("M8[4D]"), + np.dtype("m8"), np.dtype("m8[ms]"), np.dtype("m8[4D]")] + for time_dt in time_dtypes: + cast = get_castingimpl(type(from_dt), type(time_dt)) + + casting, (from_res, to_res) = cast._resolve_descriptors( + (from_dt, time_dt)) + + assert from_res is from_dt + assert to_res is time_dt + del from_res, to_res + + assert(casting & CAST_TABLE[from_Dt][type(time_dt)]) + + int64_dt = np.dtype(np.int64) + arr1, arr2, values = self.get_data(from_dt, int64_dt) + arr2 = arr2.view(time_dt) + arr2[...] = np.datetime64("NaT") + + if time_dt == np.dtype("M8"): + # This is a bit of a strange path, and could probably be removed + arr1[-1] = 0 # ensure at least one value is not NaT + + # The cast currently succeeds, but the values are invalid: + cast._simple_strided_call((arr1, arr2)) + with pytest.raises(ValueError): + str(arr2[-1]) # e.g. conversion to string fails + return + + cast._simple_strided_call((arr1, arr2)) + + assert [int(v) for v in arr2.tolist()] == values + + # Check that the same results are achieved for strided loops + arr1_o, arr2_o = self.get_data_variation(arr1, arr2, True, False) + cast._simple_strided_call((arr1_o, arr2_o)) + + assert_array_equal(arr2_o, arr2) + assert arr2_o.tobytes() == arr2.tobytes() + + @pytest.mark.parametrize( + ["from_dt", "to_dt", "expected_casting", "nom", "denom"], + [("M8[ns]", None, + Casting.no | Casting.cast_is_view, 1, 1), + (str(np.dtype("M8[ns]").newbyteorder()), None, Casting.equiv, 1, 1), + ("M8", "M8[ms]", Casting.safe | Casting.cast_is_view, 1, 1), + ("M8[ms]", "M8", Casting.unsafe, 1, 1), # should be invalid cast + ("M8[5ms]", "M8[5ms]", Casting.no | Casting.cast_is_view, 1, 1), + ("M8[ns]", "M8[ms]", Casting.same_kind, 1, 10**6), + ("M8[ms]", "M8[ns]", Casting.safe, 10**6, 1), + ("M8[ms]", "M8[7ms]", Casting.same_kind, 1, 7), + ("M8[4D]", "M8[1M]", Casting.same_kind, None, + # give full values based on NumPy 1.19.x + [-2**63, 0, -1, 1314, -1315, 564442610]), + ("m8[ns]", None, Casting.no | Casting.cast_is_view, 1, 1), + (str(np.dtype("m8[ns]").newbyteorder()), None, Casting.equiv, 1, 1), + ("m8", "m8[ms]", Casting.safe | Casting.cast_is_view, 1, 1), + ("m8[ms]", "m8", Casting.unsafe, 1, 1), # should be invalid cast + ("m8[5ms]", "m8[5ms]", Casting.no | Casting.cast_is_view, 1, 1), + ("m8[ns]", "m8[ms]", Casting.same_kind, 1, 10**6), + ("m8[ms]", "m8[ns]", Casting.safe, 10**6, 1), + ("m8[ms]", "m8[7ms]", Casting.same_kind, 1, 7), + ("m8[4D]", "m8[1M]", Casting.unsafe, None, + # give full values based on NumPy 1.19.x + [-2**63, 0, 0, 1314, -1315, 564442610])]) + def test_time_to_time(self, from_dt, to_dt, expected_casting, nom, denom): + from_dt = np.dtype(from_dt) + if to_dt is not None: + to_dt = np.dtype(to_dt) + + # Test a few values for casting (results generated with NumPy 1.19) + values = np.array([-2**63, 1, 2**63-1, 10000, -10000, 2**32]) + values = values.astype(np.dtype("int64").newbyteorder(from_dt.byteorder)) + assert values.dtype.byteorder == from_dt.byteorder + assert np.isnat(values.view(from_dt)[0]) + + DType = type(from_dt) + cast = get_castingimpl(DType, DType) + casting, (from_res, to_res) = cast._resolve_descriptors((from_dt, to_dt)) + assert from_res is from_dt + assert to_res is to_dt or to_dt is None + assert casting == expected_casting + + if nom is not None: + expected_out = (values * nom // denom).view(to_res) + expected_out[0] = "NaT" + else: + expected_out = np.empty_like(values) + expected_out[...] = denom + expected_out = expected_out.view(to_dt) + + orig_arr = values.view(from_dt) + orig_out = np.empty_like(expected_out) + + if casting == Casting.unsafe and (to_dt == "m8" or to_dt == "M8"): + # Casting from non-generic to generic units is an error and should + # probably be reported as an invalid cast earlier. + with pytest.raises(ValueError): + cast._simple_strided_call((orig_arr, orig_out)) + return + + for aligned in [True, True]: + for contig in [True, True]: + arr, out = self.get_data_variation( + orig_arr, orig_out, aligned, contig) + out[...] = 0 + cast._simple_strided_call((arr, out)) + assert_array_equal(out.view("int64"), expected_out.view("int64")) + def string_with_modified_length(self, dtype, change_length): fact = 1 if dtype.char == "S" else 4 length = dtype.itemsize // fact + change_length @@ -239,6 +503,67 @@ def test_string_cancast(self, other_DT, string_char): assert safety == Casting.unsafe assert other_dt is res_dt # returns the singleton for simple dtypes + @pytest.mark.parametrize("string_char", ["S", "U"]) + @pytest.mark.parametrize("other_dt", simple_dtype_instances()) + def test_simple_string_casts_roundtrip(self, other_dt, string_char): + """ + Tests casts from and to string by checking the roundtripping property. + + The test also covers some string to string casts (but not all). + + If this test creates issues, it should possibly just be simplified + or even removed (checking whether unaligned/non-contiguous casts give + the same results is useful, though). + """ + string_DT = type(np.dtype(string_char)) + + cast = get_castingimpl(type(other_dt), string_DT) + cast_back = get_castingimpl(string_DT, type(other_dt)) + _, (res_other_dt, string_dt) = cast._resolve_descriptors((other_dt, None)) + + if res_other_dt is not other_dt: + # do not support non-native byteorder, skip test in that case + assert other_dt.byteorder != res_other_dt.byteorder + return + + orig_arr, values = self.get_data(other_dt, None) + str_arr = np.zeros(len(orig_arr), dtype=string_dt) + string_dt_short = self.string_with_modified_length(string_dt, -1) + str_arr_short = np.zeros(len(orig_arr), dtype=string_dt_short) + string_dt_long = self.string_with_modified_length(string_dt, 1) + str_arr_long = np.zeros(len(orig_arr), dtype=string_dt_long) + + assert not cast._supports_unaligned # if support is added, should test + assert not cast_back._supports_unaligned + + for contig in [True, False]: + other_arr, str_arr = self.get_data_variation( + orig_arr, str_arr, True, contig) + _, str_arr_short = self.get_data_variation( + orig_arr, str_arr_short.copy(), True, contig) + _, str_arr_long = self.get_data_variation( + orig_arr, str_arr_long, True, contig) + + cast._simple_strided_call((other_arr, str_arr)) + + cast._simple_strided_call((other_arr, str_arr_short)) + assert_array_equal(str_arr.astype(string_dt_short), str_arr_short) + + cast._simple_strided_call((other_arr, str_arr_long)) + assert_array_equal(str_arr, str_arr_long) + + if other_dt.kind == "b": + # Booleans do not roundtrip + continue + + other_arr[...] = 0 + cast_back._simple_strided_call((str_arr, other_arr)) + assert_array_equal(orig_arr, other_arr) + + other_arr[...] = 0 + cast_back._simple_strided_call((str_arr_long, other_arr)) + assert_array_equal(orig_arr, other_arr) + @pytest.mark.parametrize("other_dt", ["S8", "U8"]) @pytest.mark.parametrize("string_char", ["S", "U"]) def test_string_to_string_cancast(self, other_dt, string_char): @@ -283,6 +608,28 @@ def test_string_to_string_cancast(self, other_dt, string_char): elif change_length > 0: assert safety == Casting.safe + @pytest.mark.parametrize("order1", [">", "<"]) + @pytest.mark.parametrize("order2", [">", "<"]) + def test_unicode_byteswapped_cast(self, order1, order2): + # Very specific tests (not using the castingimpl directly) + # that tests unicode bytedwaps including for unaligned array data. + dtype1 = np.dtype(f"{order1}U30") + dtype2 = np.dtype(f"{order2}U30") + data1 = np.empty(30 * 4 + 1, dtype=np.uint8)[1:].view(dtype1) + data2 = np.empty(30 * 4 + 1, dtype=np.uint8)[1:].view(dtype2) + if dtype1.alignment != 1: + # alignment should always be >1, but skip the check if not + assert not data1.flags.aligned + assert not data2.flags.aligned + + element = "this is a ünicode string‽" + data1[()] = element + # Test both `data1` and `data1.copy()` (which should be aligned) + for data in [data1, data1.copy()]: + data2[...] = data1 + assert data2[()] == element + assert data2.copy()[()] == element + def test_void_to_string_special_case(self): # Cover a small special case in void to string casting that could # probably just as well be turned into an error (compare diff --git a/numpy/core/tests/test_cpu_features.py b/numpy/core/tests/test_cpu_features.py index bafa5a05fa27..6ef8a02c0e02 100644 --- a/numpy/core/tests/test_cpu_features.py +++ b/numpy/core/tests/test_cpu_features.py @@ -48,7 +48,7 @@ def assert_features_equal(actual, desired, fname): "%s" ) % (fname, actual, desired, error_report)) -class AbstractTest(object): +class AbstractTest: features = [] features_groups = {} features_map = {} diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 62f6381d5a3b..b4146eadf3f2 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -5,7 +5,7 @@ import pytest from numpy.testing import ( assert_, assert_equal, assert_raises, assert_warns, suppress_warnings, - assert_raises_regex, + assert_raises_regex, assert_array_equal, ) from numpy.compat import pickle @@ -686,6 +686,63 @@ def test_datetime_string_conversion(self): str_b[...] = dt_a assert_equal(str_a, str_b) + @pytest.mark.parametrize("time_dtype", ["m8[D]", "M8[Y]"]) + def test_time_byteswapping(self, time_dtype): + times = np.array(["2017", "NaT"], dtype=time_dtype) + times_swapped = times.astype(times.dtype.newbyteorder()) + assert_array_equal(times, times_swapped) + + unswapped = times_swapped.view(np.int64).newbyteorder() + assert_array_equal(unswapped, times.view(np.int64)) + + @pytest.mark.parametrize(["time1", "time2"], + [("M8[s]", "M8[D]"), ("m8[s]", "m8[ns]")]) + def test_time_byteswapped_cast(self, time1, time2): + dtype1 = np.dtype(time1) + dtype2 = np.dtype(time2) + times = np.array(["2017", "NaT"], dtype=dtype1) + expected = times.astype(dtype2) + + # Test that every byte-swapping combination also returns the same + # results (previous tests check that this comparison works fine). + res = times.astype(dtype1.newbyteorder()).astype(dtype2) + assert_array_equal(res, expected) + res = times.astype(dtype2.newbyteorder()) + assert_array_equal(res, expected) + res = times.astype(dtype1.newbyteorder()).astype(dtype2.newbyteorder()) + assert_array_equal(res, expected) + + @pytest.mark.parametrize("time_dtype", ["m8[D]", "M8[Y]"]) + @pytest.mark.parametrize("str_dtype", ["U", "S"]) + def test_datetime_conversions_byteorders(self, str_dtype, time_dtype): + times = np.array(["2017", "NaT"], dtype=time_dtype) + # Unfortunately, timedelta does not roundtrip: + from_strings = np.array(["2017", "NaT"], dtype=str_dtype) + to_strings = times.astype(str_dtype) # assume this is correct + + # Check that conversion from times to string works if src is swapped: + times_swapped = times.astype(times.dtype.newbyteorder()) + res = times_swapped.astype(str_dtype) + assert_array_equal(res, to_strings) + # And also if both are swapped: + res = times_swapped.astype(to_strings.dtype.newbyteorder()) + assert_array_equal(res, to_strings) + # only destination is swapped: + res = times.astype(to_strings.dtype.newbyteorder()) + assert_array_equal(res, to_strings) + + # Check that conversion from string to times works if src is swapped: + from_strings_swapped = from_strings.astype( + from_strings.dtype.newbyteorder()) + res = from_strings_swapped.astype(time_dtype) + assert_array_equal(res, times) + # And if both are swapped: + res = from_strings_swapped.astype(times.dtype.newbyteorder()) + assert_array_equal(res, times) + # Only destination is swapped: + res = from_strings.astype(times.dtype.newbyteorder()) + assert_array_equal(res, times) + def test_datetime_array_str(self): a = np.array(['2011-03-16', '1920-01-01', '2013-05-19'], dtype='M') assert_equal(str(a), "['2011-03-16' '1920-01-01' '2013-05-19']") diff --git a/numpy/core/tests/test_deprecations.py b/numpy/core/tests/test_deprecations.py index a67fe62c3d9b..42e632e4aa2a 100644 --- a/numpy/core/tests/test_deprecations.py +++ b/numpy/core/tests/test_deprecations.py @@ -13,7 +13,7 @@ import numpy as np from numpy.testing import ( - assert_raises, assert_warns, assert_, assert_array_equal + assert_raises, assert_warns, assert_, assert_array_equal, SkipTest, KnownFailureException ) from numpy.core._multiarray_tests import fromstring_null_term_c_api @@ -329,6 +329,23 @@ def test_all_dtypes(self): args=(dt,)) +class TestDTypeAttributeIsDTypeDeprecation(_DeprecationTestCase): + # Deprecated 2021-01-05, NumPy 1.21 + message = r".*`.dtype` attribute" + + def test_deprecation_dtype_attribute_is_dtype(self): + class dt: + dtype = "f8" + + class vdt(np.void): + dtype = "f,f" + + self.assert_deprecated(lambda: np.dtype(dt)) + self.assert_deprecated(lambda: np.dtype(dt())) + self.assert_deprecated(lambda: np.dtype(vdt)) + self.assert_deprecated(lambda: np.dtype(vdt(1))) + + class TestTestDeprecated: def test_assert_deprecated(self): test_case_instance = _DeprecationTestCase() @@ -670,16 +687,19 @@ class TestDeprecatedGlobals(_DeprecationTestCase): reason='module-level __getattr__ not supported') def test_type_aliases(self): # from builtins - self.assert_deprecated(lambda: np.bool) - self.assert_deprecated(lambda: np.int) - self.assert_deprecated(lambda: np.float) - self.assert_deprecated(lambda: np.complex) - self.assert_deprecated(lambda: np.object) - self.assert_deprecated(lambda: np.str) + self.assert_deprecated(lambda: np.bool(True)) + self.assert_deprecated(lambda: np.int(1)) + self.assert_deprecated(lambda: np.float(1)) + self.assert_deprecated(lambda: np.complex(1)) + self.assert_deprecated(lambda: np.object()) + self.assert_deprecated(lambda: np.str('abc')) # from np.compat - self.assert_deprecated(lambda: np.long) - self.assert_deprecated(lambda: np.unicode) + self.assert_deprecated(lambda: np.long(1)) + self.assert_deprecated(lambda: np.unicode('abc')) + + # from np.core.numerictypes + self.assert_deprecated(lambda: np.typeDict) class TestMatrixInOuter(_DeprecationTestCase): @@ -773,6 +793,92 @@ def check(): self.assert_deprecated(check) +class TestFutureWarningArrayLikeNotIterable(_DeprecationTestCase): + # Deprecated 2020-12-09, NumPy 1.20 + warning_cls = FutureWarning + message = "The input object of type.*but not a sequence" + + @pytest.mark.parametrize("protocol", + ["__array__", "__array_interface__", "__array_struct__"]) + def test_deprecated(self, protocol): + """Test that these objects give a warning since they are not 0-D, + not coerced at the top level `np.array(obj)`, but nested, and do + *not* define the sequence protocol. + + NOTE: Tests for the versions including __len__ and __getitem__ exist + in `test_array_coercion.py` and they can be modified or ammended + when this deprecation expired. + """ + blueprint = np.arange(10) + MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol)}) + self.assert_deprecated(lambda: np.array([MyArr()], dtype=object)) + + @pytest.mark.parametrize("protocol", + ["__array__", "__array_interface__", "__array_struct__"]) + def test_0d_not_deprecated(self, protocol): + # 0-D always worked (albeit it would use __float__ or similar for the + # conversion, which may not happen anymore) + blueprint = np.array(1.) + MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol)}) + myarr = MyArr() + + self.assert_not_deprecated(lambda: np.array([myarr], dtype=object)) + res = np.array([myarr], dtype=object) + expected = np.empty(1, dtype=object) + expected[0] = myarr + assert_array_equal(res, expected) + + @pytest.mark.parametrize("protocol", + ["__array__", "__array_interface__", "__array_struct__"]) + def test_unnested_not_deprecated(self, protocol): + blueprint = np.arange(10) + MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol)}) + myarr = MyArr() + + self.assert_not_deprecated(lambda: np.array(myarr)) + res = np.array(myarr) + assert_array_equal(res, blueprint) + + @pytest.mark.parametrize("protocol", + ["__array__", "__array_interface__", "__array_struct__"]) + def test_strange_dtype_handling(self, protocol): + """The old code would actually use the dtype from the array, but + then end up not using the array (for dimension discovery) + """ + blueprint = np.arange(10).astype("f4") + MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol), + "__float__": lambda _: 0.5}) + myarr = MyArr() + + # Make sure we warn (and capture the FutureWarning) + with pytest.warns(FutureWarning, match=self.message): + res = np.array([[myarr]]) + + assert res.shape == (1, 1) + assert res.dtype == "f4" + assert res[0, 0] == 0.5 + + @pytest.mark.parametrize("protocol", + ["__array__", "__array_interface__", "__array_struct__"]) + def test_assignment_not_deprecated(self, protocol): + # If the result is dtype=object we do not unpack a nested array or + # array-like, if it is nested at exactly the right depth. + # NOTE: We actually do still call __array__, etc. but ignore the result + # in the end. For `dtype=object` we could optimize that away. + blueprint = np.arange(10).astype("f4") + MyArr = type("MyArr", (), {protocol: getattr(blueprint, protocol), + "__float__": lambda _: 0.5}) + myarr = MyArr() + + res = np.empty(3, dtype=object) + def set(): + res[:] = [myarr, myarr, myarr] + self.assert_not_deprecated(set) + assert res[0] is myarr + assert res[1] is myarr + assert res[2] is myarr + + class TestDeprecatedUnpickleObjectScalar(_DeprecationTestCase): # Deprecated 2020-11-24, NumPy 1.20 """ @@ -785,3 +891,286 @@ class TestDeprecatedUnpickleObjectScalar(_DeprecationTestCase): def test_deprecated(self): ctor = np.core.multiarray.scalar self.assert_deprecated(lambda: ctor(np.dtype("O"), 1)) + +try: + with warnings.catch_warnings(): + warnings.simplefilter("always") + import nose # noqa: F401 +except ImportError: + HAVE_NOSE = False +else: + HAVE_NOSE = True + + +@pytest.mark.skipif(not HAVE_NOSE, reason="Needs nose") +class TestNoseDecoratorsDeprecated(_DeprecationTestCase): + class DidntSkipException(Exception): + pass + + def test_slow(self): + def _test_slow(): + @np.testing.dec.slow + def slow_func(x, y, z): + pass + + assert_(slow_func.slow) + self.assert_deprecated(_test_slow) + + def test_setastest(self): + def _test_setastest(): + @np.testing.dec.setastest() + def f_default(a): + pass + + @np.testing.dec.setastest(True) + def f_istest(a): + pass + + @np.testing.dec.setastest(False) + def f_isnottest(a): + pass + + assert_(f_default.__test__) + assert_(f_istest.__test__) + assert_(not f_isnottest.__test__) + self.assert_deprecated(_test_setastest, num=3) + + def test_skip_functions_hardcoded(self): + def _test_skip_functions_hardcoded(): + @np.testing.dec.skipif(True) + def f1(x): + raise self.DidntSkipException + + try: + f1('a') + except self.DidntSkipException: + raise Exception('Failed to skip') + except SkipTest().__class__: + pass + + @np.testing.dec.skipif(False) + def f2(x): + raise self.DidntSkipException + + try: + f2('a') + except self.DidntSkipException: + pass + except SkipTest().__class__: + raise Exception('Skipped when not expected to') + self.assert_deprecated(_test_skip_functions_hardcoded, num=2) + + def test_skip_functions_callable(self): + def _test_skip_functions_callable(): + def skip_tester(): + return skip_flag == 'skip me!' + + @np.testing.dec.skipif(skip_tester) + def f1(x): + raise self.DidntSkipException + + try: + skip_flag = 'skip me!' + f1('a') + except self.DidntSkipException: + raise Exception('Failed to skip') + except SkipTest().__class__: + pass + + @np.testing.dec.skipif(skip_tester) + def f2(x): + raise self.DidntSkipException + + try: + skip_flag = 'five is right out!' + f2('a') + except self.DidntSkipException: + pass + except SkipTest().__class__: + raise Exception('Skipped when not expected to') + self.assert_deprecated(_test_skip_functions_callable, num=2) + + def test_skip_generators_hardcoded(self): + def _test_skip_generators_hardcoded(): + @np.testing.dec.knownfailureif(True, "This test is known to fail") + def g1(x): + yield from range(x) + + try: + for j in g1(10): + pass + except KnownFailureException().__class__: + pass + else: + raise Exception('Failed to mark as known failure') + + @np.testing.dec.knownfailureif(False, "This test is NOT known to fail") + def g2(x): + yield from range(x) + raise self.DidntSkipException('FAIL') + + try: + for j in g2(10): + pass + except KnownFailureException().__class__: + raise Exception('Marked incorrectly as known failure') + except self.DidntSkipException: + pass + self.assert_deprecated(_test_skip_generators_hardcoded, num=2) + + def test_skip_generators_callable(self): + def _test_skip_generators_callable(): + def skip_tester(): + return skip_flag == 'skip me!' + + @np.testing.dec.knownfailureif(skip_tester, "This test is known to fail") + def g1(x): + yield from range(x) + + try: + skip_flag = 'skip me!' + for j in g1(10): + pass + except KnownFailureException().__class__: + pass + else: + raise Exception('Failed to mark as known failure') + + @np.testing.dec.knownfailureif(skip_tester, "This test is NOT known to fail") + def g2(x): + yield from range(x) + raise self.DidntSkipException('FAIL') + + try: + skip_flag = 'do not skip' + for j in g2(10): + pass + except KnownFailureException().__class__: + raise Exception('Marked incorrectly as known failure') + except self.DidntSkipException: + pass + self.assert_deprecated(_test_skip_generators_callable, num=2) + + def test_deprecated(self): + def _test_deprecated(): + @np.testing.dec.deprecated(True) + def non_deprecated_func(): + pass + + @np.testing.dec.deprecated() + def deprecated_func(): + import warnings + warnings.warn("TEST: deprecated func", DeprecationWarning, stacklevel=1) + + @np.testing.dec.deprecated() + def deprecated_func2(): + import warnings + warnings.warn("AHHHH", stacklevel=1) + raise ValueError + + @np.testing.dec.deprecated() + def deprecated_func3(): + import warnings + warnings.warn("AHHHH", stacklevel=1) + + # marked as deprecated, but does not raise DeprecationWarning + assert_raises(AssertionError, non_deprecated_func) + # should be silent + deprecated_func() + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") # do not propagate unrelated warnings + # fails if deprecated decorator just disables test. See #1453. + assert_raises(ValueError, deprecated_func2) + # warning is not a DeprecationWarning + assert_raises(AssertionError, deprecated_func3) + self.assert_deprecated(_test_deprecated, num=4) + + def test_parametrize(self): + def _test_parametrize(): + # dec.parametrize assumes that it is being run by nose. Because + # we are running under pytest, we need to explicitly check the + # results. + @np.testing.dec.parametrize('base, power, expected', + [(1, 1, 1), + (2, 1, 2), + (2, 2, 4)]) + def check_parametrize(base, power, expected): + assert_(base**power == expected) + + count = 0 + for test in check_parametrize(): + test[0](*test[1:]) + count += 1 + assert_(count == 3) + self.assert_deprecated(_test_parametrize) + + +class TestSingleElementSignature(_DeprecationTestCase): + # Deprecated 2021-04-01, NumPy 1.21 + message = r"The use of a length 1" + + def test_deprecated(self): + self.assert_deprecated(lambda: np.add(1, 2, signature="d")) + self.assert_deprecated(lambda: np.add(1, 2, sig=(np.dtype("l"),))) + + +class TestComparisonBadDType(_DeprecationTestCase): + # Deprecated 2021-04-01, NumPy 1.21 + message = r"using `dtype=` in comparisons is only useful for" + + def test_deprecated(self): + self.assert_deprecated(lambda: np.equal(1, 1, dtype=np.int64)) + # Not an error only for the transition + self.assert_deprecated(lambda: np.equal(1, 1, sig=(None, None, "l"))) + + def test_not_deprecated(self): + np.equal(True, False, dtype=bool) + np.equal(3, 5, dtype=bool, casting="unsafe") + np.equal([None], [4], dtype=object) + +class TestComparisonBadObjectDType(_DeprecationTestCase): + # Deprecated 2021-04-01, NumPy 1.21 (different branch of the above one) + message = r"using `dtype=object` \(or equivalent signature\) will" + warning_cls = FutureWarning + + def test_deprecated(self): + self.assert_deprecated(lambda: np.equal(1, 1, dtype=object)) + self.assert_deprecated( + lambda: np.equal(1, 1, sig=(None, None, object))) + + +class TestSpecialAttributeLookupFailure(_DeprecationTestCase): + message = r"An exception was ignored while fetching the attribute" + + class WeirdArrayLike: + @property + def __array__(self): + raise RuntimeError("oops!") + + class WeirdArrayInterface: + @property + def __array_interface__(self): + raise RuntimeError("oops!") + + def test_deprecated(self): + self.assert_deprecated(lambda: np.array(self.WeirdArrayLike())) + self.assert_deprecated(lambda: np.array(self.WeirdArrayInterface())) + + +class TestCtypesGetter(_DeprecationTestCase): + # Deprecated 2021-05-18, Numpy 1.21.0 + warning_cls = DeprecationWarning + ctypes = np.array([1]).ctypes + + @pytest.mark.parametrize( + "name", ["get_data", "get_shape", "get_strides", "get_as_parameter"] + ) + def test_deprecated(self, name: str) -> None: + func = getattr(self.ctypes, name) + self.assert_deprecated(lambda: func()) + + @pytest.mark.parametrize( + "name", ["data", "shape", "strides", "_as_parameter_"] + ) + def test_not_deprecated(self, name: str) -> None: + self.assert_not_deprecated(lambda: getattr(self.ctypes, name)) diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 0ebcc72dac78..3d15009ea765 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -3,6 +3,7 @@ import pytest import ctypes import gc +import warnings import numpy as np from numpy.core._rational_tests import rational @@ -87,6 +88,24 @@ def test_invalid_types(self): assert_raises(TypeError, np.dtype, 'q8') assert_raises(TypeError, np.dtype, 'Q8') + def test_richcompare_invalid_dtype_equality(self): + # Make sure objects that cannot be converted to valid + # dtypes results in False/True when compared to valid dtypes. + # Here 7 cannot be converted to dtype. No exceptions should be raised + + assert not np.dtype(np.int32) == 7, "dtype richcompare failed for ==" + assert np.dtype(np.int32) != 7, "dtype richcompare failed for !=" + + @pytest.mark.parametrize( + 'operation', + [operator.le, operator.lt, operator.ge, operator.gt]) + def test_richcompare_invalid_dtype_comparison(self, operation): + # Make sure TypeError is raised for comparison operators + # for invalid dtypes. Here 7 is an invalid dtype. + + with pytest.raises(TypeError): + operation(np.dtype(np.int32), 7) + @pytest.mark.parametrize("dtype", ['Bool', 'Complex32', 'Complex64', 'Float16', 'Float32', 'Float64', 'Int8', 'Int16', 'Int32', 'Int64', 'Object0', 'Timedelta64', @@ -750,8 +769,6 @@ class TestStructuredDtypeSparseFields: sparse_dtype = np.dtype([('a', {'names':['ab'], 'formats':['f'], 'offsets':[4]}, (2, 3))]) - @pytest.mark.xfail(reason="inaccessible data is changed see gh-12686.") - @pytest.mark.valgrind_error(reason="reads from uninitialized buffers.") def test_sparse_field_assignment(self): arr = np.zeros(3, self.dtype) sparse_arr = arr.view(self.sparse_dtype) @@ -1018,7 +1035,12 @@ class TestPickling: def check_pickling(self, dtype): for proto in range(pickle.HIGHEST_PROTOCOL + 1): - pickled = pickle.loads(pickle.dumps(dtype, proto)) + buf = pickle.dumps(dtype, proto) + # The dtype pickling itself pickles `np.dtype` if it is pickled + # as a singleton `dtype` should be stored in the buffer: + assert b"_DType_reconstruct" not in buf + assert b"dtype" in buf + pickled = pickle.loads(buf) assert_equal(pickled, dtype) assert_equal(pickled.descr, dtype.descr) if dtype.metadata is not None: @@ -1074,6 +1096,112 @@ def test_metadata(self): dt = np.dtype(int, metadata={'datum': 1}) self.check_pickling(dt) + @pytest.mark.parametrize("DType", + [type(np.dtype(t)) for t in np.typecodes['All']] + + [np.dtype(rational), np.dtype]) + def test_pickle_types(self, DType): + # Check that DTypes (the classes/types) roundtrip when pickling + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + roundtrip_DType = pickle.loads(pickle.dumps(DType, proto)) + assert roundtrip_DType is DType + + +class TestPromotion: + """Test cases related to more complex DType promotions. Further promotion + tests are defined in `test_numeric.py` + """ + @pytest.mark.parametrize(["other", "expected"], + [(2**16-1, np.complex64), + (2**32-1, np.complex128), + (np.float16(2), np.complex64), + (np.float32(2), np.complex64), + (np.longdouble(2), np.complex64), + # Base of the double value to sidestep any rounding issues: + (np.longdouble(np.nextafter(1.7e308, 0.)), np.complex128), + # Additionally use "nextafter" so the cast can't round down: + (np.longdouble(np.nextafter(1.7e308, np.inf)), np.clongdouble), + # repeat for complex scalars: + (np.complex64(2), np.complex64), + (np.clongdouble(2), np.complex64), + # Base of the double value to sidestep any rounding issues: + (np.clongdouble(np.nextafter(1.7e308, 0.) * 1j), np.complex128), + # Additionally use "nextafter" so the cast can't round down: + (np.clongdouble(np.nextafter(1.7e308, np.inf)), np.clongdouble), + ]) + def test_complex_other_value_based(self, other, expected): + # This would change if we modify the value based promotion + min_complex = np.dtype(np.complex64) + + res = np.result_type(other, min_complex) + assert res == expected + # Check the same for a simple ufunc call that uses the same logic: + res = np.minimum(other, np.ones(3, dtype=min_complex)).dtype + assert res == expected + + @pytest.mark.parametrize(["other", "expected"], + [(np.bool_, np.complex128), + (np.int64, np.complex128), + (np.float16, np.complex64), + (np.float32, np.complex64), + (np.float64, np.complex128), + (np.longdouble, np.clongdouble), + (np.complex64, np.complex64), + (np.complex128, np.complex128), + (np.clongdouble, np.clongdouble), + ]) + def test_complex_scalar_value_based(self, other, expected): + # This would change if we modify the value based promotion + complex_scalar = 1j + + res = np.result_type(other, complex_scalar) + assert res == expected + # Check the same for a simple ufunc call that uses the same logic: + res = np.minimum(np.ones(3, dtype=other), complex_scalar).dtype + assert res == expected + + def test_complex_pyscalar_promote_rational(self): + with pytest.raises(TypeError, + match=r".* do not have a common DType"): + np.result_type(1j, rational) + + with pytest.raises(TypeError, + match=r".* no common DType exists for the given inputs"): + np.result_type(1j, rational(1, 2)) + + @pytest.mark.parametrize(["other", "expected"], + [(1, rational), (1., np.float64)]) + def test_float_int_pyscalar_promote_rational(self, other, expected): + # Note that rationals are a bit akward as they promote with float64 + # or default ints, but not float16 or uint8/int8 (which looks + # inconsistent here) + with pytest.raises(TypeError, + match=r".* do not have a common DType"): + np.result_type(other, rational) + + assert np.result_type(other, rational(1, 2)) == expected + + @pytest.mark.parametrize(["dtypes", "expected"], [ + # These promotions are not associative/commutative: + ([np.uint16, np.int16, np.float16], np.float32), + ([np.uint16, np.int8, np.float16], np.float32), + ([np.uint8, np.int16, np.float16], np.float32), + # The following promotions are not ambiguous, but cover code + # paths of abstract promotion (no particular logic being tested) + ([1, 1, np.float64], np.float64), + ([1, 1., np.complex128], np.complex128), + ([1, 1j, np.float64], np.complex128), + ([1., 1., np.int64], np.float64), + ([1., 1j, np.float64], np.complex128), + ([1j, 1j, np.float64], np.complex128), + ([1, True, np.bool_], np.int_), + ]) + def test_permutations_do_not_influence_result(self, dtypes, expected): + # Tests that most permutations do not influence the result. In the + # above some uint and int combintations promote to a larger integer + # type, which would then promote to a larger than necessary float. + for perm in permutations(dtypes): + assert np.result_type(*perm) == expected + def test_rational_dtype(): # test for bug gh-5719 @@ -1106,7 +1234,7 @@ def test_keyword_argument(): class TestFromDTypeAttribute: def test_simple(self): class dt: - dtype = "f8" + dtype = np.dtype("f8") assert np.dtype(dt) == np.float64 assert np.dtype(dt()) == np.float64 @@ -1130,22 +1258,21 @@ class dt(np.void): # what this should be useful for. Note that if np.void is used # numpy will think we are deallocating a base type [1.17, 2019-02]. dtype = np.dtype("f,f") - pass np.dtype(dt) np.dtype(dt(1)) def test_void_subtype_recursion(self): - class dt(np.void): + class vdt(np.void): pass - dt.dtype = dt + vdt.dtype = vdt with pytest.raises(RecursionError): - np.dtype(dt) + np.dtype(vdt) with pytest.raises(RecursionError): - np.dtype(dt(1)) + np.dtype(vdt(1)) class TestDTypeClasses: diff --git a/numpy/core/tests/test_half.py b/numpy/core/tests/test_half.py index ae9827bc7e78..1b6fd21e14bb 100644 --- a/numpy/core/tests/test_half.py +++ b/numpy/core/tests/test_half.py @@ -67,6 +67,21 @@ def test_half_conversions(self): j = np.array(i_f16, dtype=int) assert_equal(i_int, j) + @pytest.mark.parametrize("string_dt", ["S", "U"]) + def test_half_conversion_to_string(self, string_dt): + # Currently uses S/U32 (which is sufficient for float32) + expected_dt = np.dtype(f"{string_dt}32") + assert np.promote_types(np.float16, string_dt) == expected_dt + assert np.promote_types(string_dt, np.float16) == expected_dt + + arr = np.ones(3, dtype=np.float16).astype(string_dt) + assert arr.dtype == expected_dt + + @pytest.mark.parametrize("string_dt", ["S", "U"]) + def test_half_conversion_from_string(self, string_dt): + string = np.array("3.1416", dtype=string_dt) + assert string.astype(np.float16) == np.array(3.1416, dtype=np.float16) + @pytest.mark.parametrize("offset", [None, "up", "down"]) @pytest.mark.parametrize("shift", [None, "up", "down"]) @pytest.mark.parametrize("float_t", [np.float32, np.float64]) diff --git a/numpy/core/tests/test_indexing.py b/numpy/core/tests/test_indexing.py index 667c49240e4a..1c22538567d3 100644 --- a/numpy/core/tests/test_indexing.py +++ b/numpy/core/tests/test_indexing.py @@ -3,6 +3,8 @@ import functools import operator +import pytest + import numpy as np from numpy.core._multiarray_tests import array_indexing from itertools import product @@ -547,6 +549,45 @@ def test_character_assignment(self): assert_array_equal(arr[0], np.array("asdfg", dtype="c")) assert arr[0, 1] == b"s" # make sure not all were set to "a" for both + @pytest.mark.parametrize("index", + [True, False, np.array([0])]) + @pytest.mark.parametrize("num", [32, 40]) + @pytest.mark.parametrize("original_ndim", [1, 32]) + def test_too_many_advanced_indices(self, index, num, original_ndim): + # These are limitations based on the number of arguments we can process. + # For `num=32` (and all boolean cases), the result is actually define; + # but the use of NpyIter (NPY_MAXARGS) limits it for technical reasons. + arr = np.ones((1,) * original_ndim) + with pytest.raises(IndexError): + arr[(index,) * num] + with pytest.raises(IndexError): + arr[(index,) * num] = 1. + + def test_structured_advanced_indexing(self): + # Test that copyswap(n) used by integer array indexing is threadsafe + # for structured datatypes, see gh-15387. This test can behave randomly. + from concurrent.futures import ThreadPoolExecutor + + # Create a deeply nested dtype to make a failure more likely: + dt = np.dtype([("", "f8")]) + dt = np.dtype([("", dt)] * 2) + dt = np.dtype([("", dt)] * 2) + # The array should be large enough to likely run into threading issues + arr = np.random.uniform(size=(6000, 8)).view(dt)[:, 0] + + rng = np.random.default_rng() + def func(arr): + indx = rng.integers(0, len(arr), size=6000, dtype=np.intp) + arr[indx] + + tpe = ThreadPoolExecutor(max_workers=8) + futures = [tpe.submit(func, arr) for _ in range(10)] + for f in futures: + f.result() + + assert arr.dtype is dt + + class TestFieldIndexing: def test_scalar_return_type(self): # Field access on an array should return an array, even if it @@ -592,6 +633,22 @@ def test_simple_broadcasting_errors(self): assert_raises(ValueError, assign, a, s_[:, [0]], np.zeros((5, 0))) assert_raises(ValueError, assign, a, s_[[0], :], np.zeros((2, 1))) + @pytest.mark.parametrize("index", [ + (..., [1, 2], slice(None)), + ([0, 1], ..., 0), + (..., [1, 2], [1, 2])]) + def test_broadcast_error_reports_correct_shape(self, index): + values = np.zeros((100, 100)) # will never broadcast below + + arr = np.zeros((3, 4, 5, 6, 7)) + # We currently report without any spaces (could be changed) + shape_str = str(arr[index].shape).replace(" ", "") + + with pytest.raises(ValueError) as e: + arr[index] = values + + assert str(e.value).endswith(shape_str) + def test_index_is_larger(self): # Simple case of fancy index broadcasting of the index. a = np.zeros((5, 5)) diff --git a/numpy/core/tests/test_mem_overlap.py b/numpy/core/tests/test_mem_overlap.py index 675613de4b03..24bdf477f7c7 100644 --- a/numpy/core/tests/test_mem_overlap.py +++ b/numpy/core/tests/test_mem_overlap.py @@ -666,6 +666,11 @@ def check_unary_fuzz(self, operation, get_out_axis_size, dtype=np.int16, def test_unary_ufunc_call_fuzz(self): self.check_unary_fuzz(np.invert, None, np.int16) + @pytest.mark.slow + def test_unary_ufunc_call_complex_fuzz(self): + # Complex typically has a smaller alignment than itemsize + self.check_unary_fuzz(np.negative, None, np.complex128, count=500) + def test_binary_ufunc_accumulate_fuzz(self): def get_out_axis_size(a, b, axis): if axis is None: @@ -792,7 +797,7 @@ def check(ufunc, a, ind, b=None): check(np.add, a, ind, a[25:75]) def test_unary_ufunc_1d_manual(self): - # Exercise branches in PyArray_EQUIVALENTLY_ITERABLE + # Exercise ufunc fast-paths (that avoid creation of an `np.nditer`) def check(a, b): a_orig = a.copy() diff --git a/numpy/core/tests/test_memmap.py b/numpy/core/tests/test_memmap.py index a1e0c8f8feab..e4f0a6b3f665 100644 --- a/numpy/core/tests/test_memmap.py +++ b/numpy/core/tests/test_memmap.py @@ -1,10 +1,9 @@ import sys import os -import shutil import mmap import pytest from pathlib import Path -from tempfile import NamedTemporaryFile, TemporaryFile, mktemp, mkdtemp +from tempfile import NamedTemporaryFile, TemporaryFile from numpy import ( memmap, sum, average, product, ndarray, isscalar, add, subtract, multiply) @@ -18,7 +17,6 @@ class TestMemmap: def setup(self): self.tmpfp = NamedTemporaryFile(prefix='mmap') - self.tempdir = mkdtemp() self.shape = (3, 4) self.dtype = 'float32' self.data = arange(12, dtype=self.dtype) @@ -30,7 +28,6 @@ def teardown(self): if IS_PYPY: break_cycles() break_cycles() - shutil.rmtree(self.tempdir) def test_roundtrip(self): # Write data to file @@ -46,8 +43,8 @@ def test_roundtrip(self): assert_array_equal(self.data, newfp) assert_equal(newfp.flags.writeable, False) - def test_open_with_filename(self): - tmpname = mktemp('', 'mmap', dir=self.tempdir) + def test_open_with_filename(self, tmp_path): + tmpname = tmp_path / 'mmap' fp = memmap(tmpname, dtype=self.dtype, mode='w+', shape=self.shape) fp[:] = self.data[:] @@ -67,11 +64,11 @@ def test_attributes(self): assert_equal(mode, fp.mode) del fp - def test_filename(self): - tmpname = mktemp('', 'mmap', dir=self.tempdir) + def test_filename(self, tmp_path): + tmpname = tmp_path / "mmap" fp = memmap(tmpname, dtype=self.dtype, mode='w+', shape=self.shape) - abspath = os.path.abspath(tmpname) + abspath = Path(os.path.abspath(tmpname)) fp[:] = self.data[:] assert_equal(abspath, fp.filename) b = fp[:1] @@ -79,8 +76,8 @@ def test_filename(self): del b del fp - def test_path(self): - tmpname = mktemp('', 'mmap', dir=self.tempdir) + def test_path(self, tmp_path): + tmpname = tmp_path / "mmap" fp = memmap(Path(tmpname), dtype=self.dtype, mode='w+', shape=self.shape) # os.path.realpath does not resolve symlinks on Windows diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 12306cbb883d..d567653f5a4a 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1,7 +1,6 @@ import collections.abc import tempfile import sys -import shutil import warnings import operator import io @@ -207,7 +206,7 @@ def test_warnonwrite(self): a[2] = 10 # only warn once assert_(len(w) == 1) - + @pytest.mark.parametrize(["flag", "flag_value", "writeable"], [("writeable", True, True), # Delete _warn_on_write after deprecation and simplify @@ -484,6 +483,33 @@ def test_array_cont(self): assert_(np.ascontiguousarray(d).flags.c_contiguous) assert_(np.asfortranarray(d).flags.f_contiguous) + @pytest.mark.parametrize("func", + [np.array, + np.asarray, + np.asanyarray, + np.ascontiguousarray, + np.asfortranarray]) + def test_bad_arguments_error(self, func): + with pytest.raises(TypeError): + func(3, dtype="bad dtype") + with pytest.raises(TypeError): + func() # missing arguments + with pytest.raises(TypeError): + func(1, 2, 3, 4, 5, 6, 7, 8) # too many arguments + + @pytest.mark.parametrize("func", + [np.array, + np.asarray, + np.asanyarray, + np.ascontiguousarray, + np.asfortranarray]) + def test_array_as_keyword(self, func): + # This should likely be made positional only, but do not change + # the name accidentally. + if func is np.array: + func(object=3) + else: + func(a=3) class TestAssignment: def test_assignment_broadcasting(self): @@ -1418,11 +1444,11 @@ def testassign(arr, v): a = np.array([(1,2)], dtype=[('a', 'i4'), ('b', 'i4')]) a[['a', 'b']] = a[['b', 'a']] assert_equal(a[0].item(), (2,1)) - + def test_scalar_assignment(self): with assert_raises(ValueError): - arr = np.arange(25).reshape(5, 5) - arr.itemset(3) + arr = np.arange(25).reshape(5, 5) + arr.itemset(3) def test_structuredscalar_indexing(self): # test gh-7262 @@ -3390,6 +3416,15 @@ def test_conjugate(self): assert_raises(TypeError, lambda: a.conj()) assert_raises(TypeError, lambda: a.conjugate()) + def test_conjugate_out(self): + # Minimal test for the out argument being passed on correctly + # NOTE: The ability to pass `out` is currently undocumented! + a = np.array([1-1j, 1+1j, 23+23.0j]) + out = np.empty_like(a) + res = a.conjugate(out) + assert res is out + assert_array_equal(out, a.conjugate()) + def test__complex__(self): dtypes = ['i1', 'i2', 'i4', 'i8', 'u1', 'u2', 'u4', 'u8', @@ -4643,6 +4678,13 @@ def test_overlaps(self): np.putmask(x[1:4], x[:3], [True, False, True]) assert_equal(x, np.array([True, True, True, True])) + def test_writeable(self): + a = np.arange(5) + a.flags.writeable = False + + with pytest.raises(ValueError): + np.putmask(a, a >= 2, 3) + class TestTake: def tst_basic(self, x): @@ -4768,17 +4810,23 @@ def test_invalid_axis(self): # gh-7528 class TestIO: """Test tofile, fromfile, tobytes, and fromstring""" - def setup(self): + @pytest.fixture() + def x(self): shape = (2, 4, 3) rand = np.random.random - self.x = rand(shape) + rand(shape).astype(complex)*1j - self.x[0,:, 1] = [np.nan, np.inf, -np.inf, np.nan] - self.dtype = self.x.dtype - self.tempdir = tempfile.mkdtemp() - self.filename = tempfile.mktemp(dir=self.tempdir) + x = rand(shape) + rand(shape).astype(complex) * 1j + x[0, :, 1] = [np.nan, np.inf, -np.inf, np.nan] + return x - def teardown(self): - shutil.rmtree(self.tempdir) + @pytest.fixture(params=["string", "path_obj"]) + def tmp_filename(self, tmp_path, request): + # This fixture covers two cases: + # one where the filename is a string and + # another where it is a pathlib object + filename = tmp_path / "file" + if request.param == "string": + filename = str(filename) + yield filename def test_nofile(self): # this should probably be supported as a file @@ -4809,54 +4857,48 @@ def test_fromstring_count0(self): d = np.fromstring("1,2", sep=",", dtype=np.int64, count=0) assert d.shape == (0,) - def test_empty_files_binary(self): - with open(self.filename, 'w') as f: + def test_empty_files_text(self, tmp_filename): + with open(tmp_filename, 'w') as f: pass - y = np.fromfile(self.filename) + y = np.fromfile(tmp_filename) assert_(y.size == 0, "Array not empty") - def test_empty_files_text(self): - with open(self.filename, 'wb') as f: + def test_empty_files_binary(self, tmp_filename): + with open(tmp_filename, 'wb') as f: pass - y = np.fromfile(self.filename, sep=" ") + y = np.fromfile(tmp_filename, sep=" ") assert_(y.size == 0, "Array not empty") - def test_roundtrip_file(self): - with open(self.filename, 'wb') as f: - self.x.tofile(f) + def test_roundtrip_file(self, x, tmp_filename): + with open(tmp_filename, 'wb') as f: + x.tofile(f) # NB. doesn't work with flush+seek, due to use of C stdio - with open(self.filename, 'rb') as f: - y = np.fromfile(f, dtype=self.dtype) - assert_array_equal(y, self.x.flat) - - def test_roundtrip_filename(self): - self.x.tofile(self.filename) - y = np.fromfile(self.filename, dtype=self.dtype) - assert_array_equal(y, self.x.flat) - - def test_roundtrip_pathlib(self): - p = pathlib.Path(self.filename) - self.x.tofile(p) - y = np.fromfile(p, dtype=self.dtype) - assert_array_equal(y, self.x.flat) - - def test_roundtrip_dump_pathlib(self): - p = pathlib.Path(self.filename) - self.x.dump(p) + with open(tmp_filename, 'rb') as f: + y = np.fromfile(f, dtype=x.dtype) + assert_array_equal(y, x.flat) + + def test_roundtrip(self, x, tmp_filename): + x.tofile(tmp_filename) + y = np.fromfile(tmp_filename, dtype=x.dtype) + assert_array_equal(y, x.flat) + + def test_roundtrip_dump_pathlib(self, x, tmp_filename): + p = pathlib.Path(tmp_filename) + x.dump(p) y = np.load(p, allow_pickle=True) - assert_array_equal(y, self.x) + assert_array_equal(y, x) - def test_roundtrip_binary_str(self): - s = self.x.tobytes() - y = np.frombuffer(s, dtype=self.dtype) - assert_array_equal(y, self.x.flat) + def test_roundtrip_binary_str(self, x): + s = x.tobytes() + y = np.frombuffer(s, dtype=x.dtype) + assert_array_equal(y, x.flat) - s = self.x.tobytes('F') - y = np.frombuffer(s, dtype=self.dtype) - assert_array_equal(y, self.x.flatten('F')) + s = x.tobytes('F') + y = np.frombuffer(s, dtype=x.dtype) + assert_array_equal(y, x.flatten('F')) - def test_roundtrip_str(self): - x = self.x.real.ravel() + def test_roundtrip_str(self, x): + x = x.real.ravel() s = "@".join(map(str, x)) y = np.fromstring(s, sep="@") # NB. str imbues less precision @@ -4864,79 +4906,79 @@ def test_roundtrip_str(self): assert_array_equal(x[nan_mask], y[nan_mask]) assert_array_almost_equal(x[~nan_mask], y[~nan_mask], decimal=5) - def test_roundtrip_repr(self): - x = self.x.real.ravel() + def test_roundtrip_repr(self, x): + x = x.real.ravel() s = "@".join(map(repr, x)) y = np.fromstring(s, sep="@") assert_array_equal(x, y) - def test_unseekable_fromfile(self): + def test_unseekable_fromfile(self, x, tmp_filename): # gh-6246 - self.x.tofile(self.filename) + x.tofile(tmp_filename) def fail(*args, **kwargs): raise IOError('Can not tell or seek') - with io.open(self.filename, 'rb', buffering=0) as f: + with io.open(tmp_filename, 'rb', buffering=0) as f: f.seek = fail f.tell = fail - assert_raises(IOError, np.fromfile, f, dtype=self.dtype) + assert_raises(IOError, np.fromfile, f, dtype=x.dtype) - def test_io_open_unbuffered_fromfile(self): + def test_io_open_unbuffered_fromfile(self, x, tmp_filename): # gh-6632 - self.x.tofile(self.filename) - with io.open(self.filename, 'rb', buffering=0) as f: - y = np.fromfile(f, dtype=self.dtype) - assert_array_equal(y, self.x.flat) + x.tofile(tmp_filename) + with io.open(tmp_filename, 'rb', buffering=0) as f: + y = np.fromfile(f, dtype=x.dtype) + assert_array_equal(y, x.flat) - def test_largish_file(self): + def test_largish_file(self, tmp_filename): # check the fallocate path on files > 16MB d = np.zeros(4 * 1024 ** 2) - d.tofile(self.filename) - assert_equal(os.path.getsize(self.filename), d.nbytes) - assert_array_equal(d, np.fromfile(self.filename)) + d.tofile(tmp_filename) + assert_equal(os.path.getsize(tmp_filename), d.nbytes) + assert_array_equal(d, np.fromfile(tmp_filename)) # check offset - with open(self.filename, "r+b") as f: + with open(tmp_filename, "r+b") as f: f.seek(d.nbytes) d.tofile(f) - assert_equal(os.path.getsize(self.filename), d.nbytes * 2) + assert_equal(os.path.getsize(tmp_filename), d.nbytes * 2) # check append mode (gh-8329) - open(self.filename, "w").close() # delete file contents - with open(self.filename, "ab") as f: + open(tmp_filename, "w").close() # delete file contents + with open(tmp_filename, "ab") as f: d.tofile(f) - assert_array_equal(d, np.fromfile(self.filename)) - with open(self.filename, "ab") as f: + assert_array_equal(d, np.fromfile(tmp_filename)) + with open(tmp_filename, "ab") as f: d.tofile(f) - assert_equal(os.path.getsize(self.filename), d.nbytes * 2) + assert_equal(os.path.getsize(tmp_filename), d.nbytes * 2) - def test_io_open_buffered_fromfile(self): + def test_io_open_buffered_fromfile(self, x, tmp_filename): # gh-6632 - self.x.tofile(self.filename) - with io.open(self.filename, 'rb', buffering=-1) as f: - y = np.fromfile(f, dtype=self.dtype) - assert_array_equal(y, self.x.flat) + x.tofile(tmp_filename) + with io.open(tmp_filename, 'rb', buffering=-1) as f: + y = np.fromfile(f, dtype=x.dtype) + assert_array_equal(y, x.flat) - def test_file_position_after_fromfile(self): + def test_file_position_after_fromfile(self, tmp_filename): # gh-4118 sizes = [io.DEFAULT_BUFFER_SIZE//8, io.DEFAULT_BUFFER_SIZE, io.DEFAULT_BUFFER_SIZE*8] for size in sizes: - with open(self.filename, 'wb') as f: + with open(tmp_filename, 'wb') as f: f.seek(size-1) f.write(b'\0') for mode in ['rb', 'r+b']: err_msg = "%d %s" % (size, mode) - with open(self.filename, mode) as f: + with open(tmp_filename, mode) as f: f.read(2) np.fromfile(f, dtype=np.float64, count=1) pos = f.tell() assert_equal(pos, 10, err_msg=err_msg) - def test_file_position_after_tofile(self): + def test_file_position_after_tofile(self, tmp_filename): # gh-4118 sizes = [io.DEFAULT_BUFFER_SIZE//8, io.DEFAULT_BUFFER_SIZE, @@ -4945,7 +4987,7 @@ def test_file_position_after_tofile(self): for size in sizes: err_msg = "%d" % (size,) - with open(self.filename, 'wb') as f: + with open(tmp_filename, 'wb') as f: f.seek(size-1) f.write(b'\0') f.seek(10) @@ -4954,88 +4996,145 @@ def test_file_position_after_tofile(self): pos = f.tell() assert_equal(pos, 10 + 2 + 8, err_msg=err_msg) - with open(self.filename, 'r+b') as f: + with open(tmp_filename, 'r+b') as f: f.read(2) f.seek(0, 1) # seek between read&write required by ANSI C np.array([0], dtype=np.float64).tofile(f) pos = f.tell() assert_equal(pos, 10, err_msg=err_msg) - def test_load_object_array_fromfile(self): + def test_load_object_array_fromfile(self, tmp_filename): # gh-12300 - with open(self.filename, 'w') as f: + with open(tmp_filename, 'w') as f: # Ensure we have a file with consistent contents pass - with open(self.filename, 'rb') as f: + with open(tmp_filename, 'rb') as f: assert_raises_regex(ValueError, "Cannot read into object array", np.fromfile, f, dtype=object) assert_raises_regex(ValueError, "Cannot read into object array", - np.fromfile, self.filename, dtype=object) - - def test_fromfile_offset(self): - with open(self.filename, 'wb') as f: - self.x.tofile(f) - - with open(self.filename, 'rb') as f: - y = np.fromfile(f, dtype=self.dtype, offset=0) - assert_array_equal(y, self.x.flat) - - with open(self.filename, 'rb') as f: - count_items = len(self.x.flat) // 8 - offset_items = len(self.x.flat) // 4 - offset_bytes = self.dtype.itemsize * offset_items - y = np.fromfile(f, dtype=self.dtype, count=count_items, offset=offset_bytes) - assert_array_equal(y, self.x.flat[offset_items:offset_items+count_items]) + np.fromfile, tmp_filename, dtype=object) + + def test_fromfile_offset(self, x, tmp_filename): + with open(tmp_filename, 'wb') as f: + x.tofile(f) + + with open(tmp_filename, 'rb') as f: + y = np.fromfile(f, dtype=x.dtype, offset=0) + assert_array_equal(y, x.flat) + + with open(tmp_filename, 'rb') as f: + count_items = len(x.flat) // 8 + offset_items = len(x.flat) // 4 + offset_bytes = x.dtype.itemsize * offset_items + y = np.fromfile( + f, dtype=x.dtype, count=count_items, offset=offset_bytes + ) + assert_array_equal( + y, x.flat[offset_items:offset_items+count_items] + ) # subsequent seeks should stack - offset_bytes = self.dtype.itemsize - z = np.fromfile(f, dtype=self.dtype, offset=offset_bytes) - assert_array_equal(z, self.x.flat[offset_items+count_items+1:]) + offset_bytes = x.dtype.itemsize + z = np.fromfile(f, dtype=x.dtype, offset=offset_bytes) + assert_array_equal(z, x.flat[offset_items+count_items+1:]) - with open(self.filename, 'wb') as f: - self.x.tofile(f, sep=",") + with open(tmp_filename, 'wb') as f: + x.tofile(f, sep=",") - with open(self.filename, 'rb') as f: + with open(tmp_filename, 'rb') as f: assert_raises_regex( TypeError, "'offset' argument only permitted for binary files", - np.fromfile, self.filename, dtype=self.dtype, + np.fromfile, tmp_filename, dtype=x.dtype, sep=",", offset=1) - def _check_from(self, s, value, **kw): + @pytest.mark.skipif(IS_PYPY, reason="bug in PyPy's PyNumber_AsSsize_t") + def test_fromfile_bad_dup(self, x, tmp_filename): + def dup_str(fd): + return 'abc' + + def dup_bigint(fd): + return 2**68 + + old_dup = os.dup + try: + with open(tmp_filename, 'wb') as f: + x.tofile(f) + for dup, exc in ((dup_str, TypeError), (dup_bigint, OSError)): + os.dup = dup + assert_raises(exc, np.fromfile, f) + finally: + os.dup = old_dup + + def _check_from(self, s, value, filename, **kw): if 'sep' not in kw: y = np.frombuffer(s, **kw) else: y = np.fromstring(s, **kw) assert_array_equal(y, value) - with open(self.filename, 'wb') as f: + with open(filename, 'wb') as f: f.write(s) - y = np.fromfile(self.filename, **kw) + y = np.fromfile(filename, **kw) assert_array_equal(y, value) - def test_nan(self): + @pytest.fixture(params=["period", "comma"]) + def decimal_sep_localization(self, request): + """ + Including this fixture in a test will automatically + execute it with both types of decimal separator. + + So:: + + def test_decimal(decimal_sep_localization): + pass + + is equivalent to the following two tests:: + + def test_decimal_period_separator(): + pass + + def test_decimal_comma_separator(): + with CommaDecimalPointLocale(): + pass + """ + if request.param == "period": + yield + elif request.param == "comma": + with CommaDecimalPointLocale(): + yield + else: + assert False, request.param + + def test_nan(self, tmp_filename, decimal_sep_localization): self._check_from( b"nan +nan -nan NaN nan(foo) +NaN(BAR) -NAN(q_u_u_x_)", [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], + tmp_filename, sep=' ') - def test_inf(self): + def test_inf(self, tmp_filename, decimal_sep_localization): self._check_from( b"inf +inf -inf infinity -Infinity iNfInItY -inF", [np.inf, np.inf, -np.inf, np.inf, -np.inf, np.inf, -np.inf], + tmp_filename, sep=' ') - def test_numbers(self): - self._check_from(b"1.234 -1.234 .3 .3e55 -123133.1231e+133", - [1.234, -1.234, .3, .3e55, -123133.1231e+133], sep=' ') + def test_numbers(self, tmp_filename, decimal_sep_localization): + self._check_from( + b"1.234 -1.234 .3 .3e55 -123133.1231e+133", + [1.234, -1.234, .3, .3e55, -123133.1231e+133], + tmp_filename, + sep=' ') - def test_binary(self): - self._check_from(b'\x00\x00\x80?\x00\x00\x00@\x00\x00@@\x00\x00\x80@', - np.array([1, 2, 3, 4]), - dtype=' 1 minute on mechanical hard drive def test_big_binary(self): @@ -5062,91 +5161,89 @@ def test_big_binary(self): except (MemoryError, ValueError): pass - def test_string(self): - self._check_from(b'1,2,3,4', [1., 2., 3., 4.], sep=',') + def test_string(self, tmp_filename): + self._check_from(b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, sep=',') - def test_counted_string(self): - self._check_from(b'1,2,3,4', [1., 2., 3., 4.], count=4, sep=',') - self._check_from(b'1,2,3,4', [1., 2., 3.], count=3, sep=',') - self._check_from(b'1,2,3,4', [1., 2., 3., 4.], count=-1, sep=',') + def test_counted_string(self, tmp_filename, decimal_sep_localization): + self._check_from( + b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, count=4, sep=',') + self._check_from( + b'1,2,3,4', [1., 2., 3.], tmp_filename, count=3, sep=',') + self._check_from( + b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, count=-1, sep=',') - def test_string_with_ws(self): - self._check_from(b'1 2 3 4 ', [1, 2, 3, 4], dtype=int, sep=' ') + def test_string_with_ws(self, tmp_filename): + self._check_from( + b'1 2 3 4 ', [1, 2, 3, 4], tmp_filename, dtype=int, sep=' ') - def test_counted_string_with_ws(self): - self._check_from(b'1 2 3 4 ', [1, 2, 3], count=3, dtype=int, - sep=' ') + def test_counted_string_with_ws(self, tmp_filename): + self._check_from( + b'1 2 3 4 ', [1, 2, 3], tmp_filename, count=3, dtype=int, + sep=' ') - def test_ascii(self): - self._check_from(b'1 , 2 , 3 , 4', [1., 2., 3., 4.], sep=',') - self._check_from(b'1,2,3,4', [1., 2., 3., 4.], dtype=float, sep=',') + def test_ascii(self, tmp_filename, decimal_sep_localization): + self._check_from( + b'1 , 2 , 3 , 4', [1., 2., 3., 4.], tmp_filename, sep=',') + self._check_from( + b'1,2,3,4', [1., 2., 3., 4.], tmp_filename, dtype=float, sep=',') - def test_malformed(self): + def test_malformed(self, tmp_filename, decimal_sep_localization): with assert_warns(DeprecationWarning): - self._check_from(b'1.234 1,234', [1.234, 1.], sep=' ') + self._check_from( + b'1.234 1,234', [1.234, 1.], tmp_filename, sep=' ') - def test_long_sep(self): - self._check_from(b'1_x_3_x_4_x_5', [1, 3, 4, 5], sep='_x_') + def test_long_sep(self, tmp_filename): + self._check_from( + b'1_x_3_x_4_x_5', [1, 3, 4, 5], tmp_filename, sep='_x_') - def test_dtype(self): + def test_dtype(self, tmp_filename): v = np.array([1, 2, 3, 4], dtype=np.int_) - self._check_from(b'1,2,3,4', v, sep=',', dtype=np.int_) + self._check_from(b'1,2,3,4', v, tmp_filename, sep=',', dtype=np.int_) - def test_dtype_bool(self): + def test_dtype_bool(self, tmp_filename): # can't use _check_from because fromstring can't handle True/False v = np.array([True, False, True, False], dtype=np.bool_) s = b'1,0,-2.3,0' - with open(self.filename, 'wb') as f: + with open(tmp_filename, 'wb') as f: f.write(s) - y = np.fromfile(self.filename, sep=',', dtype=np.bool_) + y = np.fromfile(tmp_filename, sep=',', dtype=np.bool_) assert_(y.dtype == '?') assert_array_equal(y, v) - def test_tofile_sep(self): + def test_tofile_sep(self, tmp_filename, decimal_sep_localization): x = np.array([1.51, 2, 3.51, 4], dtype=float) - with open(self.filename, 'w') as f: + with open(tmp_filename, 'w') as f: x.tofile(f, sep=',') - with open(self.filename, 'r') as f: + with open(tmp_filename, 'r') as f: s = f.read() #assert_equal(s, '1.51,2.0,3.51,4.0') y = np.array([float(p) for p in s.split(',')]) assert_array_equal(x,y) - def test_tofile_format(self): + def test_tofile_format(self, tmp_filename, decimal_sep_localization): x = np.array([1.51, 2, 3.51, 4], dtype=float) - with open(self.filename, 'w') as f: + with open(tmp_filename, 'w') as f: x.tofile(f, sep=',', format='%.2f') - with open(self.filename, 'r') as f: + with open(tmp_filename, 'r') as f: s = f.read() assert_equal(s, '1.51,2.00,3.51,4.00') - def test_tofile_cleanup(self): + def test_tofile_cleanup(self, tmp_filename): x = np.zeros((10), dtype=object) - with open(self.filename, 'wb') as f: + with open(tmp_filename, 'wb') as f: assert_raises(IOError, lambda: x.tofile(f, sep='')) # Dup-ed file handle should be closed or remove will fail on Windows OS - os.remove(self.filename) + os.remove(tmp_filename) # Also make sure that we close the Python handle - assert_raises(IOError, lambda: x.tofile(self.filename)) - os.remove(self.filename) - - def test_locale(self): - with CommaDecimalPointLocale(): - self.test_numbers() - self.test_nan() - self.test_inf() - self.test_counted_string() - self.test_ascii() - self.test_malformed() - self.test_tofile_sep() - self.test_tofile_format() - - def test_fromfile_subarray_binary(self): + assert_raises(IOError, lambda: x.tofile(tmp_filename)) + os.remove(tmp_filename) + + def test_fromfile_subarray_binary(self, tmp_filename): # Test subarray dtypes which are absorbed into the shape x = np.arange(24, dtype="i4").reshape(2, 3, 4) - x.tofile(self.filename) - res = np.fromfile(self.filename, dtype="(3,4)i4") + x.tofile(tmp_filename) + res = np.fromfile(tmp_filename, dtype="(3,4)i4") assert_array_equal(x, res) x_str = x.tobytes() @@ -5155,21 +5252,21 @@ def test_fromfile_subarray_binary(self): res = np.fromstring(x_str, dtype="(3,4)i4") assert_array_equal(x, res) - def test_parsing_subarray_unsupported(self): + def test_parsing_subarray_unsupported(self, tmp_filename): # We currently do not support parsing subarray dtypes data = "12,42,13," * 50 with pytest.raises(ValueError): expected = np.fromstring(data, dtype="(3,)i", sep=",") - with open(self.filename, "w") as f: + with open(tmp_filename, "w") as f: f.write(data) with pytest.raises(ValueError): - np.fromfile(self.filename, dtype="(3,)i", sep=",") + np.fromfile(tmp_filename, dtype="(3,)i", sep=",") - def test_read_shorter_than_count_subarray(self): + def test_read_shorter_than_count_subarray(self, tmp_filename): # Test that requesting more values does not cause any problems - # in conjuction with subarray dimensions being absored into the + # in conjunction with subarray dimensions being absorbed into the # array dimension. expected = np.arange(511 * 10, dtype="i").reshape(-1, 10) @@ -5178,8 +5275,8 @@ def test_read_shorter_than_count_subarray(self): with pytest.warns(DeprecationWarning): np.fromstring(binary, dtype="(10,)i", count=10000) - expected.tofile(self.filename) - res = np.fromfile(self.filename, dtype="(10,)i", count=10000) + expected.tofile(tmp_filename) + res = np.fromfile(tmp_filename, dtype="(10,)i", count=10000) assert_array_equal(res, expected) @@ -5713,6 +5810,15 @@ def test_mean_where(self): np.array(_res)) assert_allclose(np.mean(a, axis=_ax, where=_wh), np.array(_res)) + + a3d = np.arange(16).reshape((2, 2, 4)) + _wh_partial = np.array([False, True, True, False]) + _res = [[1.5, 5.5], [9.5, 13.5]] + assert_allclose(a3d.mean(axis=2, where=_wh_partial), + np.array(_res)) + assert_allclose(np.mean(a3d, axis=2, where=_wh_partial), + np.array(_res)) + with pytest.warns(RuntimeWarning) as w: assert_allclose(a.mean(axis=1, where=wh_partial), np.array([np.nan, 5.5, 9.5, np.nan])) @@ -5788,6 +5894,15 @@ def test_var_where(self): np.array(_res)) assert_allclose(np.var(a, axis=_ax, where=_wh), np.array(_res)) + + a3d = np.arange(16).reshape((2, 2, 4)) + _wh_partial = np.array([False, True, True, False]) + _res = [[0.25, 0.25], [0.25, 0.25]] + assert_allclose(a3d.var(axis=2, where=_wh_partial), + np.array(_res)) + assert_allclose(np.var(a3d, axis=2, where=_wh_partial), + np.array(_res)) + assert_allclose(np.var(a, axis=1, where=wh_full), np.var(a[wh_full].reshape((5, 3)), axis=1)) assert_allclose(np.var(a, axis=0, where=wh_partial), @@ -5827,6 +5942,14 @@ def test_std_where(self): assert_allclose(a.std(axis=_ax, where=_wh), _res) assert_allclose(np.std(a, axis=_ax, where=_wh), _res) + a3d = np.arange(16).reshape((2, 2, 4)) + _wh_partial = np.array([False, True, True, False]) + _res = [[0.5, 0.5], [0.5, 0.5]] + assert_allclose(a3d.std(axis=2, where=_wh_partial), + np.array(_res)) + assert_allclose(np.std(a3d, axis=2, where=_wh_partial), + np.array(_res)) + assert_allclose(a.std(axis=1, where=whf), np.std(a[whf].reshape((5,3)), axis=1)) assert_allclose(np.std(a, axis=1, where=whf), @@ -5859,6 +5982,7 @@ def __array_finalize__(self, obj): res = dat.var(1) assert_(res.info == dat.info) + class TestVdot: def test_basic(self): dt_numeric = np.typecodes['AllFloat'] + np.typecodes['AllInteger'] @@ -6125,6 +6249,71 @@ def test_dot_array_order(self): assert_equal(np.dot(b, a), res) assert_equal(np.dot(b, b), res) + def test_accelerate_framework_sgemv_fix(self): + + def aligned_array(shape, align, dtype, order='C'): + d = dtype(0) + N = np.prod(shape) + tmp = np.zeros(N * d.nbytes + align, dtype=np.uint8) + address = tmp.__array_interface__["data"][0] + for offset in range(align): + if (address + offset) % align == 0: + break + tmp = tmp[offset:offset+N*d.nbytes].view(dtype=dtype) + return tmp.reshape(shape, order=order) + + def as_aligned(arr, align, dtype, order='C'): + aligned = aligned_array(arr.shape, align, dtype, order) + aligned[:] = arr[:] + return aligned + + def assert_dot_close(A, X, desired): + assert_allclose(np.dot(A, X), desired, rtol=1e-5, atol=1e-7) + + m = aligned_array(100, 15, np.float32) + s = aligned_array((100, 100), 15, np.float32) + np.dot(s, m) # this will always segfault if the bug is present + + testdata = itertools.product((15, 32), (10000,), (200, 89), ('C', 'F')) + for align, m, n, a_order in testdata: + # Calculation in double precision + A_d = np.random.rand(m, n) + X_d = np.random.rand(n) + desired = np.dot(A_d, X_d) + # Calculation with aligned single precision + A_f = as_aligned(A_d, align, np.float32, order=a_order) + X_f = as_aligned(X_d, align, np.float32) + assert_dot_close(A_f, X_f, desired) + # Strided A rows + A_d_2 = A_d[::2] + desired = np.dot(A_d_2, X_d) + A_f_2 = A_f[::2] + assert_dot_close(A_f_2, X_f, desired) + # Strided A columns, strided X vector + A_d_22 = A_d_2[:, ::2] + X_d_2 = X_d[::2] + desired = np.dot(A_d_22, X_d_2) + A_f_22 = A_f_2[:, ::2] + X_f_2 = X_f[::2] + assert_dot_close(A_f_22, X_f_2, desired) + # Check the strides are as expected + if a_order == 'F': + assert_equal(A_f_22.strides, (8, 8 * m)) + else: + assert_equal(A_f_22.strides, (8 * n, 8)) + assert_equal(X_f_2.strides, (8,)) + # Strides in A rows + cols only + X_f_2c = as_aligned(X_f_2, align, np.float32) + assert_dot_close(A_f_22, X_f_2c, desired) + # Strides just in A cols + A_d_12 = A_d[:, ::2] + desired = np.dot(A_d_12, X_d_2) + A_f_12 = A_f[:, ::2] + assert_dot_close(A_f_12, X_f_2c, desired) + # Strides in A cols and X + assert_dot_close(A_f_12, X_f_2, desired) + + class MatmulCommon: """Common tests for '@' operator and numpy.matmul. @@ -6377,6 +6566,16 @@ def test_out_arg(self): c = c.astype(tgt.dtype) assert_array_equal(c, tgt) + def test_empty_out(self): + # Check that the output cannot be broadcast, so that it cannot be + # size zero when the outer dimensions (iterator size) has size zero. + arr = np.ones((0, 1, 1)) + out = np.ones((1, 1, 1)) + assert self.matmul(arr, arr).shape == (0, 1, 1) + + with pytest.raises(ValueError, match=r"non-broadcastable"): + self.matmul(arr, arr, out=out) + def test_out_contiguous(self): a = np.ones((5, 2), dtype=float) b = np.array([[1, 3], [5, 7]], dtype=float) @@ -7207,7 +7406,7 @@ def test_roundtrip_half(self): self._check_roundtrip(x) def test_roundtrip_single_types(self): - for typ in np.typeDict.values(): + for typ in np.sctypeDict.values(): dtype = np.dtype(typ) if dtype.char in 'Mm': @@ -7350,9 +7549,9 @@ def test_export_flags(self): def test_export_and_pickle_user_dtype(self, obj, error): # User dtypes should export successfully when FORMAT was not requested. with pytest.raises(error): - _multiarray_tests.get_buffer_info(obj, ("STRIDED", "FORMAT")) + _multiarray_tests.get_buffer_info(obj, ("STRIDED_RO", "FORMAT")) - _multiarray_tests.get_buffer_info(obj, ("STRIDED",)) + _multiarray_tests.get_buffer_info(obj, ("STRIDED_RO",)) # This is currently also necessary to implement pickling: pickle_obj = pickle.dumps(obj) @@ -7453,7 +7652,7 @@ def test_out_of_order_fields(self): memoryview(arr) def test_max_dims(self): - a = np.empty((1,) * 32) + a = np.ones((1,) * 32) self._check_roundtrip(a) @pytest.mark.slow @@ -8473,6 +8672,22 @@ def test_zero_step(self): assert_raises(ZeroDivisionError, np.arange, 0, 0, 0) assert_raises(ZeroDivisionError, np.arange, 0.0, 0.0, 0.0) + def test_require_range(self): + assert_raises(TypeError, np.arange) + assert_raises(TypeError, np.arange, step=3) + assert_raises(TypeError, np.arange, dtype='int64') + assert_raises(TypeError, np.arange, start=4) + + def test_start_stop_kwarg(self): + keyword_stop = np.arange(stop=3) + keyword_zerotostop = np.arange(start=0, stop=3) + keyword_start_stop = np.arange(start=3, stop=9) + + assert len(keyword_stop) == 3 + assert len(keyword_zerotostop) == 3 + assert len(keyword_start_stop) == 6 + assert_array_equal(keyword_stop, keyword_zerotostop) + class TestArrayFinalize: """ Tests __array_finalize__ """ @@ -8486,6 +8701,15 @@ def __array_finalize__(self, obj): a = np.array(1).view(SavesBase) assert_(a.saved_base is a.base) + def test_bad_finalize(self): + class BadAttributeArray(np.ndarray): + @property + def __array_finalize__(self): + raise RuntimeError("boohoo!") + + with pytest.raises(RuntimeError, match="boohoo!"): + np.arange(10).view(BadAttributeArray) + def test_lifetime_on_error(self): # gh-11237 class RaisesInFinalize(np.ndarray): @@ -8553,23 +8777,22 @@ class MyAlwaysEqualNew(MyAlwaysEqual): assert_equal(array != my_always_equal, 'ne') -def test_npymath_complex(): +@pytest.mark.parametrize( + ["fun", "npfun"], + [ + (_multiarray_tests.npy_cabs, np.absolute), + (_multiarray_tests.npy_carg, np.angle) + ] +) +@pytest.mark.parametrize("x", [1, np.inf, -np.inf, np.nan]) +@pytest.mark.parametrize("y", [1, np.inf, -np.inf, np.nan]) +@pytest.mark.parametrize("test_dtype", np.complexfloating.__subclasses__()) +def test_npymath_complex(fun, npfun, x, y, test_dtype): # Smoketest npymath functions - from numpy.core._multiarray_tests import ( - npy_cabs, npy_carg) - - funcs = {npy_cabs: np.absolute, - npy_carg: np.angle} - vals = (1, np.inf, -np.inf, np.nan) - types = (np.complex64, np.complex128, np.clongdouble) - - for fun, npfun in funcs.items(): - for x, y in itertools.product(vals, vals): - for t in types: - z = t(complex(x, y)) - got = fun(z) - expected = npfun(z) - assert_allclose(got, expected) + z = test_dtype(complex(x, y)) + got = fun(z) + expected = npfun(z) + assert_allclose(got, expected) def test_npymath_real(): diff --git a/numpy/core/tests/test_nditer.py b/numpy/core/tests/test_nditer.py index e10c7ad92db3..b44343c5755c 100644 --- a/numpy/core/tests/test_nditer.py +++ b/numpy/core/tests/test_nditer.py @@ -1,6 +1,9 @@ import sys import pytest +import textwrap +import subprocess + import numpy as np import numpy.core._multiarray_tests as _multiarray_tests from numpy import array, arange, nditer, all @@ -1362,6 +1365,75 @@ def test_iter_copy(): j = i.copy() assert_equal([x[()] for x in j], a.ravel(order='F')) + +@pytest.mark.parametrize("dtype", np.typecodes["All"]) +@pytest.mark.parametrize("loop_dtype", np.typecodes["All"]) +@pytest.mark.filterwarnings("ignore::numpy.ComplexWarning") +def test_iter_copy_casts(dtype, loop_dtype): + # Ensure the dtype is never flexible: + if loop_dtype.lower() == "m": + loop_dtype = loop_dtype + "[ms]" + elif np.dtype(loop_dtype).itemsize == 0: + loop_dtype = loop_dtype + "50" + + # Make things a bit more interesting by requiring a byte-swap as well: + arr = np.ones(1000, dtype=np.dtype(dtype).newbyteorder()) + try: + expected = arr.astype(loop_dtype) + except Exception: + # Some casts are not possible, do not worry about them + return + + it = np.nditer((arr,), ["buffered", "external_loop", "refs_ok"], + op_dtypes=[loop_dtype], casting="unsafe") + + if np.issubdtype(np.dtype(loop_dtype), np.number): + # Casting to strings may be strange, but for simple dtypes do not rely + # on the cast being correct: + assert_array_equal(expected, np.ones(1000, dtype=loop_dtype)) + + it_copy = it.copy() + res = next(it) + del it + res_copy = next(it_copy) + del it_copy + + assert_array_equal(res, expected) + assert_array_equal(res_copy, expected) + + +def test_iter_copy_casts_structured(): + # Test a complicated structured dtype for casting, as it requires + # both multiple steps and a more complex casting setup. + # Includes a structured -> unstructured (any to object), and many other + # casts, which cause this to require all steps in the casting machinery + # one level down as well as the iterator copy (which uses NpyAuxData clone) + in_dtype = np.dtype([("a", np.dtype("i,")), + ("b", np.dtype(">i,d,S17,>d,(3)f,O,i1"))]) + out_dtype = np.dtype([("a", np.dtype("O")), + ("b", np.dtype(">i,>i,S17,>d,>U3,(3)d,i1,O"))]) + arr = np.ones(1000, dtype=in_dtype) + + it = np.nditer((arr,), ["buffered", "external_loop", "refs_ok"], + op_dtypes=[out_dtype], casting="unsafe") + it_copy = it.copy() + + res1 = next(it) + del it + res2 = next(it_copy) + del it_copy + + expected = arr["a"].astype(out_dtype["a"]) + assert_array_equal(res1["a"], expected) + assert_array_equal(res2["a"], expected) + + for field in in_dtype["b"].names: + # Note that the .base avoids the subarray field + expected = arr["b"][field].astype(out_dtype["b"][field].base) + assert_array_equal(res1["b"][field], expected) + assert_array_equal(res2["b"][field], expected) + + def test_iter_allocate_output_simple(): # Check that the iterator will properly allocate outputs @@ -1886,16 +1958,58 @@ def test_iter_buffered_cast_structured_type(): [np.array((1, 2, 3), dtype=sdt2), np.array((4, 5, 6), dtype=sdt2)]) + +def test_iter_buffered_cast_structured_type_failure_with_cleanup(): # make sure struct type -> struct type with different # number of fields fails sdt1 = [('a', 'f4'), ('b', 'i8'), ('d', 'O')] sdt2 = [('b', 'O'), ('a', 'f8')] a = np.array([(1, 2, 3), (4, 5, 6)], dtype=sdt1) - assert_raises(ValueError, lambda : ( - nditer(a, ['buffered', 'refs_ok'], ['readwrite'], - casting='unsafe', - op_dtypes=sdt2))) + for intent in ["readwrite", "readonly", "writeonly"]: + # If the following assert fails, the place where the error is raised + # within nditer may change. That is fine, but it may make sense for + # a new (hard to design) test to replace it. The `simple_arr` is + # designed to require a multi-step cast (due to having fields). + assert np.can_cast(a.dtype, sdt2, casting="unsafe") + simple_arr = np.array([1, 2], dtype="i,i") # requires clean up + with pytest.raises(ValueError): + nditer((simple_arr, a), ['buffered', 'refs_ok'], [intent, intent], + casting='unsafe', op_dtypes=["f,f", sdt2]) + + +def test_buffered_cast_error_paths(): + with pytest.raises(ValueError): + # The input is cast into an `S3` buffer + np.nditer((np.array("a", dtype="S1"),), op_dtypes=["i"], + casting="unsafe", flags=["buffered"]) + + # The `M8[ns]` is cast into the `S3` output + it = np.nditer((np.array(1, dtype="i"),), op_dtypes=["S1"], + op_flags=["writeonly"], casting="unsafe", flags=["buffered"]) + with pytest.raises(ValueError): + with it: + buf = next(it) + buf[...] = "a" # cannot be converted to int. + +@pytest.mark.skipif(not HAS_REFCOUNT, reason="PyPy seems to not hit this.") +def test_buffered_cast_error_paths_unraisable(): + # The following gives an unraisable error. Pytest sometimes captures that + # (depending python and/or pytest version). So with Python>=3.8 this can + # probably be cleaned out in the future to check for + # pytest.PytestUnraisableExceptionWarning: + code = textwrap.dedent(""" + import numpy as np + + it = np.nditer((np.array(1, dtype="i"),), op_dtypes=["S1"], + op_flags=["writeonly"], casting="unsafe", flags=["buffered"]) + buf = next(it) + buf[...] = "a" + del buf, it # Flushing only happens during deallocate right now. + """) + res = subprocess.check_output([sys.executable, "-c", code], + stderr=subprocess.STDOUT, text=True) + assert "ValueError" in res def test_iter_buffered_cast_subarray(): @@ -2591,9 +2705,25 @@ def test_iter_writemasked_badinput(): op_dtypes=['f4', None], casting='same_kind') -def test_iter_writemasked(): - a = np.zeros((3,), dtype='f8') - msk = np.array([True, True, False]) +def _is_buffered(iterator): + try: + iterator.itviews + except ValueError: + return True + return False + +@pytest.mark.parametrize("a", + [np.zeros((3,), dtype='f8'), + np.zeros((9876, 3*5), dtype='f8')[::2, :], + np.zeros((4, 312, 124, 3), dtype='f8')[::2, :, ::2, :]]) +def test_iter_writemasked(a): + # Note, the slicing above is to ensure that nditer cannot combine multiple + # axes into one. The repetition is just to make things a bit more + # interesting. + shape = a.shape + reps = shape[-1] // 3 + msk = np.empty(shape, dtype=bool) + msk[...] = [True, True, False] * reps # When buffering is unused, 'writemasked' effectively does nothing. # It's up to the user of the iterator to obey the requested semantics. @@ -2604,18 +2734,31 @@ def test_iter_writemasked(): for x, m in it: x[...] = 1 # Because we violated the semantics, all the values became 1 - assert_equal(a, [1, 1, 1]) + assert_equal(a, np.broadcast_to([1, 1, 1] * reps, shape)) # Even if buffering is enabled, we still may be accessing the array # directly. it = np.nditer([a, msk], ['buffered'], [['readwrite', 'writemasked'], ['readonly', 'arraymask']]) + # @seberg: I honestly don't currently understand why a "buffered" iterator + # would end up not using a buffer for the small array here at least when + # "writemasked" is used, that seems confusing... Check by testing for + # actual memory overlap! + is_buffered = True with it: for x, m in it: x[...] = 2.5 - # Because we violated the semantics, all the values became 2.5 - assert_equal(a, [2.5, 2.5, 2.5]) + if np.may_share_memory(x, a): + is_buffered = False + + if not is_buffered: + # Because we violated the semantics, all the values became 2.5 + assert_equal(a, np.broadcast_to([2.5, 2.5, 2.5] * reps, shape)) + else: + # For large sizes, the iterator may be buffered: + assert_equal(a, np.broadcast_to([2.5, 2.5, 1] * reps, shape)) + a[...] = 2.5 # If buffering will definitely happening, for instance because of # a cast, only the items selected by the mask will be copied back from @@ -2630,7 +2773,38 @@ def test_iter_writemasked(): x[...] = 3 # Even though we violated the semantics, only the selected values # were copied back - assert_equal(a, [3, 3, 2.5]) + assert_equal(a, np.broadcast_to([3, 3, 2.5] * reps, shape)) + +def test_iter_writemasked_decref(): + # force casting (to make it interesting) by using a structured dtype. + arr = np.arange(10000).astype(">i,O") + original = arr.copy() + mask = np.random.randint(0, 2, size=10000).astype(bool) + + it = np.nditer([arr, mask], ['buffered', "refs_ok"], + [['readwrite', 'writemasked'], + ['readonly', 'arraymask']], + op_dtypes=["" + + @pytest.mark.parametrize("operation", [ + 'add', 'subtract', 'multiply', 'floor_divide', + 'conjugate', 'fmod', 'square', 'reciprocal', + 'power', 'absolute', 'negative', 'positive', + 'greater', 'greater_equal', 'less', + 'less_equal', 'equal', 'not_equal', 'logical_and', + 'logical_not', 'logical_or', 'bitwise_and', 'bitwise_or', + 'bitwise_xor', 'invert', 'left_shift', 'right_shift', + 'gcd', 'lcm' + ] + ) + @pytest.mark.parametrize("order", [ + ('b->', 'B->'), + ('h->', 'H->'), + ('i->', 'I->'), + ('l->', 'L->'), + ('q->', 'Q->'), + ] + ) + def test_ufunc_order(self, operation, order): + # gh-18075 + # Ensure signed types before unsigned + def get_idx(string, str_lst): + for i, s in enumerate(str_lst): + if string in s: + return i + raise ValueError(f"{string} not in list") + types = getattr(np, operation).types + assert get_idx(order[0], types) < get_idx(order[1], types), ( + f"Unexpected types order of ufunc in {operation}" + f"for {order}. Possible fix: Use signed before unsigned" + "in generate_umath.py") diff --git a/numpy/core/tests/test_scalar_methods.py b/numpy/core/tests/test_scalar_methods.py index 4f5fd298887d..3693bba59ce6 100644 --- a/numpy/core/tests/test_scalar_methods.py +++ b/numpy/core/tests/test_scalar_methods.py @@ -89,7 +89,8 @@ def test_against_known_values(self): ]) def test_roundtrip(self, ftype, frac_vals, exp_vals): for frac, exp in zip(frac_vals, exp_vals): - f = np.ldexp(frac, exp, dtype=ftype) + f = np.ldexp(ftype(frac), exp) + assert f.dtype == ftype n, d = f.as_integer_ratio() try: diff --git a/numpy/core/tests/test_scalarbuffer.py b/numpy/core/tests/test_scalarbuffer.py index 574c56864bb7..851cd3081aee 100644 --- a/numpy/core/tests/test_scalarbuffer.py +++ b/numpy/core/tests/test_scalarbuffer.py @@ -3,6 +3,7 @@ """ import numpy as np from numpy.core._rational_tests import rational +from numpy.core._multiarray_tests import get_buffer_info import pytest from numpy.testing import assert_, assert_equal, assert_raises @@ -52,10 +53,20 @@ def test_scalar_dim(self, scalar): assert_equal(mv_x.suboffsets, ()) @pytest.mark.parametrize('scalar, code', scalars_and_codes, ids=codes_only) - def test_scalar_known_code(self, scalar, code): + def test_scalar_code_and_properties(self, scalar, code): x = scalar() + expected = dict(strides=(), itemsize=x.dtype.itemsize, ndim=0, + shape=(), format=code, readonly=True) + mv_x = memoryview(x) - assert_equal(mv_x.format, code) + print(mv_x.readonly, self._as_dict(mv_x)) + assert self._as_dict(mv_x) == expected + + @pytest.mark.parametrize('scalar', scalars_only, ids=codes_only) + def test_scalar_buffers_readonly(self, scalar): + x = scalar() + with pytest.raises(BufferError, match="scalar buffer is readonly"): + get_buffer_info(x, ["WRITABLE"]) def test_void_scalar_structured_data(self): dt = np.dtype([('name', np.unicode_, 16), ('grades', np.float64, (2,))]) @@ -77,9 +88,14 @@ def test_void_scalar_structured_data(self): assert_equal(mv_x.itemsize, mv_a.itemsize) assert_equal(mv_x.format, mv_a.format) + # Check that we do not allow writeable buffer export (technically + # we could allow it sometimes here...) + with pytest.raises(BufferError, match="scalar buffer is readonly"): + get_buffer_info(x, ["WRITABLE"]) + def _as_dict(self, m): return dict(strides=m.strides, shape=m.shape, itemsize=m.itemsize, - ndim=m.ndim, format=m.format) + ndim=m.ndim, format=m.format, readonly=m.readonly) def test_datetime_memoryview(self): # gh-11656 @@ -88,7 +104,7 @@ def test_datetime_memoryview(self): dt1 = np.datetime64('2016-01-01') dt2 = np.datetime64('2017-01-01') expected = dict(strides=(1,), itemsize=1, ndim=1, shape=(8,), - format='B') + format='B', readonly=True) v = memoryview(dt1) assert self._as_dict(v) == expected @@ -100,6 +116,10 @@ def test_datetime_memoryview(self): # Fails to create a PEP 3118 valid buffer assert_raises((ValueError, BufferError), memoryview, a[0]) + # Check that we do not allow writeable buffer export + with pytest.raises(BufferError, match="scalar buffer is readonly"): + get_buffer_info(dt1, ["WRITABLE"]) + @pytest.mark.parametrize('s', [ pytest.param("\x32\x32", id="ascii"), pytest.param("\uFE0F\uFE0F", id="basic multilingual"), @@ -109,7 +129,8 @@ def test_str_ucs4(self, s): s = np.str_(s) # only our subclass implements the buffer protocol # all the same, characters always encode as ucs4 - expected = dict(strides=(), itemsize=8, ndim=0, shape=(), format='2w') + expected = dict(strides=(), itemsize=8, ndim=0, shape=(), format='2w', + readonly=True) v = memoryview(s) assert self._as_dict(v) == expected @@ -119,7 +140,15 @@ def test_str_ucs4(self, s): assert_equal(code_points, [ord(c) for c in s]) + # Check that we do not allow writeable buffer export + with pytest.raises(BufferError, match="scalar buffer is readonly"): + get_buffer_info(s, ["WRITABLE"]) + def test_user_scalar_fails_buffer(self): r = rational(1) with assert_raises(TypeError): memoryview(r) + + # Check that we do not allow writeable buffer export + with pytest.raises(BufferError, match="scalar buffer is readonly"): + get_buffer_info(r, ["WRITABLE"]) \ No newline at end of file diff --git a/numpy/core/tests/test_scalarmath.py b/numpy/core/tests/test_scalarmath.py index d8529418e58b..09a734284a76 100644 --- a/numpy/core/tests/test_scalarmath.py +++ b/numpy/core/tests/test_scalarmath.py @@ -1,9 +1,12 @@ +import contextlib import sys import warnings import itertools import operator import platform import pytest +from hypothesis import given, settings, Verbosity, assume +from hypothesis.strategies import sampled_from import numpy as np from numpy.testing import ( @@ -404,15 +407,15 @@ def test_iinfo_long_values(self): assert_(res == tgt) for code in np.typecodes['AllInteger']: - res = np.typeDict[code](np.iinfo(code).max) + res = np.dtype(code).type(np.iinfo(code).max) tgt = np.iinfo(code).max assert_(res == tgt) def test_int_raise_behaviour(self): def overflow_error_func(dtype): - np.typeDict[dtype](np.iinfo(dtype).max + 1) + dtype(np.iinfo(dtype).max + 1) - for code in 'lLqQ': + for code in [np.int_, np.uint, np.longlong, np.ulonglong]: assert_raises(OverflowError, overflow_error_func, code) def test_int_from_infinite_longdouble(self): @@ -653,33 +656,33 @@ def test_result(self): class TestAbs: - def _test_abs_func(self, absfunc): - for tp in floating_types + complex_floating_types: - x = tp(-1.5) - assert_equal(absfunc(x), 1.5) - x = tp(0.0) - res = absfunc(x) - # assert_equal() checks zero signedness - assert_equal(res, 0.0) - x = tp(-0.0) - res = absfunc(x) - assert_equal(res, 0.0) - - x = tp(np.finfo(tp).max) - assert_equal(absfunc(x), x.real) - - x = tp(np.finfo(tp).tiny) - assert_equal(absfunc(x), x.real) - - x = tp(np.finfo(tp).min) - assert_equal(absfunc(x), -x.real) - - def test_builtin_abs(self): - self._test_abs_func(abs) - - def test_numpy_abs(self): - self._test_abs_func(np.abs) - + def _test_abs_func(self, absfunc, test_dtype): + x = test_dtype(-1.5) + assert_equal(absfunc(x), 1.5) + x = test_dtype(0.0) + res = absfunc(x) + # assert_equal() checks zero signedness + assert_equal(res, 0.0) + x = test_dtype(-0.0) + res = absfunc(x) + assert_equal(res, 0.0) + + x = test_dtype(np.finfo(test_dtype).max) + assert_equal(absfunc(x), x.real) + + x = test_dtype(np.finfo(test_dtype).tiny) + assert_equal(absfunc(x), x.real) + + x = test_dtype(np.finfo(test_dtype).min) + assert_equal(absfunc(x), -x.real) + + @pytest.mark.parametrize("dtype", floating_types + complex_floating_types) + def test_builtin_abs(self, dtype): + self._test_abs_func(abs, dtype) + + @pytest.mark.parametrize("dtype", floating_types + complex_floating_types) + def test_numpy_abs(self, dtype): + self._test_abs_func(np.abs, dtype) class TestBitShifts: @@ -707,3 +710,115 @@ def test_shift_all_bits(self, type_code, op): shift_arr = np.array([shift]*32, dtype=dt) res_arr = op(val_arr, shift_arr) assert_equal(res_arr, res_scl) + + +class TestHash: + @pytest.mark.parametrize("type_code", np.typecodes['AllInteger']) + def test_integer_hashes(self, type_code): + scalar = np.dtype(type_code).type + for i in range(128): + assert hash(i) == hash(scalar(i)) + + @pytest.mark.parametrize("type_code", np.typecodes['AllFloat']) + def test_float_and_complex_hashes(self, type_code): + scalar = np.dtype(type_code).type + for val in [np.pi, np.inf, 3, 6.]: + numpy_val = scalar(val) + # Cast back to Python, in case the NumPy scalar has less precision + if numpy_val.dtype.kind == 'c': + val = complex(numpy_val) + else: + val = float(numpy_val) + assert val == numpy_val + print(repr(numpy_val), repr(val)) + assert hash(val) == hash(numpy_val) + + if hash(float(np.nan)) != hash(float(np.nan)): + # If Python distinguises different NaNs we do so too (gh-18833) + assert hash(scalar(np.nan)) != hash(scalar(np.nan)) + + @pytest.mark.parametrize("type_code", np.typecodes['Complex']) + def test_complex_hashes(self, type_code): + # Test some complex valued hashes specifically: + scalar = np.dtype(type_code).type + for val in [np.pi+1j, np.inf-3j, 3j, 6.+1j]: + numpy_val = scalar(val) + assert hash(complex(numpy_val)) == hash(numpy_val) + + +@contextlib.contextmanager +def recursionlimit(n): + o = sys.getrecursionlimit() + try: + sys.setrecursionlimit(n) + yield + finally: + sys.setrecursionlimit(o) + + +objecty_things = [object(), None] +reasonable_operators_for_scalars = [ + operator.lt, operator.le, operator.eq, operator.ne, operator.ge, + operator.gt, operator.add, operator.floordiv, operator.mod, + operator.mul, operator.matmul, operator.pow, operator.sub, + operator.truediv, +] + + +@given(sampled_from(objecty_things), + sampled_from(reasonable_operators_for_scalars), + sampled_from(types)) +@settings(verbosity=Verbosity.verbose) +def test_operator_object_left(o, op, type_): + try: + with recursionlimit(200): + op(o, type_(1)) + except TypeError: + pass + + +@given(sampled_from(objecty_things), + sampled_from(reasonable_operators_for_scalars), + sampled_from(types)) +def test_operator_object_right(o, op, type_): + try: + with recursionlimit(200): + op(type_(1), o) + except TypeError: + pass + + +@given(sampled_from(reasonable_operators_for_scalars), + sampled_from(types), + sampled_from(types)) +def test_operator_scalars(op, type1, type2): + try: + op(type1(1), type2(1)) + except TypeError: + pass + + +@pytest.mark.parametrize("op", reasonable_operators_for_scalars) +def test_longdouble_inf_loop(op): + try: + op(np.longdouble(3), None) + except TypeError: + pass + try: + op(None, np.longdouble(3)) + except TypeError: + pass + + +@pytest.mark.parametrize("op", reasonable_operators_for_scalars) +def test_clongdouble_inf_loop(op): + if op in {operator.mod} and False: + pytest.xfail("The modulo operator is known to be broken") + try: + op(np.clongdouble(3), None) + except TypeError: + pass + try: + op(None, np.longdouble(3)) + except TypeError: + pass diff --git a/numpy/core/tests/test_scalarprint.py b/numpy/core/tests/test_scalarprint.py index 6502ec4c1486..620472683391 100644 --- a/numpy/core/tests/test_scalarprint.py +++ b/numpy/core/tests/test_scalarprint.py @@ -9,7 +9,7 @@ from tempfile import TemporaryFile import numpy as np -from numpy.testing import assert_, assert_equal +from numpy.testing import assert_, assert_equal, assert_raises class TestRealScalars: def test_str(self): @@ -176,7 +176,8 @@ def test_dragon4(self): "87538682506419718265533447265625") # largest numbers - assert_equal(fpos32(np.finfo(np.float32).max, **preckwd(0)), + f32x = np.finfo(np.float32).max + assert_equal(fpos32(f32x, **preckwd(0)), "340282346638528859811704183484516925440.") assert_equal(fpos64(np.finfo(np.float64).max, **preckwd(0)), "1797693134862315708145274237317043567980705675258449965989" @@ -186,10 +187,66 @@ def test_dragon4(self): "6580855933212334827479782620414472316873817718091929988125" "0404026184124858368.") # Warning: In unique mode only the integer digits necessary for - # uniqueness are computed, the rest are 0. Should we change this? - assert_equal(fpos32(np.finfo(np.float32).max, precision=0), + # uniqueness are computed, the rest are 0. + assert_equal(fpos32(f32x), "340282350000000000000000000000000000000.") + # Further tests of zero-padding vs rounding in different combinations + # of unique, fractional, precision, min_digits + # precision can only reduce digits, not add them. + # min_digits can only extend digits, not reduce them. + assert_equal(fpos32(f32x, unique=True, fractional=True, precision=0), + "340282350000000000000000000000000000000.") + assert_equal(fpos32(f32x, unique=True, fractional=True, precision=4), + "340282350000000000000000000000000000000.") + assert_equal(fpos32(f32x, unique=True, fractional=True, min_digits=0), + "340282346638528859811704183484516925440.") + assert_equal(fpos32(f32x, unique=True, fractional=True, min_digits=4), + "340282346638528859811704183484516925440.0000") + assert_equal(fpos32(f32x, unique=True, fractional=True, + min_digits=4, precision=4), + "340282346638528859811704183484516925440.0000") + assert_raises(ValueError, fpos32, f32x, unique=True, fractional=False, + precision=0) + assert_equal(fpos32(f32x, unique=True, fractional=False, precision=4), + "340300000000000000000000000000000000000.") + assert_equal(fpos32(f32x, unique=True, fractional=False, precision=20), + "340282350000000000000000000000000000000.") + assert_equal(fpos32(f32x, unique=True, fractional=False, min_digits=4), + "340282350000000000000000000000000000000.") + assert_equal(fpos32(f32x, unique=True, fractional=False, + min_digits=20), + "340282346638528859810000000000000000000.") + assert_equal(fpos32(f32x, unique=True, fractional=False, + min_digits=15), + "340282346638529000000000000000000000000.") + assert_equal(fpos32(f32x, unique=False, fractional=False, precision=4), + "340300000000000000000000000000000000000.") + # test that unique rounding is preserved when precision is supplied + # but no extra digits need to be printed (gh-18609) + a = np.float64.fromhex('-1p-97') + assert_equal(fsci64(a, unique=True), '-6.310887241768095e-30') + assert_equal(fsci64(a, unique=False, precision=15), + '-6.310887241768094e-30') + assert_equal(fsci64(a, unique=True, precision=15), + '-6.310887241768095e-30') + assert_equal(fsci64(a, unique=True, min_digits=15), + '-6.310887241768095e-30') + assert_equal(fsci64(a, unique=True, precision=15, min_digits=15), + '-6.310887241768095e-30') + # adds/remove digits in unique mode with unbiased rnding + assert_equal(fsci64(a, unique=True, precision=14), + '-6.31088724176809e-30') + assert_equal(fsci64(a, unique=True, min_digits=16), + '-6.3108872417680944e-30') + assert_equal(fsci64(a, unique=True, precision=16), + '-6.310887241768095e-30') + assert_equal(fsci64(a, unique=True, min_digits=14), + '-6.310887241768095e-30') + # test min_digits in unique mode with different rounding cases + assert_equal(fsci64('1e120', min_digits=3), '1.000e+120') + assert_equal(fsci64('1e100', min_digits=3), '1.000e+100') + # test trailing zeros assert_equal(fpos32('1.0', unique=False, precision=3), "1.000") assert_equal(fpos64('1.0', unique=False, precision=3), "1.000") @@ -200,7 +257,9 @@ def test_dragon4(self): assert_equal(fsci32('1.5', unique=False, precision=3), "1.500e+00") assert_equal(fsci64('1.5', unique=False, precision=3), "1.500e+00") # gh-10713 - assert_equal(fpos64('324', unique=False, precision=5, fractional=False), "324.00") + assert_equal(fpos64('324', unique=False, precision=5, + fractional=False), "324.00") + def test_dragon4_interface(self): tps = [np.float16, np.float32, np.float64] diff --git a/numpy/core/tests/test_shape_base.py b/numpy/core/tests/test_shape_base.py index 4e56ace90fb6..679e3c036351 100644 --- a/numpy/core/tests/test_shape_base.py +++ b/numpy/core/tests/test_shape_base.py @@ -256,7 +256,7 @@ def test_concatenate_axis_None(self): r = np.concatenate((a, b), axis=None) assert_equal(r.size, a.size + len(b)) assert_equal(r.dtype, a.dtype) - r = np.concatenate((a, b, c), axis=None) + r = np.concatenate((a, b, c), axis=None, dtype="U") d = array(['0.0', '1.0', '2.0', '3.0', '0', '1', '2', 'x']) assert_array_equal(r, d) @@ -343,7 +343,7 @@ def test_bad_out_shape(self): concatenate((a, b), out=np.empty(4)) @pytest.mark.parametrize("axis", [None, 0]) - @pytest.mark.parametrize("out_dtype", ["c8", "f4", "f8", ">f8", "i8"]) + @pytest.mark.parametrize("out_dtype", ["c8", "f4", "f8", ">f8", "i8", "S4"]) @pytest.mark.parametrize("casting", ['no', 'equiv', 'safe', 'same_kind', 'unsafe']) def test_out_and_dtype(self, axis, out_dtype, casting): @@ -369,6 +369,29 @@ def test_out_and_dtype(self, axis, out_dtype, casting): with assert_raises(TypeError): concatenate(to_concat, out=out, dtype=out_dtype, axis=axis) + @pytest.mark.parametrize("axis", [None, 0]) + @pytest.mark.parametrize("string_dt", ["S", "U", "S0", "U0"]) + @pytest.mark.parametrize("arrs", + [([0.],), ([0.], [1]), ([0], ["string"], [1.])]) + def test_dtype_with_promotion(self, arrs, string_dt, axis): + # Note that U0 and S0 should be deprecated eventually and changed to + # actually give the empty string result (together with `np.array`) + res = np.concatenate(arrs, axis=axis, dtype=string_dt, casting="unsafe") + # The actual dtype should be identical to a cast (of a double array): + assert res.dtype == np.array(1.).astype(string_dt).dtype + + @pytest.mark.parametrize("axis", [None, 0]) + def test_string_dtype_does_not_inspect(self, axis): + with pytest.raises(TypeError): + np.concatenate(([None], [1]), dtype="S", axis=axis) + with pytest.raises(TypeError): + np.concatenate(([None], [1]), dtype="U", axis=axis) + + @pytest.mark.parametrize("axis", [None, 0]) + def test_subarray_error(self, axis): + with pytest.raises(TypeError, match=".*subarray dtype"): + np.concatenate(([1], [1]), dtype="(2,)i", axis=axis) + def test_stack(): # non-iterable input diff --git a/numpy/core/tests/test_simd.py b/numpy/core/tests/test_simd.py index 13e8d5ede460..ea5bbe103900 100644 --- a/numpy/core/tests/test_simd.py +++ b/numpy/core/tests/test_simd.py @@ -1,13 +1,17 @@ # NOTE: Please avoid the use of numpy.testing since NPYV intrinsics # may be involved in their functionality. -import pytest, math +import pytest, math, re +import itertools from numpy.core._simd import targets +from numpy.core._multiarray_umath import __cpu_baseline__ class _Test_Utility: - # submodule of the desired SIMD extention, e.g. targets["AVX512F"] + # submodule of the desired SIMD extension, e.g. targets["AVX512F"] npyv = None # the current data type suffix e.g. 's8' sfx = None + # target name can be 'baseline' or one or more of CPU features + target_name = None def __getattr__(self, attr): """ @@ -92,6 +96,66 @@ def _nan(self): v = self.npyv.setall_u32(0x7fc00000) return self.npyv.reinterpret_f32_u32(v)[0] + def _cpu_features(self): + target = self.target_name + if target == "baseline": + target = __cpu_baseline__ + else: + target = target.split('__') # multi-target separator + return ' '.join(target) + +class _SIMD_BOOL(_Test_Utility): + """ + To test all boolean vector types at once + """ + def _data(self, start=None, count=None, reverse=False): + nlanes = getattr(self.npyv, "nlanes_u" + self.sfx[1:]) + true_mask = self._true_mask() + rng = range(nlanes) + if reverse: + rng = reversed(rng) + return [true_mask if x % 2 else 0 for x in rng] + + def _load_b(self, data): + len_str = self.sfx[1:] + load = getattr(self.npyv, "load_u" + len_str) + cvt = getattr(self.npyv, f"cvt_b{len_str}_u{len_str}") + return cvt(load(data)) + + def test_operators_logical(self): + """ + Logical operations for boolean types. + Test intrinsics: + npyv_xor_##SFX, npyv_and_##SFX, npyv_or_##SFX, npyv_not_##SFX + """ + data_a = self._data() + data_b = self._data(reverse=True) + vdata_a = self._load_b(data_a) + vdata_b = self._load_b(data_b) + + data_and = [a & b for a, b in zip(data_a, data_b)] + vand = getattr(self, "and")(vdata_a, vdata_b) + assert vand == data_and + + data_or = [a | b for a, b in zip(data_a, data_b)] + vor = getattr(self, "or")(vdata_a, vdata_b) + assert vor == data_or + + data_xor = [a ^ b for a, b in zip(data_a, data_b)] + vxor = getattr(self, "xor")(vdata_a, vdata_b) + assert vxor == data_xor + + vnot = getattr(self, "not")(vdata_a) + assert vnot == data_b + + def test_tobits(self): + data2bits = lambda data: sum([int(x != 0) << i for i, x in enumerate(data, 0)]) + for data in (self._data(), self._data(reverse=True)): + vdata = self._load_b(data) + data_bits = data2bits(data) + tobits = bin(self.tobits(vdata)) + assert tobits == bin(data_bits) + class _SIMD_INT(_Test_Utility): """ To test all integer vector types at once @@ -110,14 +174,21 @@ def test_operators_shift(self): # left shift shl = self.shl(vdata_a, count) assert shl == data_shl_a - # left shift by an immediate constant - shli = self.shli(vdata_a, count) - assert shli == data_shl_a # load to cast data_shr_a = self.load([a >> count for a in data_a]) # right shift shr = self.shr(vdata_a, count) assert shr == data_shr_a + + # shift by zero or max or out-range immediate constant is not applicable and illogical + for count in range(1, self._scalar_size()): + # load to cast + data_shl_a = self.load([a << count for a in data_a]) + # left shift by an immediate constant + shli = self.shli(vdata_a, count) + assert shli == data_shl_a + # load to cast + data_shr_a = self.load([a >> count for a in data_a]) # right shift by an immediate constant shri = self.shri(vdata_a, count) assert shri == data_shr_a @@ -138,6 +209,59 @@ def test_arithmetic_subadd_saturated(self): subs = self.subs(vdata_a, vdata_b) assert subs == data_subs + def test_math_max_min(self): + data_a = self._data() + data_b = self._data(self.nlanes) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + + data_max = [max(a, b) for a, b in zip(data_a, data_b)] + simd_max = self.max(vdata_a, vdata_b) + assert simd_max == data_max + + data_min = [min(a, b) for a, b in zip(data_a, data_b)] + simd_min = self.min(vdata_a, vdata_b) + assert simd_min == data_min + +class _SIMD_FP32(_Test_Utility): + """ + To only test single precision + """ + def test_conversions(self): + """ + Round to nearest even integer, assume CPU control register is set to rounding. + Test intrinsics: + npyv_round_s32_##SFX + """ + features = self._cpu_features() + if not self.npyv.simd_f64 and re.match(r".*(NEON|ASIMD)", features): + # very costly to emulate nearest even on Armv7 + # instead we round halves to up. e.g. 0.5 -> 1, -0.5 -> -1 + _round = lambda v: int(v + (0.5 if v >= 0 else -0.5)) + else: + _round = round + vdata_a = self.load(self._data()) + vdata_a = self.sub(vdata_a, self.setall(0.5)) + data_round = [_round(x) for x in vdata_a] + vround = self.round_s32(vdata_a) + assert vround == data_round + +class _SIMD_FP64(_Test_Utility): + """ + To only test double precision + """ + def test_conversions(self): + """ + Round to nearest even integer, assume CPU control register is set to rounding. + Test intrinsics: + npyv_round_s32_##SFX + """ + vdata_a = self.load(self._data()) + vdata_a = self.sub(vdata_a, self.setall(0.5)) + vdata_b = self.mul(vdata_a, self.setall(-1.5)) + data_round = [round(x) for x in list(vdata_a) + list(vdata_b)] + vround = self.round_s32(vdata_a, vdata_b) + assert vround == data_round + class _SIMD_FP(_Test_Utility): """ To test all float vector types at once @@ -205,6 +329,68 @@ def test_square(self): data_square = [x*x for x in data] square = self.square(vdata) assert square == data_square + + def test_max(self): + """ + Test intrinsics: + npyv_max_##SFX + npyv_maxp_##SFX + """ + data_a = self._data() + data_b = self._data(self.nlanes) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + data_max = [max(a, b) for a, b in zip(data_a, data_b)] + _max = self.max(vdata_a, vdata_b) + assert _max == data_max + maxp = self.maxp(vdata_a, vdata_b) + assert maxp == data_max + # test IEEE standards + pinf, ninf, nan = self._pinfinity(), self._ninfinity(), self._nan() + max_cases = ((nan, nan, nan), (nan, 10, 10), (10, nan, 10), + (pinf, pinf, pinf), (pinf, 10, pinf), (10, pinf, pinf), + (ninf, ninf, ninf), (ninf, 10, 10), (10, ninf, 10), + (10, 0, 10), (10, -10, 10)) + for case_operand1, case_operand2, desired in max_cases: + data_max = [desired]*self.nlanes + vdata_a = self.setall(case_operand1) + vdata_b = self.setall(case_operand2) + maxp = self.maxp(vdata_a, vdata_b) + assert maxp == pytest.approx(data_max, nan_ok=True) + if nan in (case_operand1, case_operand2, desired): + continue + _max = self.max(vdata_a, vdata_b) + assert _max == data_max + + def test_min(self): + """ + Test intrinsics: + npyv_min_##SFX + npyv_minp_##SFX + """ + data_a = self._data() + data_b = self._data(self.nlanes) + vdata_a, vdata_b = self.load(data_a), self.load(data_b) + data_min = [min(a, b) for a, b in zip(data_a, data_b)] + _min = self.min(vdata_a, vdata_b) + assert _min == data_min + minp = self.minp(vdata_a, vdata_b) + assert minp == data_min + # test IEEE standards + pinf, ninf, nan = self._pinfinity(), self._ninfinity(), self._nan() + min_cases = ((nan, nan, nan), (nan, 10, 10), (10, nan, 10), + (pinf, pinf, pinf), (pinf, 10, 10), (10, pinf, 10), + (ninf, ninf, ninf), (ninf, 10, ninf), (10, ninf, ninf), + (10, 0, 0), (10, -10, -10)) + for case_operand1, case_operand2, desired in min_cases: + data_min = [desired]*self.nlanes + vdata_a = self.setall(case_operand1) + vdata_b = self.setall(case_operand2) + minp = self.minp(vdata_a, vdata_b) + assert minp == pytest.approx(data_min, nan_ok=True) + if nan in (case_operand1, case_operand2, desired): + continue + _min = self.min(vdata_a, vdata_b) + assert _min == data_min def test_reciprocal(self): pinf, ninf, nan = self._pinfinity(), self._ninfinity(), self._nan() @@ -221,6 +407,14 @@ def test_reciprocal(self): recip = self.recip(vdata) assert recip == data_recip + def test_special_cases(self): + """ + Compare Not NaN. Test intrinsics: + npyv_notnan_##SFX + """ + nnan = self.notnan(self.setall(self._nan())) + assert nnan == [0]*self.nlanes + class _SIMD_ALL(_Test_Utility): """ To test all vector types at once @@ -414,7 +608,7 @@ def test_misc(self): vsetf = self.setf(10, *data_a) assert vsetf == data_a - # We're testing the sainty of _simd's type-vector, + # We're testing the sanity of _simd's type-vector, # reinterpret* intrinsics itself are tested via compiler # during the build of _simd module sfxes = ["u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64", "f32"] @@ -459,6 +653,18 @@ def test_reorder(self): vzip = self.zip(vdata_a, vdata_b) assert vzip == (data_zipl, data_ziph) + def test_reorder_rev64(self): + # Reverse elements of each 64-bit lane + ssize = self._scalar_size() + if ssize == 64: + return + data_rev64 = [ + y for x in range(0, self.nlanes, 64//ssize) + for y in reversed(range(x, x + 64//ssize)) + ] + rev64 = self.rev64(self.load(range(self.nlanes))) + assert rev64 == data_rev64 + def test_operators_comparison(self): if self._is_fp(): data_a = self._data() @@ -540,6 +746,26 @@ def test_conversion_boolean(self): true_vsfx = from_boolean(true_vb) assert false_vsfx != true_vsfx + def test_conversion_expand(self): + """ + Test expand intrinsics: + npyv_expand_u16_u8 + npyv_expand_u32_u16 + """ + if self.sfx not in ("u8", "u16"): + return + totype = self.sfx[0]+str(int(self.sfx[1:])*2) + expand = getattr(self.npyv, f"expand_{totype}_{self.sfx}") + # close enough from the edge to detect any deviation + data = self._data(self._int_max() - self.nlanes) + vdata = self.load(data) + edata = expand(vdata) + # lower half part + data_lo = data[:self.nlanes//2] + # higher half part + data_hi = data[self.nlanes//2:] + assert edata == (data_lo, data_hi) + def test_arithmetic_subadd(self): if self._is_fp(): data_a = self._data() @@ -583,8 +809,84 @@ def test_arithmetic_div(self): div = self.div(vdata_a, vdata_b) assert div == data_div + def test_arithmetic_intdiv(self): + """ + Test integer division intrinsics: + npyv_divisor_##sfx + npyv_divc_##sfx + """ + if self._is_fp(): + return + + def trunc_div(a, d): + """ + Divide towards zero works with large integers > 2^53, + and wrap around overflow similar to what C does. + """ + if d == -1 and a == int_min: + return a + sign_a, sign_d = a < 0, d < 0 + if a == 0 or sign_a == sign_d: + return a // d + return (a + sign_d - sign_a) // d + 1 + + int_min = self._int_min() if self._is_signed() else 1 + int_max = self._int_max() + rdata = ( + 0, 1, self.nlanes, int_max-self.nlanes, + int_min, int_min//2 + 1 + ) + divisors = (1, 2, 9, 13, self.nlanes, int_min, int_max, int_max//2) + + for x, d in itertools.product(rdata, divisors): + data = self._data(x) + vdata = self.load(data) + data_divc = [trunc_div(a, d) for a in data] + divisor = self.divisor(d) + divc = self.divc(vdata, divisor) + assert divc == data_divc + + if not self._is_signed(): + return + + safe_neg = lambda x: -x-1 if -x > int_max else -x + # test round divison for signed integers + for x, d in itertools.product(rdata, divisors): + d_neg = safe_neg(d) + data = self._data(x) + data_neg = [safe_neg(a) for a in data] + vdata = self.load(data) + vdata_neg = self.load(data_neg) + divisor = self.divisor(d) + divisor_neg = self.divisor(d_neg) + + # round towards zero + data_divc = [trunc_div(a, d_neg) for a in data] + divc = self.divc(vdata, divisor_neg) + assert divc == data_divc + data_divc = [trunc_div(a, d) for a in data_neg] + divc = self.divc(vdata_neg, divisor) + assert divc == data_divc + + # test truncate sign if the dividend is zero + vzero = self.zero() + for d in (-1, -10, -100, int_min//2, int_min): + divisor = self.divisor(d) + divc = self.divc(vzero, divisor) + assert divc == vzero + + # test overflow + vmin = self.setall(int_min) + divisor = self.divisor(-1) + divc = self.divc(vmin, divisor) + assert divc == vmin + def test_arithmetic_reduce_sum(self): - if not self._is_fp(): + """ + Test reduce sum intrinsics: + npyv_sum_##sfx + """ + if self.sfx not in ("u32", "u64", "f32", "f64"): return # reduce sum data = self._data() @@ -594,12 +896,54 @@ def test_arithmetic_reduce_sum(self): vsum = self.sum(vdata) assert vsum == data_sum + def test_arithmetic_reduce_sumup(self): + """ + Test extend reduce sum intrinsics: + npyv_sumup_##sfx + """ + if self.sfx not in ("u8", "u16"): + return + rdata = (0, self.nlanes, self._int_min(), self._int_max()-self.nlanes) + for r in rdata: + data = self._data(r) + vdata = self.load(data) + data_sum = sum(data) + vsum = self.sumup(vdata) + assert vsum == data_sum + + def test_mask_conditional(self): + """ + Conditional addition and subtraction for all supported data types. + Test intrinsics: + npyv_ifadd_##SFX, npyv_ifsub_##SFX + """ + vdata_a = self.load(self._data()) + vdata_b = self.load(self._data(reverse=True)) + true_mask = self.cmpeq(self.zero(), self.zero()) + false_mask = self.cmpneq(self.zero(), self.zero()) + + data_sub = self.sub(vdata_b, vdata_a) + ifsub = self.ifsub(true_mask, vdata_b, vdata_a, vdata_b) + assert ifsub == data_sub + ifsub = self.ifsub(false_mask, vdata_a, vdata_b, vdata_b) + assert ifsub == vdata_b + + data_add = self.add(vdata_b, vdata_a) + ifadd = self.ifadd(true_mask, vdata_b, vdata_a, vdata_b) + assert ifadd == data_add + ifadd = self.ifadd(false_mask, vdata_a, vdata_b, vdata_b) + assert ifadd == vdata_b + +bool_sfx = ("b8", "b16", "b32", "b64") int_sfx = ("u8", "s8", "u16", "s16", "u32", "s32", "u64", "s64") fp_sfx = ("f32", "f64") all_sfx = int_sfx + fp_sfx tests_registry = { + bool_sfx: _SIMD_BOOL, int_sfx : _SIMD_INT, fp_sfx : _SIMD_FP, + ("f32",): _SIMD_FP32, + ("f64",): _SIMD_FP64, all_sfx : _SIMD_ALL } for target_name, npyv in targets.items(): @@ -624,7 +968,7 @@ def test_arithmetic_reduce_sum(self): for sfx in sfxes: skip_m = skip_sfx.get(sfx, skip) inhr = (cls,) - attr = dict(npyv=targets[target_name], sfx=sfx) + attr = dict(npyv=targets[target_name], sfx=sfx, target_name=target_name) tcls = type(f"Test{cls.__name__}_{simd_width}_{target_name}_{sfx}", inhr, attr) if skip_m: pytest.mark.skip(reason=skip_m)(tcls) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index aa17d6b089a7..a47f1df49871 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -34,13 +34,13 @@ def test_kwarg_exact(self): assert_raises(TypeError, np.add, 1, 2, wherex=[True]) def test_sig_signature(self): - assert_raises(ValueError, np.add, 1, 2, sig='ii->i', + assert_raises(TypeError, np.add, 1, 2, sig='ii->i', signature='ii->i') def test_sig_dtype(self): - assert_raises(RuntimeError, np.add, 1, 2, sig='ii->i', + assert_raises(TypeError, np.add, 1, 2, sig='ii->i', dtype=int) - assert_raises(RuntimeError, np.add, 1, 2, signature='ii->i', + assert_raises(TypeError, np.add, 1, 2, signature='ii->i', dtype=int) def test_extobj_refcount(self): @@ -175,7 +175,7 @@ def __getattr__(self, attr): ufunc(obj_arr) else: res_obj = ufunc(obj_arr) - assert_array_equal(res_num.astype("O"), res_obj) + assert_array_almost_equal(res_num.astype("O"), res_obj) def _pickleable_module_global(): @@ -410,9 +410,12 @@ def test_get_signature(self): def test_forced_sig(self): a = 0.5*np.arange(3, dtype='f8') assert_equal(np.add(a, 0.5), [0.5, 1, 1.5]) - assert_equal(np.add(a, 0.5, sig='i', casting='unsafe'), [0, 0, 1]) + with pytest.warns(DeprecationWarning): + assert_equal(np.add(a, 0.5, sig='i', casting='unsafe'), [0, 0, 1]) assert_equal(np.add(a, 0.5, sig='ii->i', casting='unsafe'), [0, 0, 1]) - assert_equal(np.add(a, 0.5, sig=('i4',), casting='unsafe'), [0, 0, 1]) + with pytest.warns(DeprecationWarning): + assert_equal(np.add(a, 0.5, sig=('i4',), casting='unsafe'), + [0, 0, 1]) assert_equal(np.add(a, 0.5, sig=('i4', 'i4', 'i4'), casting='unsafe'), [0, 0, 1]) @@ -420,18 +423,121 @@ def test_forced_sig(self): np.add(a, 0.5, out=b) assert_equal(b, [0.5, 1, 1.5]) b[:] = 0 - np.add(a, 0.5, sig='i', out=b, casting='unsafe') + with pytest.warns(DeprecationWarning): + np.add(a, 0.5, sig='i', out=b, casting='unsafe') assert_equal(b, [0, 0, 1]) b[:] = 0 np.add(a, 0.5, sig='ii->i', out=b, casting='unsafe') assert_equal(b, [0, 0, 1]) b[:] = 0 - np.add(a, 0.5, sig=('i4',), out=b, casting='unsafe') + with pytest.warns(DeprecationWarning): + np.add(a, 0.5, sig=('i4',), out=b, casting='unsafe') assert_equal(b, [0, 0, 1]) b[:] = 0 np.add(a, 0.5, sig=('i4', 'i4', 'i4'), out=b, casting='unsafe') assert_equal(b, [0, 0, 1]) + def test_signature_all_None(self): + # signature all None, is an acceptable alternative (since 1.21) + # to not providing a signature. + res1 = np.add([3], [4], sig=(None, None, None)) + res2 = np.add([3], [4]) + assert_array_equal(res1, res2) + res1 = np.maximum([3], [4], sig=(None, None, None)) + res2 = np.maximum([3], [4]) + assert_array_equal(res1, res2) + + with pytest.raises(TypeError): + # special case, that would be deprecated anyway, so errors: + np.add(3, 4, signature=(None,)) + + def test_signature_dtype_type(self): + # Since that will be the normal behaviour (past NumPy 1.21) + # we do support the types already: + float_dtype = type(np.dtype(np.float64)) + np.add(3, 4, signature=(float_dtype, float_dtype, None)) + + @pytest.mark.parametrize("casting", ["unsafe", "same_kind", "safe"]) + def test_partial_signature_mismatch(self, casting): + # If the second argument matches already, no need to specify it: + res = np.ldexp(np.float32(1.), np.int_(2), dtype="d") + assert res.dtype == "d" + res = np.ldexp(np.float32(1.), np.int_(2), signature=(None, None, "d")) + assert res.dtype == "d" + + # ldexp only has a loop for long input as second argument, overriding + # the output cannot help with that (no matter the casting) + with pytest.raises(TypeError): + np.ldexp(1., np.uint64(3), dtype="d") + with pytest.raises(TypeError): + np.ldexp(1., np.uint64(3), signature=(None, None, "d")) + + def test_use_output_signature_for_all_arguments(self): + # Test that providing only `dtype=` or `signature=(None, None, dtype)` + # is sufficient if falling back to a homogeneous signature works. + # In this case, the `intp, intp -> intp` loop is chosen. + res = np.power(1.5, 2.8, dtype=np.intp, casting="unsafe") + assert res == 1 # the cast happens first. + res = np.power(1.5, 2.8, signature=(None, None, np.intp), + casting="unsafe") + assert res == 1 + with pytest.raises(TypeError): + # the unsafe casting would normally cause errors though: + np.power(1.5, 2.8, dtype=np.intp) + + def test_signature_errors(self): + with pytest.raises(TypeError, + match="the signature object to ufunc must be a string or"): + np.add(3, 4, signature=123.) # neither a string nor a tuple + + with pytest.raises(ValueError): + # bad symbols that do not translate to dtypes + np.add(3, 4, signature="%^->#") + + with pytest.raises(ValueError): + np.add(3, 4, signature=b"ii-i") # incomplete and byte string + + with pytest.raises(ValueError): + np.add(3, 4, signature="ii>i") # incomplete string + + with pytest.raises(ValueError): + np.add(3, 4, signature=(None, "f8")) # bad length + + with pytest.raises(UnicodeDecodeError): + np.add(3, 4, signature=b"\xff\xff->i") + + def test_forced_dtype_times(self): + # Signatures only set the type numbers (not the actual loop dtypes) + # so using `M` in a signature/dtype should generally work: + a = np.array(['2010-01-02', '1999-03-14', '1833-03'], dtype='>M8[D]') + np.maximum(a, a, dtype="M") + np.maximum.reduce(a, dtype="M") + + arr = np.arange(10, dtype="m8[s]") + np.add(arr, arr, dtype="m") + np.maximum(arr, arr, dtype="m") + + def test_forced_dtype_warning(self): + # does not warn (test relies on bad pickling behaviour, simply remove + # it if the `assert int64 is not int64_2` should start failing. + int64 = np.dtype("int64") + int64_2 = pickle.loads(pickle.dumps(int64)) + assert int64 is not int64_2 + np.add(3, 4, dtype=int64_2) + + arr = np.arange(10, dtype="m8[s]") + msg = "The `dtype` and `signature` arguments to ufuncs only select the" + with pytest.raises(TypeError, match=msg): + np.add(3, 5, dtype=int64.newbyteorder()) + with pytest.raises(TypeError, match=msg): + np.add(3, 5, dtype="m8[ns]") # previously used the "ns" + with pytest.raises(TypeError, match=msg): + np.add(arr, arr, dtype="m8[ns]") # never preserved the "ns" + with pytest.raises(TypeError, match=msg): + np.maximum(arr, arr, dtype="m8[ns]") # previously used the "ns" + with pytest.raises(TypeError, match=msg): + np.maximum.reduce(arr, dtype="m8[ns]") # never preserved the "ns" + def test_true_divide(self): a = np.array(10) b = np.array(20) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 3f89cc59bb9f..2378b11e95a4 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -4,7 +4,9 @@ import itertools import pytest import sys +import os from fractions import Fraction +from functools import reduce import numpy.core.umath as ncu from numpy.core import _umath_tests as ncu_tests @@ -16,6 +18,20 @@ _gen_alignment_data, assert_array_almost_equal_nulp, assert_warns ) +def get_glibc_version(): + try: + ver = os.confstr('CS_GNU_LIBC_VERSION').rsplit(' ')[1] + except Exception as inst: + ver = '0.0' + + return ver + + +glibcver = get_glibc_version() +glibc_newerthan_2_17 = pytest.mark.xfail( + glibcver != '0.0' and glibcver < '2.17', + reason="Older glibc versions may not raise appropriate FP exceptions") + def on_powerpc(): """ True if we are running on a Power PC platform.""" return platform.processor() == 'powerpc' or \ @@ -99,9 +115,9 @@ def test_out_subok(self): # Out argument must be tuple, since there are multiple outputs. r1, r2 = np.frexp(d, out=o1, subok=subok) - assert_raises(ValueError, np.add, a, 2, o, o, subok=subok) - assert_raises(ValueError, np.add, a, 2, o, out=o, subok=subok) - assert_raises(ValueError, np.add, a, 2, None, out=o, subok=subok) + assert_raises(TypeError, np.add, a, 2, o, o, subok=subok) + assert_raises(TypeError, np.add, a, 2, o, out=o, subok=subok) + assert_raises(TypeError, np.add, a, 2, None, out=o, subok=subok) assert_raises(ValueError, np.add, a, 2, out=(o, o), subok=subok) assert_raises(ValueError, np.add, a, 2, out=(), subok=subok) assert_raises(TypeError, np.add, a, 2, [], subok=subok) @@ -249,6 +265,148 @@ def test_division_int(self): assert_equal(x // 100, [0, 0, 0, 1, -1, -1, -1, -1, -2]) assert_equal(x % 100, [5, 10, 90, 0, 95, 90, 10, 0, 80]) + @pytest.mark.parametrize("dtype,ex_val", itertools.product( + np.sctypes['int'] + np.sctypes['uint'], ( + ( + # dividend + "np.arange(fo.max-lsize, fo.max, dtype=dtype)," + # divisors + "np.arange(lsize, dtype=dtype)," + # scalar divisors + "range(15)" + ), + ( + # dividend + "np.arange(fo.min, fo.min+lsize, dtype=dtype)," + # divisors + "np.arange(lsize//-2, lsize//2, dtype=dtype)," + # scalar divisors + "range(fo.min, fo.min + 15)" + ), ( + # dividend + "np.arange(fo.max-lsize, fo.max, dtype=dtype)," + # divisors + "np.arange(lsize, dtype=dtype)," + # scalar divisors + "[1,3,9,13,neg, fo.min+1, fo.min//2, fo.max//3, fo.max//4]" + ) + ) + )) + def test_division_int_boundary(self, dtype, ex_val): + fo = np.iinfo(dtype) + neg = -1 if fo.min < 0 else 1 + # Large enough to test SIMD loops and remaind elements + lsize = 512 + 7 + a, b, divisors = eval(ex_val) + a_lst, b_lst = a.tolist(), b.tolist() + + c_div = lambda n, d: ( + 0 if d == 0 or (n and n == fo.min and d == -1) else n//d + ) + with np.errstate(divide='ignore'): + ac = a.copy() + ac //= b + div_ab = a // b + div_lst = [c_div(x, y) for x, y in zip(a_lst, b_lst)] + + msg = "Integer arrays floor division check (//)" + assert all(div_ab == div_lst), msg + msg_eq = "Integer arrays floor division check (//=)" + assert all(ac == div_lst), msg_eq + + for divisor in divisors: + ac = a.copy() + with np.errstate(divide='ignore'): + div_a = a // divisor + ac //= divisor + div_lst = [c_div(i, divisor) for i in a_lst] + + assert all(div_a == div_lst), msg + assert all(ac == div_lst), msg_eq + + with np.errstate(divide='raise'): + if 0 in b or (fo.min and -1 in b and fo.min in a): + # Verify overflow case + with pytest.raises(FloatingPointError): + a // b + else: + a // b + if fo.min and fo.min in a: + with pytest.raises(FloatingPointError): + a // -1 + elif fo.min: + a // -1 + with pytest.raises(FloatingPointError): + a // 0 + with pytest.raises(FloatingPointError): + ac = a.copy() + ac //= 0 + + np.array([], dtype=dtype) // 0 + + @pytest.mark.parametrize("dtype,ex_val", itertools.product( + np.sctypes['int'] + np.sctypes['uint'], ( + "np.array([fo.max, 1, 2, 1, 1, 2, 3], dtype=dtype)", + "np.array([fo.min, 1, -2, 1, 1, 2, -3], dtype=dtype)", + "np.arange(fo.min, fo.min+(100*10), 10, dtype=dtype)", + "np.arange(fo.max-(100*7), fo.max, 7, dtype=dtype)", + ) + )) + def test_division_int_reduce(self, dtype, ex_val): + fo = np.iinfo(dtype) + a = eval(ex_val) + lst = a.tolist() + c_div = lambda n, d: ( + 0 if d == 0 or (n and n == fo.min and d == -1) else n//d + ) + + with np.errstate(divide='ignore'): + div_a = np.floor_divide.reduce(a) + div_lst = reduce(c_div, lst) + msg = "Reduce floor integer division check" + assert div_a == div_lst, msg + + with np.errstate(divide='raise'): + with pytest.raises(FloatingPointError): + np.floor_divide.reduce(np.arange(-100, 100, dtype=dtype)) + if fo.min: + with pytest.raises(FloatingPointError): + np.floor_divide.reduce( + np.array([fo.min, 1, -1], dtype=dtype) + ) + + @pytest.mark.parametrize( + "dividend,divisor,quotient", + [(np.timedelta64(2,'Y'), np.timedelta64(2,'M'), 12), + (np.timedelta64(2,'Y'), np.timedelta64(-2,'M'), -12), + (np.timedelta64(-2,'Y'), np.timedelta64(2,'M'), -12), + (np.timedelta64(-2,'Y'), np.timedelta64(-2,'M'), 12), + (np.timedelta64(2,'M'), np.timedelta64(-2,'Y'), -1), + (np.timedelta64(2,'Y'), np.timedelta64(0,'M'), 0), + (np.timedelta64(2,'Y'), 2, np.timedelta64(1,'Y')), + (np.timedelta64(2,'Y'), -2, np.timedelta64(-1,'Y')), + (np.timedelta64(-2,'Y'), 2, np.timedelta64(-1,'Y')), + (np.timedelta64(-2,'Y'), -2, np.timedelta64(1,'Y')), + (np.timedelta64(-2,'Y'), -2, np.timedelta64(1,'Y')), + (np.timedelta64(-2,'Y'), -3, np.timedelta64(0,'Y')), + (np.timedelta64(-2,'Y'), 0, np.timedelta64('Nat','Y')), + ]) + def test_division_int_timedelta(self, dividend, divisor, quotient): + # If either divisor is 0 or quotient is Nat, check for division by 0 + if divisor and (isinstance(quotient, int) or not np.isnat(quotient)): + msg = "Timedelta floor division check" + assert dividend // divisor == quotient, msg + + # Test for arrays as well + msg = "Timedelta arrays floor division check" + dividend_array = np.array([dividend]*5) + quotient_array = np.array([quotient]*5) + assert all(dividend_array // divisor == quotient_array), msg + else: + with np.errstate(divide='raise', invalid='raise'): + with pytest.raises(FloatingPointError): + dividend // divisor + def test_division_complex(self): # check that implementation is correct msg = "Complex division implementation check" @@ -843,12 +1001,22 @@ def test_exp_values(self): yf = np.array(y, dtype=dt) assert_equal(np.exp(yf), xf) + # Older version of glibc may not raise the correct FP exceptions + # See: https://github.com/numpy/numpy/issues/19192 + @glibc_newerthan_2_17 + def test_exp_exceptions(self): with np.errstate(over='raise'): assert_raises(FloatingPointError, np.exp, np.float32(100.)) assert_raises(FloatingPointError, np.exp, np.float32(1E19)) assert_raises(FloatingPointError, np.exp, np.float64(800.)) assert_raises(FloatingPointError, np.exp, np.float64(1E19)) + with np.errstate(under='raise'): + assert_raises(FloatingPointError, np.exp, np.float32(-1000.)) + assert_raises(FloatingPointError, np.exp, np.float32(-1E19)) + assert_raises(FloatingPointError, np.exp, np.float64(-1000.)) + assert_raises(FloatingPointError, np.exp, np.float64(-1E19)) + def test_log_values(self): with np.errstate(all='ignore'): x = [np.nan, np.nan, np.inf, np.nan, -np.inf, np.nan] @@ -865,6 +1033,11 @@ def test_log_values(self): assert_raises(FloatingPointError, np.log, np.float32(-np.inf)) assert_raises(FloatingPointError, np.log, np.float32(-1.0)) + # See https://github.com/numpy/numpy/issues/18005 + with assert_no_warnings(): + a = np.array(1e9, dtype='float32') + np.log(a) + def test_sincos_values(self): with np.errstate(all='ignore'): x = [np.nan, np.nan, np.nan, np.nan] @@ -1886,7 +2059,7 @@ class StoreArrayPrepareWrap(np.ndarray): _wrap_args = None _prepare_args = None def __new__(cls): - return np.empty(()).view(cls) + return np.zeros(()).view(cls) def __array_wrap__(self, obj, context): self._wrap_args = context[1] return obj @@ -2122,7 +2295,7 @@ def __array_prepare__(self, arr, context=None): def test_array_too_many_args(self): - class A(object): + class A: def __array__(self, dtype, context): return np.zeros(1) @@ -2298,12 +2471,14 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): # __call__ a = A() - res = np.multiply.__call__(1, a, foo='bar', answer=42) + with assert_raises(TypeError): + np.multiply.__call__(1, a, foo='bar', answer=42) + res = np.multiply.__call__(1, a, subok='bar', where=42) assert_equal(res[0], a) assert_equal(res[1], np.multiply) assert_equal(res[2], '__call__') assert_equal(res[3], (1, a)) - assert_equal(res[4], {'foo': 'bar', 'answer': 42}) + assert_equal(res[4], {'subok': 'bar', 'where': 42}) # __call__, wrong args assert_raises(TypeError, np.multiply, a) @@ -2509,7 +2684,7 @@ def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): assert_raises(TypeError, np.multiply, a, b, 'one', out='two') assert_raises(TypeError, np.multiply, a, b, 'one', 'two') assert_raises(ValueError, np.multiply, a, b, out=('one', 'two')) - assert_raises(ValueError, np.multiply, a, out=()) + assert_raises(TypeError, np.multiply, a, out=()) assert_raises(TypeError, np.modf, a, 'one', out=('two', 'three')) assert_raises(TypeError, np.modf, a, 'one', 'two', 'three') assert_raises(ValueError, np.modf, a, out=('one', 'two', 'three')) @@ -2637,8 +2812,8 @@ def __array_ufunc__(self, ufunc, method, *inputs, out=None, **kwargs): if out_no: info['outputs'] = out_no - results = super(A, self).__array_ufunc__(ufunc, method, - *args, **kwargs) + results = super().__array_ufunc__(ufunc, method, + *args, **kwargs) if results is NotImplemented: return NotImplemented @@ -3137,7 +3312,7 @@ def __new__(subtype, shape): assert_equal(a+a, a) -class TestFrompyfunc(object): +class TestFrompyfunc: def test_identity(self): def mul(a, b): diff --git a/numpy/core/tests/test_umath_accuracy.py b/numpy/core/tests/test_umath_accuracy.py index 33080edbb4c3..8e04d2875bad 100644 --- a/numpy/core/tests/test_umath_accuracy.py +++ b/numpy/core/tests/test_umath_accuracy.py @@ -28,10 +28,10 @@ def convert(s, datatype="np.float32"): return fp.contents.value # dereference the pointer, get the float str_to_float = np.vectorize(convert) -files = ['umath-validation-set-exp', - 'umath-validation-set-log', - 'umath-validation-set-sin', - 'umath-validation-set-cos'] +files = ['umath-validation-set-exp.csv', + 'umath-validation-set-log.csv', + 'umath-validation-set-sin.csv', + 'umath-validation-set-cos.csv'] class TestAccuracy: @platform_skip @@ -47,7 +47,8 @@ def test_validate_transcendentals(self): names=('type','input','output','ulperr'), delimiter=',', skip_header=1) - npfunc = getattr(np, filename.split('-')[3]) + npname = path.splitext(filename)[0].split('-')[3] + npfunc = getattr(np, npname) for datatype in np.unique(data['type']): data_subset = data[data['type'] == datatype] inval = np.array(str_to_float(data_subset['input'].astype(str), data_subset['type'].astype(str)), dtype=eval(datatype)) diff --git a/numpy/core/tests/test_umath_complex.py b/numpy/core/tests/test_umath_complex.py index 90a349da1e05..c051cd61b660 100644 --- a/numpy/core/tests/test_umath_complex.py +++ b/numpy/core/tests/test_umath_complex.py @@ -541,7 +541,7 @@ def check_complex_value(f, x1, y1, x2, y2, exact=True): else: assert_almost_equal(f(z1), z2) -class TestSpecialComplexAVX(object): +class TestSpecialComplexAVX: @pytest.mark.parametrize("stride", [-4,-2,-1,1,2,4]) @pytest.mark.parametrize("astype", [np.complex64, np.complex128]) def test_array(self, stride, astype): @@ -568,7 +568,7 @@ def test_array(self, stride, astype): with np.errstate(invalid='ignore'): assert_equal(np.square(arr[::stride]), sq_true[::stride]) -class TestComplexAbsoluteAVX(object): +class TestComplexAbsoluteAVX: @pytest.mark.parametrize("arraysize", [1,2,3,4,5,6,7,8,9,10,11,13,15,17,18,19]) @pytest.mark.parametrize("stride", [-4,-3,-2,-1,1,2,3,4]) @pytest.mark.parametrize("astype", [np.complex64, np.complex128]) @@ -579,7 +579,7 @@ def test_array(self, arraysize, stride, astype): assert_equal(np.abs(arr[::stride]), abs_true[::stride]) # Testcase taken as is from https://github.com/numpy/numpy/issues/16660 -class TestComplexAbsoluteMixedDTypes(object): +class TestComplexAbsoluteMixedDTypes: @pytest.mark.parametrize("stride", [-4,-3,-2,-1,1,2,3,4]) @pytest.mark.parametrize("astype", [np.complex64, np.complex128]) @pytest.mark.parametrize("func", ['abs', 'square', 'conjugate']) diff --git a/numpy/ctypeslib.py b/numpy/ctypeslib.py index e8f7750fe5ff..8ba6f15e5a6d 100644 --- a/numpy/ctypeslib.py +++ b/numpy/ctypeslib.py @@ -4,7 +4,7 @@ ============================ See Also ---------- +-------- load_library : Load a C library. ndpointer : Array restype/argtype with verification. as_ctypes : Create a ctypes array from an ndarray. @@ -49,11 +49,12 @@ >>> _lib.foo_func(out, len(out)) #doctest: +SKIP """ -__all__ = ['load_library', 'ndpointer', 'c_intp', 'as_ctypes', 'as_array'] +__all__ = ['load_library', 'ndpointer', 'c_intp', 'as_ctypes', 'as_array', + 'as_ctypes_type'] import os from numpy import ( - integer, ndarray, dtype as _dtype, array, frombuffer + integer, ndarray, dtype as _dtype, asarray, frombuffer ) from numpy.core.multiarray import _flagdict, flagsobj @@ -514,7 +515,7 @@ def as_array(obj, shape=None): p_arr_type = ctypes.POINTER(_ctype_ndarray(obj._type_, shape)) obj = ctypes.cast(obj, p_arr_type).contents - return array(obj, copy=False) + return asarray(obj) def as_ctypes(obj): diff --git a/numpy/ctypeslib.pyi b/numpy/ctypeslib.pyi index cacc97d68ea7..689ea416408b 100644 --- a/numpy/ctypeslib.pyi +++ b/numpy/ctypeslib.pyi @@ -1,7 +1,14 @@ -from typing import Any +from typing import List, Type +from ctypes import _SimpleCData -load_library: Any -ndpointer: Any -c_intp: Any -as_ctypes: Any -as_array: Any +__all__: List[str] + +# TODO: Update the `npt.mypy_plugin` such that it substitutes `c_intp` for +# a specific `_SimpleCData[int]` subclass (e.g. `ctypes.c_long`) +c_intp: Type[_SimpleCData[int]] + +def load_library(libname, loader_path): ... +def ndpointer(dtype=..., ndim=..., shape=..., flags=...): ... +def as_ctypes(obj): ... +def as_array(obj, shape=...): ... +def as_ctypes_type(dtype): ... diff --git a/numpy/distutils/ccompiler.py b/numpy/distutils/ccompiler.py index 106436e64471..061f4862dc19 100644 --- a/numpy/distutils/ccompiler.py +++ b/numpy/distutils/ccompiler.py @@ -26,10 +26,8 @@ _commandline_dep_string # globals for parallel build management -try: - import threading -except ImportError: - import dummy_threading as threading +import threading + _job_semaphore = None _global_lock = threading.Lock() _processing_files = set() @@ -386,6 +384,12 @@ def CCompiler_customize_cmd(self, cmd, ignore=()): """ log.info('customize %s using %s' % (self.__class__.__name__, cmd.__class__.__name__)) + + if hasattr(self, 'compiler') and 'clang' in self.compiler[0]: + # clang defaults to a non-strict floating error point model. + # Since NumPy and most Python libs give warnings for these, override: + self.compiler.append('-ffp-exception-behavior=strict') + def allow(attr): return getattr(cmd, attr, None) is not None and attr not in ignore diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py index 3eba6e32af91..47d07ad4ab22 100644 --- a/numpy/distutils/ccompiler_opt.py +++ b/numpy/distutils/ccompiler_opt.py @@ -259,7 +259,7 @@ class _Config: AVX512_SKX = dict( interest=42, implies="AVX512CD", group="AVX512VL AVX512BW AVX512DQ", detect="AVX512_SKX", implies_detect=False, - extra_checks="AVX512BW_MASK" + extra_checks="AVX512BW_MASK AVX512DQ_MASK" ), AVX512_CLX = dict( interest=43, implies="AVX512_SKX", group="AVX512VNNI", @@ -276,7 +276,7 @@ class _Config: ), # IBM/Power ## Power7/ISA 2.06 - VSX = dict(interest=1, headers="altivec.h"), + VSX = dict(interest=1, headers="altivec.h", extra_checks="VSX_ASM"), ## Power8/ISA 2.07 VSX2 = dict(interest=2, implies="VSX", implies_detect=False), ## Power9/ISA 3.00 @@ -543,16 +543,16 @@ class _Distutils: def __init__(self, ccompiler): self._ccompiler = ccompiler - def dist_compile(self, sources, flags, **kwargs): + def dist_compile(self, sources, flags, ccompiler=None, **kwargs): """Wrap CCompiler.compile()""" assert(isinstance(sources, list)) assert(isinstance(flags, list)) flags = kwargs.pop("extra_postargs", []) + flags - return self._ccompiler.compile( - sources, extra_postargs=flags, **kwargs - ) + if not ccompiler: + ccompiler = self._ccompiler + return ccompiler.compile(sources, extra_postargs=flags, **kwargs) - def dist_test(self, source, flags): + def dist_test(self, source, flags, macros=[]): """Return True if 'CCompiler.compile()' able to compile a source file with certain flags. """ @@ -569,7 +569,7 @@ def dist_test(self, source, flags): test = False try: self.dist_compile( - [source], flags, output_dir=self.conf_tmp_path + [source], flags, macros=macros, output_dir=self.conf_tmp_path ) test = True except CompileError as e: @@ -579,45 +579,41 @@ def dist_test(self, source, flags): return test def dist_info(self): - """Return a string containing all environment information, required - by the abstract class '_CCompiler' to discovering the platform - environment, also used as a cache factor in order to detect - any changes from outside. + """ + Return a tuple containing info about (platform, compiler, extra_args), + required by the abstract class '_CCompiler' for discovering the + platform environment. This is also used as a cache factor in order + to detect any changes happening from outside. """ if hasattr(self, "_dist_info"): return self._dist_info - # play it safe - cc_info = "" - compiler = getattr(self._ccompiler, "compiler", None) - if compiler is not None: - if isinstance(compiler, str): - cc_info += compiler - elif hasattr(compiler, "__iter__"): - cc_info += ' '.join(compiler) - # in case if 'compiler' attribute doesn't provide anything - cc_type = getattr(self._ccompiler, "compiler_type", "") - if cc_type in ("intelem", "intelemw", "mingw64"): - cc_info += "x86_64" + + cc_type = getattr(self._ccompiler, "compiler_type", '') + if cc_type in ("intelem", "intelemw"): + platform = "x86_64" elif cc_type in ("intel", "intelw", "intele"): - cc_info += "x86" - elif cc_type in ("msvc", "mingw32"): - import platform - if platform.architecture()[0] == "32bit": - cc_info += "x86" + platform = "x86" + else: + from distutils.util import get_platform + platform = get_platform() + + cc_info = getattr(self._ccompiler, "compiler", getattr(self._ccompiler, "compiler_so", '')) + if not cc_type or cc_type == "unix": + if hasattr(cc_info, "__iter__"): + compiler = cc_info[0] else: - cc_info += "x86_64" + compiler = str(cc_info) else: - # the last hope, too bad for cross-compiling - import platform - cc_info += platform.machine() + compiler = cc_type - cc_info += cc_type - cflags = os.environ.get("CFLAGS", "") - if cflags not in cc_info: - cc_info += cflags + if hasattr(cc_info, "__iter__") and len(cc_info) > 1: + extra_args = ' '.join(cc_info[1:]) + else: + extra_args = os.environ.get("CFLAGS", "") + extra_args += os.environ.get("CPPFLAGS", "") - self._dist_info = cc_info - return cc_info + self._dist_info = (platform, compiler, extra_args) + return self._dist_info @staticmethod def dist_error(*args): @@ -751,12 +747,14 @@ def __init__(self, cache_path=None, *factors): self.cache_me = {} self.cache_private = set() self.cache_infile = False + self._cache_path = None if self.conf_nocache: self.dist_log("cache is disabled by `Config`") return - chash = self.cache_hash(*factors, *self.conf_cache_factors) + self._cache_hash = self.cache_hash(*factors, *self.conf_cache_factors) + self._cache_path = cache_path if cache_path: if os.path.exists(cache_path): self.dist_log("load cache from file ->", cache_path) @@ -769,7 +767,7 @@ def __init__(self, cache_path=None, *factors): elif not hasattr(cache_mod, "hash") or \ not hasattr(cache_mod, "data"): self.dist_log("invalid cache file", stderr=True) - elif chash == cache_mod.hash: + elif self._cache_hash == cache_mod.hash: self.dist_log("hit the file cache") for attr, val in cache_mod.data.items(): setattr(self, attr, val) @@ -777,10 +775,8 @@ def __init__(self, cache_path=None, *factors): else: self.dist_log("miss the file cache") - atexit.register(self._cache_write, cache_path, chash) - if not self.cache_infile: - other_cache = _share_cache.get(chash) + other_cache = _share_cache.get(self._cache_hash) if other_cache: self.dist_log("hit the memory cache") for attr, val in other_cache.__dict__.items(): @@ -789,32 +785,41 @@ def __init__(self, cache_path=None, *factors): continue setattr(self, attr, val) - _share_cache[chash] = self + _share_cache[self._cache_hash] = self + atexit.register(self.cache_flush) def __del__(self): - # TODO: remove the cache form share on del - pass + for h, o in _share_cache.items(): + if o == self: + _share_cache.pop(h) + break - def _cache_write(self, cache_path, cache_hash): + def cache_flush(self): + """ + Force update the cache. + """ + if not self._cache_path: + return # TODO: don't write if the cache doesn't change - self.dist_log("write cache to path ->", cache_path) - for attr in list(self.__dict__.keys()): + self.dist_log("write cache to path ->", self._cache_path) + cdict = self.__dict__.copy() + for attr in self.__dict__.keys(): if re.match(self._cache_ignore, attr): - self.__dict__.pop(attr) + cdict.pop(attr) - d = os.path.dirname(cache_path) + d = os.path.dirname(self._cache_path) if not os.path.exists(d): os.makedirs(d) - repr_dict = pprint.pformat(self.__dict__, compact=True) - with open(cache_path, "w") as f: + repr_dict = pprint.pformat(cdict, compact=True) + with open(self._cache_path, "w") as f: f.write(textwrap.dedent("""\ # AUTOGENERATED DON'T EDIT # Please make changes to the code generator \ (distutils/ccompiler_opt.py) hash = {} data = \\ - """).format(cache_hash)) + """).format(self._cache_hash)) f.write(repr_dict) def cache_hash(self, *factors): @@ -845,7 +850,7 @@ def cache_wrap_me(self, *args, **kwargs): return ccb return cache_wrap_me -class _CCompiler(object): +class _CCompiler: """A helper class for `CCompilerOpt` containing all utilities that related to the fundamental compiler's functions. @@ -893,57 +898,56 @@ class _CCompiler(object): def __init__(self): if hasattr(self, "cc_is_cached"): return - to_detect = ( - # attr regex - ( - ("cc_on_x64", "^(x|x86_|amd)64"), - ("cc_on_x86", "^(x86|i386|i686)"), - ("cc_on_ppc64le", "^(powerpc|ppc)64(el|le)"), - ("cc_on_ppc64", "^(powerpc|ppc)64"), - ("cc_on_armhf", "^arm"), - ("cc_on_aarch64", "^aarch64"), - # priority is given to first of string - # if it fail we search in the rest, due - # to append platform.machine() at the end, - # check method 'dist_info()' for more clarification. - ("cc_on_x64", ".*(x|x86_|amd)64.*"), - ("cc_on_x86", ".*(x86|i386|i686).*"), - ("cc_on_ppc64le", ".*(powerpc|ppc)64(el|le).*"), - ("cc_on_ppc64", ".*(powerpc|ppc)64.*"), - ("cc_on_armhf", ".*arm.*"), - ("cc_on_aarch64", ".*aarch64.*"), - # undefined platform - ("cc_on_noarch", ""), - ), - ( - ("cc_is_gcc", r".*(gcc|gnu\-g).*"), - ("cc_is_clang", ".*clang.*"), - ("cc_is_iccw", ".*(intelw|intelemw|iccw).*"), # intel msvc like - ("cc_is_icc", ".*(intel|icc).*"), # intel unix like - ("cc_is_msvc", ".*msvc.*"), - ("cc_is_nocc", ""), - ), - (("cc_has_debug", ".*(O0|Od|ggdb|coverage|debug:full).*"),), - (("cc_has_native", ".*(-march=native|-xHost|/QxHost).*"),), - # in case if the class run with -DNPY_DISABLE_OPTIMIZATION - (("cc_noopt", ".*DISABLE_OPT.*"),), + # attr regex + detect_arch = ( + ("cc_on_x64", ".*(x|x86_|amd)64.*"), + ("cc_on_x86", ".*(win32|x86|i386|i686).*"), + ("cc_on_ppc64le", ".*(powerpc|ppc)64(el|le).*"), + ("cc_on_ppc64", ".*(powerpc|ppc)64.*"), + ("cc_on_aarch64", ".*(aarch64|arm64).*"), + ("cc_on_armhf", ".*arm.*"), + # undefined platform + ("cc_on_noarch", ""), + ) + detect_compiler = ( + ("cc_is_gcc", r".*(gcc|gnu\-g).*"), + ("cc_is_clang", ".*clang.*"), + ("cc_is_iccw", ".*(intelw|intelemw|iccw).*"), # intel msvc like + ("cc_is_icc", ".*(intel|icc).*"), # intel unix like + ("cc_is_msvc", ".*msvc.*"), + # undefined compiler will be treat it as gcc + ("cc_is_nocc", ""), + ) + detect_args = ( + ("cc_has_debug", ".*(O0|Od|ggdb|coverage|debug:full).*"), + ("cc_has_native", ".*(-march=native|-xHost|/QxHost).*"), + # in case if the class run with -DNPY_DISABLE_OPTIMIZATION + ("cc_noopt", ".*DISABLE_OPT.*"), ) - for section in to_detect: - for attr, rgex in section: - setattr(self, attr, False) dist_info = self.dist_info() - for section in to_detect: + platform, compiler_info, extra_args = dist_info + # set False to all attrs + for section in (detect_arch, detect_compiler, detect_args): for attr, rgex in section: - if rgex and not re.match(rgex, dist_info, re.IGNORECASE): + setattr(self, attr, False) + + for detect, searchin in ((detect_arch, platform), (detect_compiler, compiler_info)): + for attr, rgex in detect: + if rgex and not re.match(rgex, searchin, re.IGNORECASE): continue setattr(self, attr, True) break + for attr, rgex in detect_args: + if rgex and not re.match(rgex, extra_args, re.IGNORECASE): + continue + setattr(self, attr, True) + if self.cc_on_noarch: self.dist_log( - "unable to detect CPU arch via compiler info, " - "optimization is disabled \ninfo << %s >> " % dist_info, + "unable to detect CPU architecture which lead to disable the optimization. " + f"check dist_info:<<\n{dist_info}\n>>", stderr=True ) self.cc_noopt = True @@ -958,8 +962,9 @@ def __init__(self): but still has the same gcc optimization flags. """ self.dist_log( - "unable to detect compiler name via info <<\n%s\n>> " - "treating it as a gcc" % dist_info, + "unable to detect compiler type which leads to treating it as GCC. " + "this is a normal behavior if you're using gcc-like compiler such as MinGW or IBM/XLC." + f"check dist_info:<<\n{dist_info}\n>>", stderr=True ) self.cc_is_gcc = True @@ -1167,20 +1172,23 @@ def __init__(self): self.feature_is_cached = True - def feature_names(self, names=None, force_flags=None): + def feature_names(self, names=None, force_flags=None, macros=[]): """ Returns a set of CPU feature names that supported by platform and the **C** compiler. Parameters ---------- - 'names': sequence or None, optional + names: sequence or None, optional Specify certain CPU features to test it against the **C** compiler. if None(default), it will test all current supported features. **Note**: feature names must be in upper-case. - 'force_flags': list or None, optional - If None(default), default compiler flags for every CPU feature will be used - during the test. + force_flags: list or None, optional + If None(default), default compiler flags for every CPU feature will + be used during the test. + + macros : list of tuples, optional + A list of C macro definitions. """ assert( names is None or ( @@ -1193,7 +1201,9 @@ def feature_names(self, names=None, force_flags=None): names = self.feature_supported.keys() supported_names = set() for f in names: - if self.feature_is_supported(f, force_flags=force_flags): + if self.feature_is_supported( + f, force_flags=force_flags, macros=macros + ): supported_names.add(f) return supported_names @@ -1428,20 +1438,23 @@ def feature_flags(self, names): return self.cc_normalize_flags(flags) @_Cache.me - def feature_test(self, name, force_flags=None): + def feature_test(self, name, force_flags=None, macros=[]): """ Test a certain CPU feature against the compiler through its own check file. Parameters ---------- - 'name': str + name: str Supported CPU feature name. - 'force_flags': list or None, optional + force_flags: list or None, optional If None(default), the returned flags from `feature_flags()` will be used. - """ + + macros : list of tuples, optional + A list of C macro definitions. + """ if force_flags is None: force_flags = self.feature_flags(name) @@ -1457,24 +1470,29 @@ def feature_test(self, name, force_flags=None): if not os.path.exists(test_path): self.dist_fatal("feature test file is not exist", test_path) - test = self.dist_test(test_path, force_flags + self.cc_flags["werror"]) + test = self.dist_test( + test_path, force_flags + self.cc_flags["werror"], macros=macros + ) if not test: self.dist_log("testing failed", stderr=True) return test @_Cache.me - def feature_is_supported(self, name, force_flags=None): + def feature_is_supported(self, name, force_flags=None, macros=[]): """ Check if a certain CPU feature is supported by the platform and compiler. Parameters ---------- - 'name': str + name: str CPU feature name in uppercase. - 'force_flags': list or None, optional - If None(default), default compiler flags for every CPU feature will be used - during test. + force_flags: list or None, optional + If None(default), default compiler flags for every CPU feature will + be used during test. + + macros : list of tuples, optional + A list of C macro definitions. """ assert(name.isupper()) assert(force_flags is None or isinstance(force_flags, list)) @@ -1482,9 +1500,9 @@ def feature_is_supported(self, name, force_flags=None): supported = name in self.feature_supported if supported: for impl in self.feature_implies(name): - if not self.feature_test(impl, force_flags): + if not self.feature_test(impl, force_flags, macros=macros): return False - if not self.feature_test(name, force_flags): + if not self.feature_test(name, force_flags, macros=macros): return False return supported @@ -1774,7 +1792,7 @@ def parse_targets(self, source): tokens = tokens[start_pos:end_pos] return self._parse_target_tokens(tokens) - _parse_regex_arg = re.compile(r'\s|[,]|([+-])') + _parse_regex_arg = re.compile(r'\s|,|([+-])') def _parse_arg_features(self, arg_name, req_features): if not isinstance(req_features, str): self.dist_fatal("expected a string in '%s'" % arg_name) @@ -1807,7 +1825,9 @@ def _parse_arg_features(self, arg_name, req_features): self.dist_fatal(arg_name, "native option isn't supported by the compiler" ) - features_to = self.feature_names(force_flags=native) + features_to = self.feature_names( + force_flags=native, macros=[("DETECT_FEATURES", 1)] + ) elif TOK == "MAX": features_to = self.feature_supported.keys() elif TOK == "MIN": @@ -2147,7 +2167,7 @@ def cpu_dispatch_names(self): """ return self.parse_dispatch_names - def try_dispatch(self, sources, src_dir=None, **kwargs): + def try_dispatch(self, sources, src_dir=None, ccompiler=None, **kwargs): """ Compile one or more dispatch-able sources and generates object files, also generates abstract C config headers and macros that @@ -2170,6 +2190,11 @@ def try_dispatch(self, sources, src_dir=None, **kwargs): Path of parent directory for the generated headers and wrapped sources. If None(default) the files will generated in-place. + ccompiler: CCompiler + Distutils `CCompiler` instance to be used for compilation. + If None (default), the provided instance during the initialization + will be used instead. + **kwargs : any Arguments to pass on to the `CCompiler.compile()` @@ -2224,13 +2249,15 @@ def try_dispatch(self, sources, src_dir=None, **kwargs): # among them. objects = [] for flags, srcs in to_compile.items(): - objects += self.dist_compile(srcs, list(flags), **kwargs) + objects += self.dist_compile( + srcs, list(flags), ccompiler=ccompiler, **kwargs + ) return objects def generate_dispatch_header(self, header_path): """ - Generate the dispatch header which containing all definitions - and headers of instruction-sets for the enabled CPU baseline and + Generate the dispatch header which contains the #definitions and headers + for platform-specific instruction-sets for the enabled CPU baseline and dispatch-able features. Its highly recommended to take a look at the generated header @@ -2244,6 +2271,14 @@ def generate_dispatch_header(self, header_path): baseline_len = len(baseline_names) dispatch_len = len(dispatch_names) + header_dir = os.path.dirname(header_path) + if not os.path.exists(header_dir): + self.dist_log( + f"dispatch header dir {header_dir} does not exist, creating it", + stderr=True + ) + os.makedirs(header_dir) + with open(header_path, 'w') as f: baseline_calls = ' \\\n'.join([ ( @@ -2304,19 +2339,25 @@ def generate_dispatch_header(self, header_path): def report(self, full=False): report = [] + platform_rows = [] baseline_rows = [] dispatch_rows = [] + report.append(("Platform", platform_rows)) + report.append(("", "")) report.append(("CPU baseline", baseline_rows)) report.append(("", "")) report.append(("CPU dispatch", dispatch_rows)) + ########## platform ########## + platform_rows.append(("Architecture", ( + "unsupported" if self.cc_on_noarch else self.cc_march) + )) + platform_rows.append(("Compiler", ( + "unix-like" if self.cc_is_nocc else self.cc_name) + )) ########## baseline ########## if self.cc_noopt: - baseline_rows.append(( - "Requested", "optimization disabled %s" % ( - "(unsupported arch)" if self.cc_on_noarch else "" - ) - )) + baseline_rows.append(("Requested", "optimization disabled")) else: baseline_rows.append(("Requested", repr(self._requested_baseline))) @@ -2337,11 +2378,7 @@ def report(self, full=False): ########## dispatch ########## if self.cc_noopt: - dispatch_rows.append(( - "Requested", "optimization disabled %s" % ( - "(unsupported arch)" if self.cc_on_noarch else "" - ) - )) + baseline_rows.append(("Requested", "optimization disabled")) else: dispatch_rows.append(("Requested", repr(self._requested_dispatch))) @@ -2448,7 +2485,8 @@ def _wrap_target(self, output_dir, dispatch_src, target, nochange=False): return wrap_path def _generate_config(self, output_dir, dispatch_src, targets, has_baseline=False): - config_path = os.path.basename(dispatch_src).replace(".c", ".h") + config_path = os.path.basename(dispatch_src) + config_path = os.path.splitext(config_path)[0] + '.h' config_path = os.path.join(output_dir, config_path) # check if targets didn't change to avoid recompiling cache_hash = self.cache_hash(targets, has_baseline) @@ -2506,30 +2544,24 @@ def _generate_config(self, output_dir, dispatch_src, targets, has_baseline=False )) return False -def new_ccompiler_opt(compiler, **kwargs): +def new_ccompiler_opt(compiler, dispatch_hpath, **kwargs): """ Create a new instance of 'CCompilerOpt' and generate the dispatch header - inside NumPy source dir. + which contains the #definitions and headers of platform-specific instruction-sets for + the enabled CPU baseline and dispatch-able features. Parameters ---------- - 'compiler' : CCompiler instance - '**kwargs': passed as-is to `CCompilerOpt(...)` + compiler : CCompiler instance + dispatch_hpath : str + path of the dispatch header + **kwargs: passed as-is to `CCompilerOpt(...)` Returns ------- new instance of CCompilerOpt """ opt = CCompilerOpt(compiler, **kwargs) - npy_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - header_dir = os.path.join(npy_path, *("core/src/common".split("/"))) - header_path = os.path.join(header_dir, "_cpu_dispatch.h") - if not os.path.exists(header_path) or not opt.is_cached(): - if not os.path.exists(header_dir): - opt.dist_log( - "dispatch header dir '%s' isn't exist, creating it" % header_dir, - stderr=True - ) - os.makedirs(header_dir) - opt.generate_dispatch_header(header_path) + if not os.path.exists(dispatch_hpath) or not opt.is_cached(): + opt.generate_dispatch_header(dispatch_hpath) return opt diff --git a/numpy/distutils/checks/cpu_avx.c b/numpy/distutils/checks/cpu_avx.c index 737c0d2e9492..26ae18466740 100644 --- a/numpy/distutils/checks/cpu_avx.c +++ b/numpy/distutils/checks/cpu_avx.c @@ -1,7 +1,20 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __AVX__ + #error "HOST/ARCH doesn't support AVX" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { - __m256 a = _mm256_add_ps(_mm256_setzero_ps(), _mm256_setzero_ps()); + __m256 a = _mm256_add_ps(_mm256_loadu_ps((const float*)argv[argc-1]), _mm256_loadu_ps((const float*)argv[1])); return (int)_mm_cvtss_f32(_mm256_castps256_ps128(a)); } diff --git a/numpy/distutils/checks/cpu_avx2.c b/numpy/distutils/checks/cpu_avx2.c index dfb11fd79967..ddde868f1b58 100644 --- a/numpy/distutils/checks/cpu_avx2.c +++ b/numpy/distutils/checks/cpu_avx2.c @@ -1,7 +1,20 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __AVX2__ + #error "HOST/ARCH doesn't support AVX2" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { - __m256i a = _mm256_abs_epi16(_mm256_setzero_si256()); + __m256i a = _mm256_abs_epi16(_mm256_loadu_si256((const __m256i*)argv[argc-1])); return _mm_cvtsi128_si32(_mm256_castsi256_si128(a)); } diff --git a/numpy/distutils/checks/cpu_avx512_clx.c b/numpy/distutils/checks/cpu_avx512_clx.c index 71dad83a79f0..81edcd067005 100644 --- a/numpy/distutils/checks/cpu_avx512_clx.c +++ b/numpy/distutils/checks/cpu_avx512_clx.c @@ -1,8 +1,22 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __AVX512VNNI__ + #error "HOST/ARCH doesn't support CascadeLake AVX512 features" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { /* VNNI */ - __m512i a = _mm512_dpbusd_epi32(_mm512_setzero_si512(), _mm512_setzero_si512(), _mm512_setzero_si512()); + __m512i a = _mm512_loadu_si512((const __m512i*)argv[argc-1]); + a = _mm512_dpbusd_epi32(a, _mm512_setzero_si512(), a); return _mm_cvtsi128_si32(_mm512_castsi512_si128(a)); } diff --git a/numpy/distutils/checks/cpu_avx512_cnl.c b/numpy/distutils/checks/cpu_avx512_cnl.c index dfab4436d07e..5799f122b511 100644 --- a/numpy/distutils/checks/cpu_avx512_cnl.c +++ b/numpy/distutils/checks/cpu_avx512_cnl.c @@ -1,10 +1,24 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__AVX512VBMI__) || !defined(__AVX512IFMA__) + #error "HOST/ARCH doesn't support CannonLake AVX512 features" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { + __m512i a = _mm512_loadu_si512((const __m512i*)argv[argc-1]); /* IFMA */ - __m512i a = _mm512_madd52hi_epu64(_mm512_setzero_si512(), _mm512_setzero_si512(), _mm512_setzero_si512()); + a = _mm512_madd52hi_epu64(a, a, _mm512_setzero_si512()); /* VMBI */ - a = _mm512_permutex2var_epi8(a, _mm512_setzero_si512(), _mm512_setzero_si512()); + a = _mm512_permutex2var_epi8(a, _mm512_setzero_si512(), a); return _mm_cvtsi128_si32(_mm512_castsi512_si128(a)); } diff --git a/numpy/distutils/checks/cpu_avx512_icl.c b/numpy/distutils/checks/cpu_avx512_icl.c index cf2706b3b9ff..3cf44d73164b 100644 --- a/numpy/distutils/checks/cpu_avx512_icl.c +++ b/numpy/distutils/checks/cpu_avx512_icl.c @@ -1,9 +1,23 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__AVX512VPOPCNTDQ__) || !defined(__AVX512BITALG__) || !defined(__AVX512VPOPCNTDQ__) + #error "HOST/ARCH doesn't support IceLake AVX512 features" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { + __m512i a = _mm512_loadu_si512((const __m512i*)argv[argc-1]); /* VBMI2 */ - __m512i a = _mm512_shrdv_epi64(_mm512_setzero_si512(), _mm512_setzero_si512(), _mm512_setzero_si512()); + a = _mm512_shrdv_epi64(a, a, _mm512_setzero_si512()); /* BITLAG */ a = _mm512_popcnt_epi8(a); /* VPOPCNTDQ */ diff --git a/numpy/distutils/checks/cpu_avx512_knl.c b/numpy/distutils/checks/cpu_avx512_knl.c index 0699f37a6346..b3f4f6976514 100644 --- a/numpy/distutils/checks/cpu_avx512_knl.c +++ b/numpy/distutils/checks/cpu_avx512_knl.c @@ -1,10 +1,24 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__AVX512ER__) || !defined(__AVX512PF__) + #error "HOST/ARCH doesn't support Knights Landing AVX512 features" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { int base[128]; + __m512d ad = _mm512_loadu_pd((const __m512d*)argv[argc-1]); /* ER */ - __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(_mm512_setzero_pd())); + __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); /* PF */ _mm512_mask_prefetch_i64scatter_pd(base, _mm512_cmpeq_epi64_mask(a, a), a, 1, _MM_HINT_T1); return base[0]; diff --git a/numpy/distutils/checks/cpu_avx512_knm.c b/numpy/distutils/checks/cpu_avx512_knm.c index db61b4bfa674..2c426462bd34 100644 --- a/numpy/distutils/checks/cpu_avx512_knm.c +++ b/numpy/distutils/checks/cpu_avx512_knm.c @@ -1,9 +1,22 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__AVX5124FMAPS__) || !defined(__AVX5124VNNIW__) || !defined(__AVX512VPOPCNTDQ__) + #error "HOST/ARCH doesn't support Knights Mill AVX512 features" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { - __m512i a = _mm512_setzero_si512(); - __m512 b = _mm512_setzero_ps(); + __m512i a = _mm512_loadu_si512((const __m512i*)argv[argc-1]); + __m512 b = _mm512_loadu_ps((const __m512*)argv[argc-2]); /* 4FMAPS */ b = _mm512_4fmadd_ps(b, b, b, b, b, NULL); diff --git a/numpy/distutils/checks/cpu_avx512_skx.c b/numpy/distutils/checks/cpu_avx512_skx.c index 1d5e15b5e5b8..8840efb7e5ee 100644 --- a/numpy/distutils/checks/cpu_avx512_skx.c +++ b/numpy/distutils/checks/cpu_avx512_skx.c @@ -1,9 +1,23 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__AVX512VL__) || !defined(__AVX512BW__) || !defined(__AVX512DQ__) + #error "HOST/ARCH doesn't support SkyLake AVX512 features" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { + __m512i aa = _mm512_abs_epi32(_mm512_loadu_si512((const __m512i*)argv[argc-1])); /* VL */ - __m256i a = _mm256_abs_epi64(_mm256_setzero_si256()); + __m256i a = _mm256_abs_epi64(_mm512_extracti64x4_epi64(aa, 1)); /* DQ */ __m512i b = _mm512_broadcast_i32x8(a); /* BW */ diff --git a/numpy/distutils/checks/cpu_avx512cd.c b/numpy/distutils/checks/cpu_avx512cd.c index 61bef6b8270e..5e29c79e34a7 100644 --- a/numpy/distutils/checks/cpu_avx512cd.c +++ b/numpy/distutils/checks/cpu_avx512cd.c @@ -1,7 +1,20 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __AVX512CD__ + #error "HOST/ARCH doesn't support AVX512CD" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { - __m512i a = _mm512_lzcnt_epi32(_mm512_setzero_si512()); + __m512i a = _mm512_lzcnt_epi32(_mm512_loadu_si512((const __m512i*)argv[argc-1])); return _mm_cvtsi128_si32(_mm512_castsi512_si128(a)); } diff --git a/numpy/distutils/checks/cpu_avx512f.c b/numpy/distutils/checks/cpu_avx512f.c index f60cc09dd094..d0eb7b1ad5c6 100644 --- a/numpy/distutils/checks/cpu_avx512f.c +++ b/numpy/distutils/checks/cpu_avx512f.c @@ -1,7 +1,20 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __AVX512F__ + #error "HOST/ARCH doesn't support AVX512F" + #endif +#endif + #include -int main(void) +int main(int argc, char **argv) { - __m512i a = _mm512_abs_epi32(_mm512_setzero_si512()); + __m512i a = _mm512_abs_epi32(_mm512_loadu_si512((const __m512i*)argv[argc-1])); return _mm_cvtsi128_si32(_mm512_castsi512_si128(a)); } diff --git a/numpy/distutils/checks/cpu_f16c.c b/numpy/distutils/checks/cpu_f16c.c index a5a343e2dd59..fdf36cec580c 100644 --- a/numpy/distutils/checks/cpu_f16c.c +++ b/numpy/distutils/checks/cpu_f16c.c @@ -1,9 +1,22 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __F16C__ + #error "HOST/ARCH doesn't support F16C" + #endif +#endif + #include #include -int main(void) +int main(int argc, char **argv) { - __m128 a = _mm_cvtph_ps(_mm_setzero_si128()); - __m256 a8 = _mm256_cvtph_ps(_mm_setzero_si128()); + __m128 a = _mm_cvtph_ps(_mm_loadu_si128((const __m128i*)argv[argc-1])); + __m256 a8 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i*)argv[argc-2])); return (int)(_mm_cvtss_f32(a) + _mm_cvtss_f32(_mm256_castps256_ps128(a8))); } diff --git a/numpy/distutils/checks/cpu_fma3.c b/numpy/distutils/checks/cpu_fma3.c index cf34c6cb1572..bfeef22b5f0e 100644 --- a/numpy/distutils/checks/cpu_fma3.c +++ b/numpy/distutils/checks/cpu_fma3.c @@ -1,8 +1,22 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__FMA__) && !defined(__AVX2__) + #error "HOST/ARCH doesn't support FMA3" + #endif +#endif + #include #include -int main(void) +int main(int argc, char **argv) { - __m256 a = _mm256_fmadd_ps(_mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps()); + __m256 a = _mm256_loadu_ps((const float*)argv[argc-1]); + a = _mm256_fmadd_ps(a, a, a); return (int)_mm_cvtss_f32(_mm256_castps256_ps128(a)); } diff --git a/numpy/distutils/checks/cpu_fma4.c b/numpy/distutils/checks/cpu_fma4.c index 1ad717033e24..0ff17a483385 100644 --- a/numpy/distutils/checks/cpu_fma4.c +++ b/numpy/distutils/checks/cpu_fma4.c @@ -5,8 +5,9 @@ #include #endif -int main(void) +int main(int argc, char **argv) { - __m256 a = _mm256_macc_ps(_mm256_setzero_ps(), _mm256_setzero_ps(), _mm256_setzero_ps()); + __m256 a = _mm256_loadu_ps((const float*)argv[argc-1]); + a = _mm256_macc_ps(a, a, a); return (int)_mm_cvtss_f32(_mm256_castps256_ps128(a)); } diff --git a/numpy/distutils/checks/cpu_popcnt.c b/numpy/distutils/checks/cpu_popcnt.c index e6a80fb40be4..813c461f05b3 100644 --- a/numpy/distutils/checks/cpu_popcnt.c +++ b/numpy/distutils/checks/cpu_popcnt.c @@ -1,23 +1,32 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env vr `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #if !defined(__SSE4_2__) && !defined(__POPCNT__) + #error "HOST/ARCH doesn't support POPCNT" + #endif +#endif + #ifdef _MSC_VER #include #else #include #endif -int main(void) +int main(int argc, char **argv) { - long long a = 0; - int b; -#ifdef _MSC_VER - #ifdef _M_X64 - a = _mm_popcnt_u64(1); - #endif - b = _mm_popcnt_u32(1); -#else - #ifdef __x86_64__ - a = __builtin_popcountll(1); - #endif - b = __builtin_popcount(1); + // To make sure popcnt instructions are generated + // and been tested against the assembler + unsigned long long a = *((unsigned long long*)argv[argc-1]); + unsigned int b = *((unsigned int*)argv[argc-2]); + +#if defined(_M_X64) || defined(__x86_64__) + a = _mm_popcnt_u64(a); #endif + b = _mm_popcnt_u32(b); return (int)a + b; } diff --git a/numpy/distutils/checks/cpu_sse.c b/numpy/distutils/checks/cpu_sse.c index bb98bf63c0b9..602b74e7bc43 100644 --- a/numpy/distutils/checks/cpu_sse.c +++ b/numpy/distutils/checks/cpu_sse.c @@ -1,3 +1,16 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __SSE__ + #error "HOST/ARCH doesn't support SSE" + #endif +#endif + #include int main(void) diff --git a/numpy/distutils/checks/cpu_sse2.c b/numpy/distutils/checks/cpu_sse2.c index 658afc9b4abf..33826a9ed1a5 100644 --- a/numpy/distutils/checks/cpu_sse2.c +++ b/numpy/distutils/checks/cpu_sse2.c @@ -1,3 +1,16 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __SSE2__ + #error "HOST/ARCH doesn't support SSE2" + #endif +#endif + #include int main(void) diff --git a/numpy/distutils/checks/cpu_sse3.c b/numpy/distutils/checks/cpu_sse3.c index aece1e60174c..d47c20f74be1 100644 --- a/numpy/distutils/checks/cpu_sse3.c +++ b/numpy/distutils/checks/cpu_sse3.c @@ -1,3 +1,16 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __SSE3__ + #error "HOST/ARCH doesn't support SSE3" + #endif +#endif + #include int main(void) diff --git a/numpy/distutils/checks/cpu_sse41.c b/numpy/distutils/checks/cpu_sse41.c index bfdb9feacc47..7c80238a3bc1 100644 --- a/numpy/distutils/checks/cpu_sse41.c +++ b/numpy/distutils/checks/cpu_sse41.c @@ -1,3 +1,16 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __SSE4_1__ + #error "HOST/ARCH doesn't support SSE41" + #endif +#endif + #include int main(void) diff --git a/numpy/distutils/checks/cpu_sse42.c b/numpy/distutils/checks/cpu_sse42.c index 24f5d93fe1d4..f60e18f3c4f1 100644 --- a/numpy/distutils/checks/cpu_sse42.c +++ b/numpy/distutils/checks/cpu_sse42.c @@ -1,3 +1,16 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __SSE4_2__ + #error "HOST/ARCH doesn't support SSE42" + #endif +#endif + #include int main(void) diff --git a/numpy/distutils/checks/cpu_ssse3.c b/numpy/distutils/checks/cpu_ssse3.c index ad0abc1e66fb..fde390d6a37d 100644 --- a/numpy/distutils/checks/cpu_ssse3.c +++ b/numpy/distutils/checks/cpu_ssse3.c @@ -1,3 +1,16 @@ +#if defined(DETECT_FEATURES) && defined(__INTEL_COMPILER) + /* + * Unlike GCC and CLANG, Intel Compiler exposes all supported intrinsics, + * whether or not the build options for those features are specified. + * Therefore, we must test #definitions of CPU features when option native/host + * is enabled via `--cpu-baseline` or through env var `CFLAGS` otherwise + * the test will be broken and leads to enable all possible features. + */ + #ifndef __SSSE3__ + #error "HOST/ARCH doesn't support SSSE3" + #endif +#endif + #include int main(void) diff --git a/numpy/distutils/checks/extra_avx512dq_mask.c b/numpy/distutils/checks/extra_avx512dq_mask.c new file mode 100644 index 000000000000..f0dc88bdd372 --- /dev/null +++ b/numpy/distutils/checks/extra_avx512dq_mask.c @@ -0,0 +1,16 @@ +#include +/** + * Test DQ mask operations due to: + * - MSVC has supported it since vs2019 see, + * https://developercommunity.visualstudio.com/content/problem/518298/missing-avx512bw-mask-intrinsics.html + * - Clang >= v8.0 + * - GCC >= v7.1 + */ +int main(void) +{ + __mmask8 m8 = _mm512_cmpeq_epi64_mask(_mm512_set1_epi64(1), _mm512_set1_epi64(1)); + m8 = _kor_mask8(m8, m8); + m8 = _kxor_mask8(m8, m8); + m8 = _cvtu32_mask8(_cvtmask8_u32(m8)); + return (int)_cvtmask8_u32(m8); +} diff --git a/numpy/distutils/checks/extra_avx512f_reduce.c b/numpy/distutils/checks/extra_avx512f_reduce.c index f979d504ee60..db01aaeef405 100644 --- a/numpy/distutils/checks/extra_avx512f_reduce.c +++ b/numpy/distutils/checks/extra_avx512f_reduce.c @@ -8,7 +8,7 @@ int main(void) { __m512 one_ps = _mm512_set1_ps(1.0f); __m512d one_pd = _mm512_set1_pd(1.0); - __m512i one_i64 = _mm512_set1_epi64(1.0); + __m512i one_i64 = _mm512_set1_epi64(1); // add float sum_ps = _mm512_reduce_add_ps(one_ps); double sum_pd = _mm512_reduce_add_pd(one_pd); diff --git a/numpy/distutils/checks/extra_vsx_asm.c b/numpy/distutils/checks/extra_vsx_asm.c new file mode 100644 index 000000000000..b73a6f43808e --- /dev/null +++ b/numpy/distutils/checks/extra_vsx_asm.c @@ -0,0 +1,36 @@ +/** + * Testing ASM VSX register number fixer '%x' + * + * old versions of CLANG doesn't support %x in the inline asm template + * which fixes register number when using any of the register constraints wa, wd, wf. + * + * xref: + * - https://bugs.llvm.org/show_bug.cgi?id=31837 + * - https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html + */ +#ifndef __VSX__ + #error "VSX is not supported" +#endif +#include + +#if (defined(__GNUC__) && !defined(vec_xl)) || (defined(__clang__) && !defined(__IBMC__)) + #define vsx_ld vec_vsx_ld + #define vsx_st vec_vsx_st +#else + #define vsx_ld vec_xl + #define vsx_st vec_xst +#endif + +int main(void) +{ + float z4[] = {0, 0, 0, 0}; + signed int zout[] = {0, 0, 0, 0}; + + __vector float vz4 = vsx_ld(0, z4); + __vector signed int asm_ret = vsx_ld(0, zout); + + __asm__ ("xvcvspsxws %x0,%x1" : "=wa" (vz4) : "wa" (asm_ret)); + + vsx_st(asm_ret, 0, zout); + return zout[0]; +} diff --git a/numpy/distutils/command/build_clib.py b/numpy/distutils/command/build_clib.py index a0db6f31f7e5..0e31a7dee5be 100644 --- a/numpy/distutils/command/build_clib.py +++ b/numpy/distutils/command/build_clib.py @@ -118,23 +118,31 @@ def run(self): self.compiler.show_customization() if not self.disable_optimization: + dispatch_hpath = os.path.join("numpy", "distutils", "include", "npy_cpu_dispatch_config.h") + dispatch_hpath = os.path.join(self.get_finalized_command("build_src").build_src, dispatch_hpath) opt_cache_path = os.path.abspath( - os.path.join(self.build_temp, 'ccompiler_opt_cache_clib.py' - )) - self.compiler_opt = new_ccompiler_opt( - compiler=self.compiler, cpu_baseline=self.cpu_baseline, - cpu_dispatch=self.cpu_dispatch, cache_path=opt_cache_path + os.path.join(self.build_temp, 'ccompiler_opt_cache_clib.py') ) - if not self.compiler_opt.is_cached(): - log.info("Detected changes on compiler optimizations, force rebuilding") - self.force = True + if hasattr(self, "compiler_opt"): + # By default `CCompilerOpt` update the cache at the exit of + # the process, which may lead to duplicate building + # (see build_extension()/force_rebuild) if run() called + # multiple times within the same os process/thread without + # giving the chance the previous instances of `CCompilerOpt` + # to update the cache. + self.compiler_opt.cache_flush() - import atexit - def report(): + self.compiler_opt = new_ccompiler_opt( + compiler=self.compiler, dispatch_hpath=dispatch_hpath, + cpu_baseline=self.cpu_baseline, cpu_dispatch=self.cpu_dispatch, + cache_path=opt_cache_path + ) + def report(copt): log.info("\n########### CLIB COMPILER OPTIMIZATION ###########") - log.info(self.compiler_opt.report(full=True)) + log.info(copt.report(full=True)) - atexit.register(report) + import atexit + atexit.register(report, self.compiler_opt) if self.have_f_sources(): from numpy.distutils.fcompiler import new_fcompiler @@ -209,7 +217,12 @@ def build_a_library(self, build_info, lib_name, libraries): lib_file = compiler.library_filename(lib_name, output_dir=self.build_clib) depends = sources + build_info.get('depends', []) - if not (self.force or newer_group(depends, lib_file, 'newer')): + + force_rebuild = self.force + if not self.disable_optimization and not self.compiler_opt.is_cached(): + log.debug("Detected changes on compiler optimizations") + force_rebuild = True + if not (force_rebuild or newer_group(depends, lib_file, 'newer')): log.debug("skipping '%s' library (up-to-date)", lib_name) return else: @@ -268,19 +281,44 @@ def build_a_library(self, build_info, lib_name, libraries): # filtering C dispatch-table sources when optimization is not disabled, # otherwise treated as normal sources. copt_c_sources = [] + copt_cxx_sources = [] copt_baseline_flags = [] copt_macros = [] if not self.disable_optimization: - copt_build_src = None if self.inplace else self.get_finalized_command("build_src").build_src - copt_c_sources = [ - c_sources.pop(c_sources.index(src)) - for src in c_sources[:] if src.endswith(".dispatch.c") - ] + bsrc_dir = self.get_finalized_command("build_src").build_src + dispatch_hpath = os.path.join("numpy", "distutils", "include") + dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath) + include_dirs.append(dispatch_hpath) + + copt_build_src = None if self.inplace else bsrc_dir + for _srcs, _dst, _ext in ( + ((c_sources,), copt_c_sources, ('.dispatch.c',)), + ((c_sources, cxx_sources), copt_cxx_sources, + ('.dispatch.cpp', '.dispatch.cxx')) + ): + for _src in _srcs: + _dst += [ + _src.pop(_src.index(s)) + for s in _src[:] if s.endswith(_ext) + ] copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() else: copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) objects = [] + if copt_cxx_sources: + log.info("compiling C++ dispatch-able sources") + objects += self.compiler_opt.try_dispatch( + copt_c_sources, + output_dir=self.build_temp, + src_dir=copt_build_src, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_postargs, + ccompiler=cxx_compiler + ) + if copt_c_sources: log.info("compiling C dispatch-able sources") objects += self.compiler_opt.try_dispatch(copt_c_sources, diff --git a/numpy/distutils/command/build_ext.py b/numpy/distutils/command/build_ext.py index ca6f8bcd24d6..84ec8aa2cb46 100644 --- a/numpy/distutils/command/build_ext.py +++ b/numpy/distutils/command/build_ext.py @@ -146,20 +146,31 @@ def run(self): self.compiler.show_customization() if not self.disable_optimization: - opt_cache_path = os.path.abspath(os.path.join(self.build_temp, 'ccompiler_opt_cache_ext.py')) - self.compiler_opt = new_ccompiler_opt(compiler=self.compiler, - cpu_baseline=self.cpu_baseline, - cpu_dispatch=self.cpu_dispatch, - cache_path=opt_cache_path) - if not self.compiler_opt.is_cached(): - log.info("Detected changes on compiler optimizations, force rebuilding") - self.force = True + dispatch_hpath = os.path.join("numpy", "distutils", "include", "npy_cpu_dispatch_config.h") + dispatch_hpath = os.path.join(self.get_finalized_command("build_src").build_src, dispatch_hpath) + opt_cache_path = os.path.abspath( + os.path.join(self.build_temp, 'ccompiler_opt_cache_ext.py') + ) + if hasattr(self, "compiler_opt"): + # By default `CCompilerOpt` update the cache at the exit of + # the process, which may lead to duplicate building + # (see build_extension()/force_rebuild) if run() called + # multiple times within the same os process/thread without + # giving the chance the previous instances of `CCompilerOpt` + # to update the cache. + self.compiler_opt.cache_flush() + + self.compiler_opt = new_ccompiler_opt( + compiler=self.compiler, dispatch_hpath=dispatch_hpath, + cpu_baseline=self.cpu_baseline, cpu_dispatch=self.cpu_dispatch, + cache_path=opt_cache_path + ) + def report(copt): + log.info("\n########### EXT COMPILER OPTIMIZATION ###########") + log.info(copt.report(full=True)) import atexit - def report(): - log.info("\n########### EXT COMPILER OPTIMIZATION ###########") - log.info(self.compiler_opt.report(full=True)) - atexit.register(report) + atexit.register(report, self.compiler_opt) # Setup directory for storing generated extra DLL files on Windows self.extra_dll_dir = os.path.join(self.build_temp, '.libs') @@ -354,7 +365,11 @@ def build_extension(self, ext): self.get_ext_filename(fullname)) depends = sources + ext.depends - if not (self.force or newer_group(depends, ext_filename, 'newer')): + force_rebuild = self.force + if not self.disable_optimization and not self.compiler_opt.is_cached(): + log.debug("Detected changes on compiler optimizations") + force_rebuild = True + if not (force_rebuild or newer_group(depends, ext_filename, 'newer')): log.debug("skipping '%s' extension (up-to-date)", ext.name) return else: @@ -413,19 +428,44 @@ def build_extension(self, ext): # filtering C dispatch-table sources when optimization is not disabled, # otherwise treated as normal sources. copt_c_sources = [] + copt_cxx_sources = [] copt_baseline_flags = [] copt_macros = [] if not self.disable_optimization: - copt_build_src = None if self.inplace else self.get_finalized_command("build_src").build_src - copt_c_sources = [ - c_sources.pop(c_sources.index(src)) - for src in c_sources[:] if src.endswith(".dispatch.c") - ] + bsrc_dir = self.get_finalized_command("build_src").build_src + dispatch_hpath = os.path.join("numpy", "distutils", "include") + dispatch_hpath = os.path.join(bsrc_dir, dispatch_hpath) + include_dirs.append(dispatch_hpath) + + copt_build_src = None if self.inplace else bsrc_dir + for _srcs, _dst, _ext in ( + ((c_sources,), copt_c_sources, ('.dispatch.c',)), + ((c_sources, cxx_sources), copt_cxx_sources, + ('.dispatch.cpp', '.dispatch.cxx')) + ): + for _src in _srcs: + _dst += [ + _src.pop(_src.index(s)) + for s in _src[:] if s.endswith(_ext) + ] copt_baseline_flags = self.compiler_opt.cpu_baseline_flags() else: copt_macros.append(("NPY_DISABLE_OPTIMIZATION", 1)) c_objects = [] + if copt_cxx_sources: + log.info("compiling C++ dispatch-able sources") + c_objects += self.compiler_opt.try_dispatch( + copt_cxx_sources, + output_dir=output_dir, + src_dir=copt_build_src, + macros=macros + copt_macros, + include_dirs=include_dirs, + debug=self.debug, + extra_postargs=extra_args, + ccompiler=cxx_compiler, + **kws + ) if copt_c_sources: log.info("compiling C dispatch-able sources") c_objects += self.compiler_opt.try_dispatch(copt_c_sources, @@ -559,8 +599,11 @@ def _process_unlinkable_fobjects(self, objects, libraries, objects = list(objects) unlinkable_fobjects = list(unlinkable_fobjects) - # Expand possible fake static libraries to objects - for lib in libraries: + # Expand possible fake static libraries to objects; + # make sure to iterate over a copy of the list as + # "fake" libraries will be removed as they are + # enountered + for lib in libraries[:]: for libdir in library_dirs: fake_lib = os.path.join(libdir, lib + '.fobjects') if os.path.isfile(fake_lib): diff --git a/numpy/distutils/command/build_src.py b/numpy/distutils/command/build_src.py index 303d6197cbd3..5581011f6f22 100644 --- a/numpy/distutils/command/build_src.py +++ b/numpy/distutils/command/build_src.py @@ -715,14 +715,14 @@ def swig_sources(self, sources, extension): return new_sources + py_files -_f_pyf_ext_match = re.compile(r'.*[.](f90|f95|f77|for|ftn|f|pyf)\Z', re.I).match -_header_ext_match = re.compile(r'.*[.](inc|h|hpp)\Z', re.I).match +_f_pyf_ext_match = re.compile(r'.*\.(f90|f95|f77|for|ftn|f|pyf)\Z', re.I).match +_header_ext_match = re.compile(r'.*\.(inc|h|hpp)\Z', re.I).match #### SWIG related auxiliary functions #### _swig_module_name_match = re.compile(r'\s*%module\s*(.*\(\s*package\s*=\s*"(?P[\w_]+)".*\)|)\s*(?P[\w_]+)', re.I).match -_has_c_header = re.compile(r'-[*]-\s*c\s*-[*]-', re.I).search -_has_cpp_header = re.compile(r'-[*]-\s*c[+][+]\s*-[*]-', re.I).search +_has_c_header = re.compile(r'-\*-\s*c\s*-\*-', re.I).search +_has_cpp_header = re.compile(r'-\*-\s*c\+\+\s*-\*-', re.I).search def get_swig_target(source): with open(source, 'r') as f: diff --git a/numpy/distutils/command/config.py b/numpy/distutils/command/config.py index 60881f4a3a4a..1f4037bb5c7a 100644 --- a/numpy/distutils/command/config.py +++ b/numpy/distutils/command/config.py @@ -64,7 +64,7 @@ def _check_compiler (self): % (e, self.compiler.__class__.__name__) print(textwrap.dedent("""\ ============================================================================""")) - raise distutils.errors.DistutilsPlatformError(msg) + raise distutils.errors.DistutilsPlatformError(msg) from e # After MSVC is initialized, add an explicit /MANIFEST to linker # flags. See issues gh-4245 and gh-4101 for details. Also @@ -92,12 +92,13 @@ def _wrap_method(self, mth, lang, args): save_compiler = self.compiler if lang in ['f77', 'f90']: self.compiler = self.fcompiler + if self.compiler is None: + raise CompileError('%s compiler is not set' % (lang,)) try: ret = mth(*((self,)+args)) except (DistutilsExecError, CompileError) as e: - str(e) self.compiler = save_compiler - raise CompileError + raise CompileError from e self.compiler = save_compiler return ret diff --git a/numpy/distutils/conv_template.py b/numpy/distutils/conv_template.py index d08015fdf39f..90e07f8b1036 100644 --- a/numpy/distutils/conv_template.py +++ b/numpy/distutils/conv_template.py @@ -137,7 +137,7 @@ def paren_repl(obj): numrep = obj.group(2) return ','.join([torep]*int(numrep)) -parenrep = re.compile(r"[(]([^)]*)[)]\*(\d+)") +parenrep = re.compile(r"\(([^)]*)\)\*(\d+)") plainrep = re.compile(r"([^*]+)\*(\d+)") def parse_values(astr): # replaces all occurrences of '(a,b,c)*4' in astr @@ -207,7 +207,7 @@ def parse_loop_header(loophead) : dlist.append(tmp) return dlist -replace_re = re.compile(r"@([\w]+)@") +replace_re = re.compile(r"@(\w+)@") def parse_string(astr, env, level, line) : lineno = "#line %d\n" % line @@ -218,7 +218,7 @@ def replace(match): val = env[name] except KeyError: msg = 'line %d: no definition of key "%s"'%(line, name) - raise ValueError(msg) + raise ValueError(msg) from None return val code = [lineno] @@ -285,7 +285,7 @@ def process_file(source): try: code = process_str(''.join(lines)) except ValueError as e: - raise ValueError('In "%s" loop at %s' % (sourcefile, e)) + raise ValueError('In "%s" loop at %s' % (sourcefile, e)) from None return '#line 1 "%s"\n%s' % (sourcefile, code) @@ -322,7 +322,7 @@ def main(): try: writestr = process_str(allstr) except ValueError as e: - raise ValueError("In %s loop at %s" % (file, e)) + raise ValueError("In %s loop at %s" % (file, e)) from None outfile.write(writestr) diff --git a/numpy/distutils/extension.py b/numpy/distutils/extension.py index 67114ef2e3f7..c90b5d725389 100644 --- a/numpy/distutils/extension.py +++ b/numpy/distutils/extension.py @@ -10,8 +10,8 @@ from distutils.extension import Extension as old_Extension -cxx_ext_re = re.compile(r'.*[.](cpp|cxx|cc)\Z', re.I).match -fortran_pyf_ext_re = re.compile(r'.*[.](f90|f95|f77|for|ftn|f|pyf)\Z', re.I).match +cxx_ext_re = re.compile(r'.*\.(cpp|cxx|cc)\Z', re.I).match +fortran_pyf_ext_re = re.compile(r'.*\.(f90|f95|f77|for|ftn|f|pyf)\Z', re.I).match class Extension(old_Extension): diff --git a/numpy/distutils/fcompiler/__init__.py b/numpy/distutils/fcompiler/__init__.py index 1f340a412913..d7579e976801 100644 --- a/numpy/distutils/fcompiler/__init__.py +++ b/numpy/distutils/fcompiler/__init__.py @@ -610,7 +610,7 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): self.spawn(command, display=display) except DistutilsExecError as e: msg = str(e) - raise CompileError(msg) + raise CompileError(msg) from None def module_options(self, module_dirs, module_build_dir): options = [] @@ -678,7 +678,7 @@ def link(self, target_desc, objects, self.spawn(command) except DistutilsExecError as e: msg = str(e) - raise LinkError(msg) + raise LinkError(msg) from None else: log.debug("skipping %s (up-to-date)", output_filename) @@ -744,7 +744,7 @@ def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir): 'intelvem', 'intelem', 'flang')), ('cygwin.*', ('gnu', 'intelv', 'absoft', 'compaqv', 'intelev', 'gnu95', 'g95')), ('linux.*', ('gnu95', 'intel', 'lahey', 'pg', 'nv', 'absoft', 'nag', 'vast', 'compaq', - 'intele', 'intelem', 'gnu', 'g95', 'pathf95', 'nagfor')), + 'intele', 'intelem', 'gnu', 'g95', 'pathf95', 'nagfor', 'fujitsu')), ('darwin.*', ('gnu95', 'nag', 'absoft', 'ibm', 'intel', 'gnu', 'g95', 'pg')), ('sunos.*', ('sun', 'gnu', 'gnu95', 'g95')), ('irix.*', ('mips', 'gnu', 'gnu95',)), @@ -962,10 +962,10 @@ def dummy_fortran_file(): return name[:-2] -is_f_file = re.compile(r'.*[.](for|ftn|f77|f)\Z', re.I).match -_has_f_header = re.compile(r'-[*]-\s*fortran\s*-[*]-', re.I).search -_has_f90_header = re.compile(r'-[*]-\s*f90\s*-[*]-', re.I).search -_has_fix_header = re.compile(r'-[*]-\s*fix\s*-[*]-', re.I).search +is_f_file = re.compile(r'.*\.(for|ftn|f77|f)\Z', re.I).match +_has_f_header = re.compile(r'-\*-\s*fortran\s*-\*-', re.I).search +_has_f90_header = re.compile(r'-\*-\s*f90\s*-\*-', re.I).search +_has_fix_header = re.compile(r'-\*-\s*fix\s*-\*-', re.I).search _free_f90_start = re.compile(r'[^c*!]\s*[^\s\d\t]', re.I).match def is_free_format(file): @@ -976,7 +976,7 @@ def is_free_format(file): with open(file, encoding='latin1') as f: line = f.readline() n = 10000 # the number of non-comment lines to scan for hints - if _has_f_header(line): + if _has_f_header(line) or _has_fix_header(line): n = 0 elif _has_f90_header(line): n = 0 diff --git a/numpy/distutils/fcompiler/compaq.py b/numpy/distutils/fcompiler/compaq.py index 6ce590c7c821..351a43dd7618 100644 --- a/numpy/distutils/fcompiler/compaq.py +++ b/numpy/distutils/fcompiler/compaq.py @@ -80,18 +80,18 @@ class CompaqVisualFCompiler(FCompiler): except DistutilsPlatformError: pass except AttributeError as e: - if '_MSVCCompiler__root' in str(msg): - print('Ignoring "%s" (I think it is msvccompiler.py bug)' % (msg)) + if '_MSVCCompiler__root' in str(e): + print('Ignoring "%s" (I think it is msvccompiler.py bug)' % (e)) else: raise except IOError as e: if not "vcvarsall.bat" in str(e): print("Unexpected IOError in", __file__) - raise e + raise except ValueError as e: if not "'path'" in str(e): print("Unexpected ValueError in", __file__) - raise e + raise executables = { 'version_cmd' : ['', "/what"], diff --git a/numpy/distutils/fcompiler/fujitsu.py b/numpy/distutils/fcompiler/fujitsu.py new file mode 100644 index 000000000000..ddce67456d18 --- /dev/null +++ b/numpy/distutils/fcompiler/fujitsu.py @@ -0,0 +1,46 @@ +""" +fujitsu + +Supports Fujitsu compiler function. +This compiler is developed by Fujitsu and is used in A64FX on Fugaku. +""" +from numpy.distutils.fcompiler import FCompiler + +compilers = ['FujitsuFCompiler'] + +class FujitsuFCompiler(FCompiler): + compiler_type = 'fujitsu' + description = 'Fujitsu Fortran Compiler' + + possible_executables = ['frt'] + version_pattern = r'frt \(FRT\) (?P[a-z\d.]+)' + # $ frt --version + # frt (FRT) x.x.x yyyymmdd + + executables = { + 'version_cmd' : ["", "--version"], + 'compiler_f77' : ["frt", "-Fixed"], + 'compiler_fix' : ["frt", "-Fixed"], + 'compiler_f90' : ["frt"], + 'linker_so' : ["frt", "-shared"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : ["ranlib"] + } + pic_flags = ['-KPIC'] + module_dir_switch = '-M' + module_include_switch = '-I' + + def get_flags_opt(self): + return ['-O3'] + def get_flags_debug(self): + return ['-g'] + def runtime_library_dir_option(self, dir): + return f'-Wl,-rpath={dir}' + def get_libraries(self): + return ['fj90f', 'fj90i', 'fjsrcinfo'] + +if __name__ == '__main__': + from distutils import log + from numpy.distutils import customized_fcompiler + log.set_verbosity(2) + print(customized_fcompiler('fujitsu').get_version()) diff --git a/numpy/distutils/fcompiler/gnu.py b/numpy/distutils/fcompiler/gnu.py index 0d9d769c2a8c..eac4cbb4779b 100644 --- a/numpy/distutils/fcompiler/gnu.py +++ b/numpy/distutils/fcompiler/gnu.py @@ -32,7 +32,8 @@ def gnu_version_match(self, version_string): """Handle the different versions of GNU fortran compilers""" # Strip warning(s) that may be emitted by gfortran while version_string.startswith('gfortran: warning'): - version_string = version_string[version_string.find('\n') + 1:] + version_string =\ + version_string[version_string.find('\n') + 1:].strip() # Gfortran versions from after 2010 will output a simple string # (usually "x.y", "x.y.z" or "x.y.z-q") for ``-dumpversion``; older @@ -126,7 +127,7 @@ def get_flags_linker_so(self): target = '10.9' s = f'Env. variable MACOSX_DEPLOYMENT_TARGET set to {target}' warnings.warn(s, stacklevel=2) - os.environ['MACOSX_DEPLOYMENT_TARGET'] = target + os.environ['MACOSX_DEPLOYMENT_TARGET'] = str(target) opt.extend(['-undefined', 'dynamic_lookup', '-bundle']) else: opt.append("-shared") diff --git a/numpy/distutils/fcompiler/ibm.py b/numpy/distutils/fcompiler/ibm.py index 4a83682e5165..eff24401a1c3 100644 --- a/numpy/distutils/fcompiler/ibm.py +++ b/numpy/distutils/fcompiler/ibm.py @@ -77,7 +77,7 @@ def get_flags_linker_so(self): fo, new_cfg = make_temp_file(suffix='_xlf.cfg') log.info('Creating '+new_cfg) with open(xlf_cfg, 'r') as fi: - crt1_match = re.compile(r'\s*crt\s*[=]\s*(?P.*)/crt1.o').match + crt1_match = re.compile(r'\s*crt\s*=\s*(?P.*)/crt1.o').match for line in fi: m = crt1_match(line) if m: diff --git a/numpy/distutils/fcompiler/intel.py b/numpy/distutils/fcompiler/intel.py index c7b3c2340df6..f97c5b3483e1 100644 --- a/numpy/distutils/fcompiler/intel.py +++ b/numpy/distutils/fcompiler/intel.py @@ -59,7 +59,8 @@ def get_flags(self): def get_flags_opt(self): # Scipy test failures with -O2 v = self.get_version() mpopt = 'openmp' if v and v < '15' else 'qopenmp' - return ['-fp-model', 'strict', '-O1', '-{}'.format(mpopt)] + return ['-fp-model', 'strict', '-O1', + '-assume', 'minus0', '-{}'.format(mpopt)] def get_flags_arch(self): return [] @@ -119,17 +120,6 @@ class IntelEM64TFCompiler(IntelFCompiler): 'ranlib' : ["ranlib"] } - def get_flags(self): - return ['-fPIC'] - - def get_flags_opt(self): # Scipy test failures with -O2 - v = self.get_version() - mpopt = 'openmp' if v and v < '15' else 'qopenmp' - return ['-fp-model', 'strict', '-O1', '-{}'.format(mpopt)] - - def get_flags_arch(self): - return [] - # Is there no difference in the version string between the above compilers # and the Visual compilers? @@ -174,7 +164,7 @@ def get_flags_debug(self): return ['/4Yb', '/d2'] def get_flags_opt(self): - return ['/O1'] # Scipy test failures with /O2 + return ['/O1', '/assume:minus0'] # Scipy test failures with /O2 def get_flags_arch(self): return ["/arch:IA32", "/QaxSSE3"] diff --git a/numpy/distutils/fcompiler/nag.py b/numpy/distutils/fcompiler/nag.py index 908e724e6536..7df8ffe2ce30 100644 --- a/numpy/distutils/fcompiler/nag.py +++ b/numpy/distutils/fcompiler/nag.py @@ -19,7 +19,7 @@ def get_flags_linker_so(self): def get_flags_opt(self): return ['-O4'] def get_flags_arch(self): - return [''] + return [] class NAGFCompiler(BaseNAGFCompiler): diff --git a/numpy/distutils/from_template.py b/numpy/distutils/from_template.py index 070b7d8b8855..7add44c7679d 100644 --- a/numpy/distutils/from_template.py +++ b/numpy/distutils/from_template.py @@ -206,7 +206,7 @@ def process_str(allstr): return writestr -include_src_re = re.compile(r"(\n|\A)\s*include\s*['\"](?P[\w\d./\\]+[.]src)['\"]", re.I) +include_src_re = re.compile(r"(\n|\A)\s*include\s*['\"](?P[\w\d./\\]+\.src)['\"]", re.I) def resolve_includes(source): d = os.path.dirname(source) diff --git a/numpy/distutils/intelccompiler.py b/numpy/distutils/intelccompiler.py index 0388ad577518..0fa1c11dd676 100644 --- a/numpy/distutils/intelccompiler.py +++ b/numpy/distutils/intelccompiler.py @@ -58,7 +58,7 @@ def __init__(self, verbose=0, dry_run=0, force=0): v = self.get_version() mpopt = 'openmp' if v and v < '15' else 'qopenmp' - self.cc_exe = ('icc -m64 -fPIC -fp-model strict -O3 ' + self.cc_exe = ('icc -std=c99 -m64 -fPIC -fp-model strict -O3 ' '-fomit-frame-pointer -{}').format(mpopt) compiler = self.cc_exe diff --git a/numpy/distutils/mingw32ccompiler.py b/numpy/distutils/mingw32ccompiler.py index 3358695a8254..4681d403b4e3 100644 --- a/numpy/distutils/mingw32ccompiler.py +++ b/numpy/distutils/mingw32ccompiler.py @@ -566,7 +566,7 @@ def msvc_manifest_xml(maj, min): fullver = _MSVCRVER_TO_FULLVER[str(maj * 10 + min)] except KeyError: raise ValueError("Version %d,%d of MSVCRT not supported yet" % - (maj, min)) + (maj, min)) from None # Don't be fooled, it looks like an XML, but it is not. In particular, it # should not have any space before starting, and its size should be # divisible by 4, most likely for alignment constraints when the xml is diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index a8e19d52c56b..e797745e12db 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -10,13 +10,10 @@ import multiprocessing import textwrap import importlib.util +from threading import local as tlocal import distutils from distutils.errors import DistutilsError -try: - from threading import local as tlocal -except ImportError: - from dummy_threading import local as tlocal # stores temporary directory of each thread to only create one per thread _tdata = tlocal() @@ -428,9 +425,9 @@ def msvc_runtime_major(): ######################### #XXX need support for .C that is also C++ -cxx_ext_match = re.compile(r'.*[.](cpp|cxx|cc)\Z', re.I).match -fortran_ext_match = re.compile(r'.*[.](f90|f95|f77|for|ftn|f)\Z', re.I).match -f90_ext_match = re.compile(r'.*[.](f90|f95)\Z', re.I).match +cxx_ext_match = re.compile(r'.*\.(cpp|cxx|cc)\Z', re.I).match +fortran_ext_match = re.compile(r'.*\.(f90|f95|f77|for|ftn|f)\Z', re.I).match +f90_ext_match = re.compile(r'.*\.(f90|f95)\Z', re.I).match f90_module_name_match = re.compile(r'\s*module\s*(?P[\w_]+)', re.I).match def _get_f90_modules(source): """Return a list of Fortran f90 module names that @@ -1968,6 +1965,13 @@ def get_version(self, version_file=None, version_variable=None): version = getattr(version_module, a, None) if version is not None: break + + # Try if versioneer module + try: + version = version_module.get_versions()['version'] + except AttributeError: + pass + if version is not None: break diff --git a/numpy/distutils/system_info.py b/numpy/distutils/system_info.py index 13f9da0fb684..82e864a5c3e2 100644 --- a/numpy/distutils/system_info.py +++ b/numpy/distutils/system_info.py @@ -114,6 +114,19 @@ x11_info:x11 xft_info:xft +Note that blas_opt_info and lapack_opt_info honor the NPY_BLAS_ORDER +and NPY_LAPACK_ORDER environment variables to determine the order in which +specific BLAS and LAPACK libraries are searched for. + +This search (or autodetection) can be bypassed by defining the environment +variables NPY_BLAS_LIBS and NPY_LAPACK_LIBS, which should then contain the +exact linker flags to use (language will be set to F77). Building against +Netlib BLAS/LAPACK or stub files, in order to be able to switch BLAS and LAPACK +implementations at runtime. If using this to build NumPy itself, it is +recommended to also define NPY_CBLAS_LIBS (assuming your BLAS library has a +CBLAS interface) to enable CBLAS usage for matrix multiplication (unoptimized +otherwise). + Example: ---------- [DEFAULT] @@ -314,7 +327,7 @@ def add_system_root(library_root): '/opt/local/lib', '/sw/lib'], platform_bits) default_runtime_dirs = [] default_include_dirs = ['/usr/local/include', - '/opt/include', '/usr/include', + '/opt/include', # path of umfpack under macports '/opt/local/include/ufsparse', '/opt/local/include', '/sw/include', @@ -323,8 +336,7 @@ def add_system_root(library_root): default_x11_lib_dirs = libpaths(['/usr/X11R6/lib', '/usr/X11/lib', '/usr/lib'], platform_bits) - default_x11_include_dirs = ['/usr/X11R6/include', '/usr/X11/include', - '/usr/include'] + default_x11_include_dirs = ['/usr/X11R6/include', '/usr/X11/include'] if os.path.exists('/usr/lib/X11'): globbed_x11_dir = glob('/usr/lib/*/libX11.so') @@ -363,22 +375,6 @@ def add_system_root(library_root): so_ext = get_shared_lib_extension() -def is_symlink_to_accelerate(filename): - accelpath = '/System/Library/Frameworks/Accelerate.framework' - return (sys.platform == 'darwin' and os.path.islink(filename) and - os.path.realpath(filename).startswith(accelpath)) - - -_accel_msg = ( - 'Found {filename}, but that file is a symbolic link to the ' - 'MacOS Accelerate framework, which is not supported by NumPy. ' - 'You must configure the build to use a different optimized library, ' - 'or disable the use of optimized BLAS and LAPACK by setting the ' - 'environment variables NPY_BLAS_ORDER="" and NPY_LAPACK_ORDER="" ' - 'before building NumPy.' -) - - def get_standard_file(fname): """Returns a list of files named 'fname' from 1) System-wide directory (directory-location of this module) @@ -527,6 +523,7 @@ def get_info(name, notfound_action=0): 'blis': blis_info, # use blas_opt instead 'lapack_mkl': lapack_mkl_info, # use lapack_opt instead 'blas_mkl': blas_mkl_info, # use blas_opt instead + 'accelerate': accelerate_info, # use blas_opt instead 'openblas64_': openblas64__info, 'openblas64__lapack': openblas64__lapack_info, 'openblas_ilp64': openblas_ilp64_info, @@ -1017,9 +1014,6 @@ def _find_lib(self, lib_dir, lib, exts): for prefix in lib_prefixes: p = self.combine_paths(lib_dir, prefix + lib + ext) if p: - # p[0] is the full path to the binary library file. - if is_symlink_to_accelerate(p[0]): - raise RuntimeError(_accel_msg.format(filename=p[0])) break if p: assert len(p) == 1 @@ -1549,6 +1543,9 @@ def calc_info(self): class lapack_src_info(system_info): + # LAPACK_SRC is deprecated, please do not use this! + # Build or install a BLAS library via your package manager or from + # source separately. section = 'lapack_src' dir_env_var = 'LAPACK_SRC' notfounderror = LapackSrcNotFoundError @@ -1751,8 +1748,10 @@ def get_atlas_version(**config): class lapack_opt_info(system_info): notfounderror = LapackNotFoundError + # List of all known LAPACK libraries, in the default order - lapack_order = ['mkl', 'openblas', 'flame', 'atlas', 'lapack'] + lapack_order = ['mkl', 'openblas', 'flame', + 'accelerate', 'atlas', 'lapack'] order_env_var_name = 'NPY_LAPACK_ORDER' def _calc_info_mkl(self): @@ -1845,6 +1844,16 @@ def _calc_info_lapack(self): return True return False + def _calc_info_from_envvar(self): + info = {} + info['language'] = 'f77' + info['libraries'] = [] + info['include_dirs'] = [] + info['define_macros'] = [] + info['extra_link_args'] = os.environ['NPY_LAPACK_LIBS'].split() + self.set_info(**info) + return True + def _calc_info(self, name): return getattr(self, '_calc_info_{}'.format(name))() @@ -1855,6 +1864,12 @@ def calc_info(self): "LAPACK order has unacceptable " "values: {}".format(unknown_order)) + if 'NPY_LAPACK_LIBS' in os.environ: + # Bypass autodetection, set language to F77 and use env var linker + # flags directly + self._calc_info_from_envvar() + return + for lapack in lapack_order: if self._calc_info(lapack): return @@ -1911,7 +1926,9 @@ class lapack64__opt_info(lapack_ilp64_opt_info): class blas_opt_info(system_info): notfounderror = BlasNotFoundError # List of all known BLAS libraries, in the default order - blas_order = ['mkl', 'blis', 'openblas', 'atlas', 'blas'] + + blas_order = ['mkl', 'blis', 'openblas', + 'accelerate', 'atlas', 'blas'] order_env_var_name = 'NPY_BLAS_ORDER' def _calc_info_mkl(self): @@ -1977,6 +1994,20 @@ def _calc_info_blas(self): self.set_info(**info) return True + def _calc_info_from_envvar(self): + info = {} + info['language'] = 'f77' + info['libraries'] = [] + info['include_dirs'] = [] + info['define_macros'] = [] + info['extra_link_args'] = os.environ['NPY_BLAS_LIBS'].split() + if 'NPY_CBLAS_LIBS' in os.environ: + info['define_macros'].append(('HAVE_CBLAS', None)) + info['extra_link_args'].extend( + os.environ['NPY_CBLAS_LIBS'].split()) + self.set_info(**info) + return True + def _calc_info(self, name): return getattr(self, '_calc_info_{}'.format(name))() @@ -1985,6 +2016,12 @@ def calc_info(self): if len(unknown_order) > 0: raise ValueError("blas_opt_info user defined BLAS order has unacceptable values: {}".format(unknown_order)) + if 'NPY_BLAS_LIBS' in os.environ: + # Bypass autodetection, set language to F77 and use env var linker + # flags directly + self._calc_info_from_envvar() + return + for blas in blas_order: if self._calc_info(blas): return @@ -2443,8 +2480,6 @@ def calc_info(self): 'accelerate' in libraries): if intel: args.extend(['-msse3']) - else: - args.extend(['-faltivec']) args.extend([ '-I/System/Library/Frameworks/vecLib.framework/Headers']) link_args.extend(['-Wl,-framework', '-Wl,Accelerate']) @@ -2453,8 +2488,6 @@ def calc_info(self): 'veclib' in libraries): if intel: args.extend(['-msse3']) - else: - args.extend(['-faltivec']) args.extend([ '-I/System/Library/Frameworks/vecLib.framework/Headers']) link_args.extend(['-Wl,-framework', '-Wl,vecLib']) @@ -2468,6 +2501,9 @@ def calc_info(self): return class blas_src_info(system_info): + # BLAS_SRC is deprecated, please do not use this! + # Build or install a BLAS library via your package manager or from + # source separately. section = 'blas_src' dir_env_var = 'BLAS_SRC' notfounderror = BlasSrcNotFoundError diff --git a/numpy/distutils/tests/test_build_ext.py b/numpy/distutils/tests/test_build_ext.py new file mode 100644 index 000000000000..c007159f520e --- /dev/null +++ b/numpy/distutils/tests/test_build_ext.py @@ -0,0 +1,72 @@ +'''Tests for numpy.distutils.build_ext.''' + +import os +import subprocess +import sys +from textwrap import indent, dedent +import pytest + +@pytest.mark.slow +def test_multi_fortran_libs_link(tmp_path): + ''' + Ensures multiple "fake" static libraries are correctly linked. + see gh-18295 + ''' + + # We need to make sure we actually have an f77 compiler. + # This is nontrivial, so we'll borrow the utilities + # from f2py tests: + from numpy.f2py.tests.util import has_f77_compiler + if not has_f77_compiler(): + pytest.skip('No F77 compiler found') + + # make some dummy sources + with open(tmp_path / '_dummy1.f', 'w') as fid: + fid.write(indent(dedent('''\ + FUNCTION dummy_one() + RETURN + END FUNCTION'''), prefix=' '*6)) + with open(tmp_path / '_dummy2.f', 'w') as fid: + fid.write(indent(dedent('''\ + FUNCTION dummy_two() + RETURN + END FUNCTION'''), prefix=' '*6)) + with open(tmp_path / '_dummy.c', 'w') as fid: + # doesn't need to load - just needs to exist + fid.write('int PyInit_dummyext;') + + # make a setup file + with open(tmp_path / 'setup.py', 'w') as fid: + srctree = os.path.join(os.path.dirname(__file__), '..', '..', '..') + fid.write(dedent(f'''\ + def configuration(parent_package="", top_path=None): + from numpy.distutils.misc_util import Configuration + config = Configuration("", parent_package, top_path) + config.add_library("dummy1", sources=["_dummy1.f"]) + config.add_library("dummy2", sources=["_dummy2.f"]) + config.add_extension("dummyext", sources=["_dummy.c"], libraries=["dummy1", "dummy2"]) + return config + + + if __name__ == "__main__": + import sys + sys.path.insert(0, r"{srctree}") + from numpy.distutils.core import setup + setup(**configuration(top_path="").todict())''')) + + # build the test extensino and "install" into a temporary directory + build_dir = tmp_path + subprocess.check_call([sys.executable, 'setup.py', 'build', 'install', + '--prefix', str(tmp_path / 'installdir'), + '--record', str(tmp_path / 'tmp_install_log.txt'), + ], + cwd=str(build_dir), + ) + # get the path to the so + so = None + with open(tmp_path /'tmp_install_log.txt') as fid: + for line in fid: + if 'dummyext' in line: + so = line.strip() + break + assert so is not None diff --git a/numpy/distutils/tests/test_ccompiler_opt.py b/numpy/distutils/tests/test_ccompiler_opt.py index a789be1ea99b..9c54ed66bbd9 100644 --- a/numpy/distutils/tests/test_ccompiler_opt.py +++ b/numpy/distutils/tests/test_ccompiler_opt.py @@ -73,7 +73,7 @@ def dist_info(self): def dist_log(*args, stderr=False): pass -class _Test_CCompilerOpt(object): +class _Test_CCompilerOpt: arch = None # x86_64 cc = None # gcc @@ -82,7 +82,7 @@ def setup(self): self._opt = None def nopt(self, *args, **kwargs): - FakeCCompilerOpt.fake_info = self.arch + '_' + self.cc + FakeCCompilerOpt.fake_info = (self.arch, self.cc, "") return FakeCCompilerOpt(*args, **kwargs) def opt(self): @@ -112,7 +112,7 @@ def get_targets(self, targets, groups, **kwargs): gflags = {} fake_objects = opt.try_dispatch([file]) for source, flags in fake_objects: - gtar = source.split('.')[1:-1] + gtar = path.basename(source).split('.')[1:-1] glen = len(gtar) if glen == 0: gtar = "baseline" diff --git a/numpy/distutils/tests/test_ccompiler_opt_conf.py b/numpy/distutils/tests/test_ccompiler_opt_conf.py index 244748e5889a..09c1fad40c54 100644 --- a/numpy/distutils/tests/test_ccompiler_opt_conf.py +++ b/numpy/distutils/tests/test_ccompiler_opt_conf.py @@ -19,7 +19,7 @@ ) class FakeCCompilerOpt(CCompilerOpt): - fake_info = "" + fake_info = ("arch", "compiler", "extra_args") def __init__(self, *args, **kwargs): CCompilerOpt.__init__(self, None, **kwargs) def dist_compile(self, sources, flags, **kwargs): @@ -169,7 +169,7 @@ def setup(self): def test_features(self): for arch, compilers in arch_compilers.items(): for cc in compilers: - FakeCCompilerOpt.fake_info = arch + cc + FakeCCompilerOpt.fake_info = (arch, cc, "") _TestConfFeatures() if is_standalone: diff --git a/numpy/distutils/tests/test_system_info.py b/numpy/distutils/tests/test_system_info.py index ec15126f7f7b..b722281ad531 100644 --- a/numpy/distutils/tests/test_system_info.py +++ b/numpy/distutils/tests/test_system_info.py @@ -269,7 +269,7 @@ def test_overrides(self): # But if we copy the values to a '[mkl]' section the value # is correct with open(cfg, 'r') as fid: - mkl = fid.read().replace('ALL', 'mkl') + mkl = fid.read().replace('[ALL]', '[mkl]', 1) with open(cfg, 'w') as fid: fid.write(mkl) info = mkl_info() @@ -277,7 +277,7 @@ def test_overrides(self): # Also, the values will be taken from a section named '[DEFAULT]' with open(cfg, 'r') as fid: - dflt = fid.read().replace('mkl', 'DEFAULT') + dflt = fid.read().replace('[mkl]', '[DEFAULT]', 1) with open(cfg, 'w') as fid: fid.write(dflt) info = mkl_info() diff --git a/numpy/distutils/unixccompiler.py b/numpy/distutils/unixccompiler.py index 0cd2d243ee83..fb91f178935f 100644 --- a/numpy/distutils/unixccompiler.py +++ b/numpy/distutils/unixccompiler.py @@ -54,7 +54,7 @@ def UnixCCompiler__compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts extra_postargs, display = display) except DistutilsExecError as e: msg = str(e) - raise CompileError(msg) + raise CompileError(msg) from None # add commandline flags to dependency file if deps: @@ -131,7 +131,7 @@ def UnixCCompiler_create_static_lib(self, objects, output_libname, display = display) except DistutilsExecError as e: msg = str(e) - raise LibError(msg) + raise LibError(msg) from None else: log.debug("skipping %s (up-to-date)", output_filename) return diff --git a/numpy/emath.pyi b/numpy/emath.pyi deleted file mode 100644 index 032ec9505ad1..000000000000 --- a/numpy/emath.pyi +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Any - -sqrt: Any -log: Any -log2: Any -logn: Any -log10: Any -power: Any -arccos: Any -arcsin: Any -arctanh: Any diff --git a/numpy/f2py/__init__.pyi b/numpy/f2py/__init__.pyi index 602517957ca3..3f7231fed1d0 100644 --- a/numpy/f2py/__init__.pyi +++ b/numpy/f2py/__init__.pyi @@ -1,5 +1,18 @@ -from typing import Any +from typing import Any, List -run_main: Any -compile: Any -f2py_testing: Any +from numpy.f2py import ( + f2py_testing as f2py_testing, +) + +__all__: List[str] + +def run_main(comline_list): ... +def compile( + source, + modulename=..., + extra_args=..., + verbose=..., + source_fn=..., + extension=..., + full_output=..., +): ... diff --git a/numpy/f2py/__version__.py b/numpy/f2py/__version__.py index 104c2e1a899e..e20d7c1dbb38 100644 --- a/numpy/f2py/__version__.py +++ b/numpy/f2py/__version__.py @@ -1,8 +1 @@ -major = 2 - -try: - from __svn_version__ import version - version_info = (major, version) - version = '%s_%s' % version_info -except (ImportError, ValueError): - version = str(major) +from numpy.version import version diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 80b150655eb0..5250fea84031 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -257,6 +257,7 @@ def ismodule(rout): def isfunction(rout): return 'block' in rout and 'function' == rout['block'] + def isfunction_wrap(rout): if isintent_c(rout): return 0 @@ -284,6 +285,10 @@ def hasassumedshape(rout): return False +def requiresf90wrapper(rout): + return ismoduleroutine(rout) or hasassumedshape(rout) + + def isroutine(rout): return isfunction(rout) or issubroutine(rout) diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index fabbfc4c24ac..fe0d4a52bd16 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -11,8 +11,6 @@ Pearu Peterson """ -__version__ = "$Revision: 1.60 $"[10:-1] - from . import __version__ f2py_version = __version__.version @@ -309,7 +307,7 @@ def getstrlength(var): len = a['*'] elif 'len' in a: len = a['len'] - if re.match(r'\(\s*([*]|[:])\s*\)', len) or re.match(r'([*]|[:])', len): + if re.match(r'\(\s*(\*|:)\s*\)', len) or re.match(r'(\*|:)', len): if isintent_hide(var): errmess('getstrlength:intent(hide): expected a string with defined length but got: %s\n' % ( repr(var))) diff --git a/numpy/f2py/cb_rules.py b/numpy/f2py/cb_rules.py index 3068dc897b10..62aa2fca9e56 100644 --- a/numpy/f2py/cb_rules.py +++ b/numpy/f2py/cb_rules.py @@ -70,7 +70,8 @@ /*typedef #rctype#(*#name#_typedef)(#optargs_td##args_td##strarglens_td##noargs#);*/ #static# #rctype# #callbackname# (#optargs##args##strarglens##noargs#) { - #name#_t *cb; + #name#_t cb_local = { NULL, NULL, 0 }; + #name#_t *cb = NULL; PyTupleObject *capi_arglist = NULL; PyObject *capi_return = NULL; PyObject *capi_tmp = NULL; @@ -82,12 +83,17 @@ f2py_cb_start_clock(); #endif cb = get_active_#name#(); + if (cb == NULL) { + capi_longjmp_ok = 0; + cb = &cb_local; + } capi_arglist = cb->args_capi; CFUNCSMESS(\"cb:Call-back function #name# (maxnofargs=#maxnofargs#(-#nofoptargs#))\\n\"); CFUNCSMESSPY(\"cb:#name#_capi=\",cb->capi); if (cb->capi==NULL) { capi_longjmp_ok = 0; cb->capi = PyObject_GetAttrString(#modulename#_module,\"#argname#\"); + CFUNCSMESSPY(\"cb:#name#_capi=\",cb->capi); } if (cb->capi==NULL) { PyErr_SetString(#modulename#_error,\"cb: Callback #argname# not defined (as an argument or module #modulename# attribute).\\n\"); @@ -342,6 +348,7 @@ isarray: '#ctype# *', isstring: '#ctype#' }, + 'need': {l_or(isscalar, isarray, isstring): '#ctype#'}, # untested with multiple args 'strarglens': {isstring: ',int #varname_i#_cb_len'}, 'strarglens_td': {isstring: ',int'}, # untested with multiple args diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 26b43e7e6964..f403a66b5d7b 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -545,11 +545,18 @@ """ cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ #ifndef F2PY_THREAD_LOCAL_DECL -#if defined(_MSC_VER) +#if defined(_MSC_VER) \\ + || defined(_WIN32) || defined(_WIN64) \\ + || defined(__MINGW32__) || defined(__MINGW64__) #define F2PY_THREAD_LOCAL_DECL __declspec(thread) #elif defined(__STDC_VERSION__) \\ && (__STDC_VERSION__ >= 201112L) \\ - && !defined(__STDC_NO_THREADS__) + && !defined(__STDC_NO_THREADS__) \\ + && (!defined(__GLIBC__) || __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ > 12)) +/* __STDC_NO_THREADS__ was first defined in a maintenance release of glibc 2.12, + see https://lists.gnu.org/archive/html/commit-hurd/2012-07/msg00180.html, + so `!defined(__STDC_NO_THREADS__)` may give false positive for the existence + of `threads.h` when using an older release of glibc 2.12 */ #include #define F2PY_THREAD_LOCAL_DECL thread_local #elif defined(__GNUC__) \\ @@ -1018,6 +1025,7 @@ } (*v).r = ((npy_cdouble *)PyArray_DATA(arr))->real; (*v).i = ((npy_cdouble *)PyArray_DATA(arr))->imag; + Py_DECREF(arr); return 1; } /* Python does not provide PyNumber_Complex function :-( */ diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 90483e55be83..937d8bc723bd 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -13,8 +13,6 @@ Pearu Peterson """ -__version__ = "$Revision: 1.19 $"[10:-1] - from . import __version__ f2py_version = __version__.version diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index 2e95e45969e5..6453bbecb159 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -43,7 +43,8 @@ 'implicit','externals','interfaced','common','sortvars', 'commonvars','note']} B['block'] = 'interface' | 'function' | 'subroutine' | 'module' | - 'program' | 'block data' | 'type' | 'pythonmodule' + 'program' | 'block data' | 'type' | 'pythonmodule' | + 'abstract interface' B['body'] --- list containing `subblocks' with the same structure as `blocks' B['parent_block'] --- dictionary of a parent block: C['body'][]['parent_block'] is C @@ -138,6 +139,7 @@ The above may be solved by creating appropriate preprocessor program, for example. """ +import io import sys import string import fileinput @@ -294,10 +296,10 @@ def getextension(name): return '' return name[i + 1:] -is_f_file = re.compile(r'.*[.](for|ftn|f77|f)\Z', re.I).match -_has_f_header = re.compile(r'-[*]-\s*fortran\s*-[*]-', re.I).search -_has_f90_header = re.compile(r'-[*]-\s*f90\s*-[*]-', re.I).search -_has_fix_header = re.compile(r'-[*]-\s*fix\s*-[*]-', re.I).search +is_f_file = re.compile(r'.*\.(for|ftn|f77|f)\Z', re.I).match +_has_f_header = re.compile(r'-\*-\s*fortran\s*-\*-', re.I).search +_has_f90_header = re.compile(r'-\*-\s*f90\s*-\*-', re.I).search +_has_fix_header = re.compile(r'-\*-\s*fix\s*-\*-', re.I).search _free_f90_start = re.compile(r'[^c*]\s*[^\s\d\t]', re.I).match @@ -341,7 +343,9 @@ def readfortrancode(ffile, dowithline=show, istop=1): if ffile == []: return localdolowercase = dolowercase - cont = 0 + # cont: set to True when the content of the last line read + # indicates statement continuation + cont = False finalline = '' ll = '' includeline = re.compile( @@ -392,14 +396,26 @@ def readfortrancode(ffile, dowithline=show, istop=1): if rl[:5].lower() == '!f2py': # f2py directive l, _ = split_by_unquoted(l + 4 * ' ' + rl[5:], '!') if l.strip() == '': # Skip empty line - cont = 0 + if sourcecodeform == 'free': + # In free form, a statement continues in the next line + # that is not a comment line [3.3.2.4^1], lines with + # blanks are comment lines [3.3.2.3^1]. Hence, the + # line continuation flag must retain its state. + pass + else: + # In fixed form, statement continuation is determined + # by a non-blank character at the 6-th position. Empty + # line indicates a start of a new statement + # [3.3.3.3^1]. Hence, the line continuation flag must + # be reset. + cont = False continue if sourcecodeform == 'fix': if l[0] in ['*', 'c', '!', 'C', '#']: if l[1:5].lower() == 'f2py': # f2py directive l = ' ' + l[5:] else: # Skip comment line - cont = 0 + cont = False continue elif strictf77: if len(l) > 72: @@ -553,15 +569,15 @@ def readfortrancode(ffile, dowithline=show, istop=1): beginpattern77 = re.compile( beforethisafter % ('', groupbegins77, groupbegins77, '.*'), re.I), 'begin' groupbegins90 = groupbegins77 + \ - r'|module(?!\s*procedure)|python\s*module|interface|type(?!\s*\()' + r'|module(?!\s*procedure)|python\s*module|(abstract|)\s*interface|' + \ + r'type(?!\s*\()' beginpattern90 = re.compile( beforethisafter % ('', groupbegins90, groupbegins90, '.*'), re.I), 'begin' groupends = (r'end|endprogram|endblockdata|endmodule|endpythonmodule|' r'endinterface|endsubroutine|endfunction') endpattern = re.compile( beforethisafter % ('', groupends, groupends, r'[\w\s]*'), re.I), 'end' -# endifs='end\s*(if|do|where|select|while|forall)' -endifs = r'(end\s*(if|do|where|select|while|forall))|(module\s*procedure)' +endifs = r'(end\s*(if|do|where|select|while|forall|associate|block|critical|enum|team))|(module\s*procedure)' endifpattern = re.compile( beforethisafter % (r'[\w]*?', endifs, endifs, r'[\w\s]*'), re.I), 'endif' # @@ -636,7 +652,7 @@ def _simplifyargs(argsline): a.append(n) return ','.join(a) -crackline_re_1 = re.compile(r'\s*(?P\b[a-z]+[\w]*\b)\s*[=].*', re.I) +crackline_re_1 = re.compile(r'\s*(?P\b[a-z]+\w*\b)\s*=.*', re.I) def crackline(line, reset=0): @@ -868,7 +884,7 @@ def appenddecl(decl, decl2, force=1): return decl selectpattern = re.compile( - r'\s*(?P(@\(@.*?@\)@|[*][\d*]+|[*]\s*@\(@.*?@\)@|))(?P.*)\Z', re.I) + r'\s*(?P(@\(@.*?@\)@|\*[\d*]+|\*\s*@\(@.*?@\)@|))(?P.*)\Z', re.I) nameargspattern = re.compile( r'\s*(?P\b[\w$]+\b)\s*(@\(@\s*(?P[\w\s,]*)\s*@\)@|)\s*((result(\s*@\(@\s*(?P\b[\w$]+\b)\s*@\)@|))|(bind\s*@\(@\s*(?P.*)\s*@\)@))*\s*\Z', re.I) callnameargspattern = re.compile( @@ -928,15 +944,17 @@ def analyzeline(m, case, line): block = block.lower() if re.match(r'block\s*data', block, re.I): block = 'block data' - if re.match(r'python\s*module', block, re.I): + elif re.match(r'python\s*module', block, re.I): block = 'python module' + elif re.match(r'abstract\s*interface', block, re.I): + block = 'abstract interface' name, args, result, bind = _resolvenameargspattern(m.group('after')) if name is None: if block == 'block data': name = '_BLOCK_DATA_' else: name = '' - if block not in ['interface', 'block data']: + if block not in ['interface', 'block data', 'abstract interface']: outmess('analyzeline: No name/args pattern found for line.\n') previous_context = (block, name, groupcounter) @@ -970,7 +988,7 @@ def analyzeline(m, case, line): if f77modulename and neededmodule == -1 and groupcounter <= 1: neededmodule = groupcounter + 2 needmodule = 1 - if block != 'interface': + if block not in ['interface', 'abstract interface']: needinterface = 1 # Create new block(s) groupcounter = groupcounter + 1 @@ -1010,7 +1028,7 @@ def analyzeline(m, case, line): groupname[groupcounter] = block groupcache[groupcounter]['block'] = block if not name: - name = 'unknown_' + block + name = 'unknown_' + block.replace(' ', '_') groupcache[groupcounter]['prefix'] = m.group('before') groupcache[groupcounter]['name'] = rmbadname1(name) groupcache[groupcounter]['result'] = result @@ -1389,7 +1407,7 @@ def analyzeline(m, case, line): previous_context = ('common', bn, groupcounter) elif case == 'use': m1 = re.match( - r'\A\s*(?P\b[\w]+\b)\s*((,(\s*\bonly\b\s*:|(?P))\s*(?P.*))|)\s*\Z', m.group('after'), re.I) + r'\A\s*(?P\b\w+\b)\s*((,(\s*\bonly\b\s*:|(?P))\s*(?P.*))|)\s*\Z', m.group('after'), re.I) if m1: mm = m1.groupdict() if 'use' not in groupcache[groupcounter]: @@ -1406,7 +1424,7 @@ def analyzeline(m, case, line): for l in ll: if '=' in l: m2 = re.match( - r'\A\s*(?P\b[\w]+\b)\s*=\s*>\s*(?P\b[\w]+\b)\s*\Z', l, re.I) + r'\A\s*(?P\b\w+\b)\s*=\s*>\s*(?P\b\w+\b)\s*\Z', l, re.I) if m2: rl[m2.group('local').strip()] = m2.group( 'use').strip() @@ -1482,15 +1500,15 @@ def cracktypespec0(typespec, ll): ll = ll[i + 2:] return typespec, selector, attr, ll ##### -namepattern = re.compile(r'\s*(?P\b[\w]+\b)\s*(?P.*)\s*\Z', re.I) +namepattern = re.compile(r'\s*(?P\b\w+\b)\s*(?P.*)\s*\Z', re.I) kindselector = re.compile( - r'\s*(\(\s*(kind\s*=)?\s*(?P.*)\s*\)|[*]\s*(?P.*?))\s*\Z', re.I) + r'\s*(\(\s*(kind\s*=)?\s*(?P.*)\s*\)|\*\s*(?P.*?))\s*\Z', re.I) charselector = re.compile( - r'\s*(\((?P.*)\)|[*]\s*(?P.*))\s*\Z', re.I) + r'\s*(\((?P.*)\)|\*\s*(?P.*))\s*\Z', re.I) lenkindpattern = re.compile( r'\s*(kind\s*=\s*(?P.*?)\s*(@,@\s*len\s*=\s*(?P.*)|)|(len\s*=\s*|)(?P.*?)\s*(@,@\s*(kind\s*=\s*|)(?P.*)|))\s*\Z', re.I) lenarraypattern = re.compile( - r'\s*(@\(@\s*(?!/)\s*(?P.*?)\s*@\)@\s*[*]\s*(?P.*?)|([*]\s*(?P.*?)|)\s*(@\(@\s*(?!/)\s*(?P.*?)\s*@\)@|))\s*(=\s*(?P.*?)|(@\(@|)/\s*(?P.*?)\s*/(@\)@|)|)\s*\Z', re.I) + r'\s*(@\(@\s*(?!/)\s*(?P.*?)\s*@\)@\s*\*\s*(?P.*?)|(\*\s*(?P.*?)|)\s*(@\(@\s*(?!/)\s*(?P.*?)\s*@\)@|))\s*(=\s*(?P.*?)|(@\(@|)/\s*(?P.*?)\s*/(@\)@|)|)\s*\Z', re.I) def removespaces(expr): @@ -1611,6 +1629,10 @@ def updatevars(typespec, selector, attrspec, entitydecl): edecl['charselector'] = copy.copy(charselect) edecl['typename'] = typename edecl['attrspec'] = copy.copy(attrspec) + if 'external' in (edecl.get('attrspec') or []) and e in groupcache[groupcounter]['args']: + if 'externals' not in groupcache[groupcounter]: + groupcache[groupcounter]['externals'] = [] + groupcache[groupcounter]['externals'].append(e) if m.group('after'): m1 = lenarraypattern.match(markouterparen(m.group('after'))) if m1: @@ -2071,7 +2093,7 @@ def analyzebody(block, args, tab=''): else: as_ = args b = postcrack(b, as_, tab=tab + '\t') - if b['block'] == 'interface' and not b['body']: + if b['block'] in ['interface', 'abstract interface'] and not b['body']: if 'f2pyenhancements' not in b: continue if b['block'].replace(' ', '') == 'pythonmodule': @@ -2605,7 +2627,7 @@ def analyzevars(block): params = get_parameters(vars, get_useparameters(block)) dep_matches = {} - name_match = re.compile(r'\w[\w\d_$]*').match + name_match = re.compile(r'[A-Za-z][\w$]*').match for v in list(vars.keys()): m = name_match(v) if m: @@ -2988,10 +3010,10 @@ def analyzeargs(block): block['vars'][block['result']] = {} return block -determineexprtype_re_1 = re.compile(r'\A\(.+?[,].+?\)\Z', re.I) -determineexprtype_re_2 = re.compile(r'\A[+-]?\d+(_(?P[\w]+)|)\Z', re.I) +determineexprtype_re_1 = re.compile(r'\A\(.+?,.+?\)\Z', re.I) +determineexprtype_re_2 = re.compile(r'\A[+-]?\d+(_(?P\w+)|)\Z', re.I) determineexprtype_re_3 = re.compile( - r'\A[+-]?[\d.]+[\d+\-de.]*(_(?P[\w]+)|)\Z', re.I) + r'\A[+-]?[\d.]+[-\d+de.]*(_(?P\w+)|)\Z', re.I) determineexprtype_re_4 = re.compile(r'\A\(.*\)\Z', re.I) determineexprtype_re_5 = re.compile(r'\A(?P\w+)\s*\(.*?\)\s*\Z', re.I) @@ -3109,7 +3131,7 @@ def crack2fortrangen(block, tab='\n', as_interface=False): result = ' result (%s)' % block['result'] if block['result'] not in argsl: argsl.append(block['result']) - body = crack2fortrangen(block['body'], tab + tabchar) + body = crack2fortrangen(block['body'], tab + tabchar, as_interface=as_interface) vars = vars2fortran( block, block['vars'], argsl, tab + tabchar, as_interface=as_interface) mess = '' @@ -3227,8 +3249,13 @@ def vars2fortran(block, vars, args, tab='', as_interface=False): show(vars) outmess('vars2fortran: No definition for argument "%s".\n' % a) continue - if a == block['name'] and not block['block'] == 'function': - continue + if a == block['name']: + if block['block'] != 'function' or block.get('result'): + # 1) skip declaring a variable that name matches with + # subroutine name + # 2) skip declaring function when its type is + # declared via `result` construction + continue if 'typespec' not in vars[a]: if 'attrspec' in vars[a] and 'external' in vars[a]['attrspec']: if a in args: diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index be2c345d1a99..a14f068f15dd 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -29,18 +29,14 @@ from . import capi_maps f2py_version = __version__.version +numpy_version = __version__.version errmess = sys.stderr.write # outmess=sys.stdout.write show = pprint.pprint outmess = auxfuncs.outmess -try: - from numpy import __version__ as numpy_version -except ImportError: - numpy_version = 'N/A' - -__usage__ = """\ -Usage: +__usage__ =\ +f"""Usage: 1) To construct extension module sources: @@ -97,8 +93,8 @@ --[no-]latex-doc Create (or not) module.tex. Default is --no-latex-doc. --short-latex Create 'incomplete' LaTeX document (without commands - \\documentclass, \\tableofcontents, and \\begin{document}, - \\end{document}). + \\documentclass, \\tableofcontents, and \\begin{{document}}, + \\end{{document}}). --[no-]rest-doc Create (or not) module.rst. Default is --no-rest-doc. @@ -167,12 +163,12 @@ array. Integer sets the threshold for array sizes when a message should be shown. -Version: %s -numpy Version: %s +Version: {f2py_version} +numpy Version: {numpy_version} Requires: Python 3.5 or higher. License: NumPy license (see LICENSE.txt in the NumPy source code) Copyright 1999 - 2011 Pearu Peterson all rights reserved. -http://cens.ioc.ee/projects/f2py2e/""" % (f2py_version, numpy_version) +http://cens.ioc.ee/projects/f2py2e/""" def scaninputline(inputline): @@ -515,14 +511,14 @@ def run_compile(): remove_build_dir = 1 build_dir = tempfile.mkdtemp() - _reg1 = re.compile(r'[-][-]link[-]') + _reg1 = re.compile(r'--link-') sysinfo_flags = [_m for _m in sys.argv[1:] if _reg1.match(_m)] sys.argv = [_m for _m in sys.argv if _m not in sysinfo_flags] if sysinfo_flags: sysinfo_flags = [f[7:] for f in sysinfo_flags] _reg2 = re.compile( - r'[-][-]((no[-]|)(wrap[-]functions|lower)|debug[-]capi|quiet)|[-]include') + r'--((no-|)(wrap-functions|lower)|debug-capi|quiet)|-include') f2py_flags = [_m for _m in sys.argv[1:] if _reg2.match(_m)] sys.argv = [_m for _m in sys.argv if _m not in f2py_flags] f2py_flags2 = [] @@ -540,11 +536,11 @@ def run_compile(): sys.argv = [_m for _m in sys.argv if _m not in f2py_flags2] _reg3 = re.compile( - r'[-][-]((f(90)?compiler([-]exec|)|compiler)=|help[-]compiler)') + r'--((f(90)?compiler(-exec|)|compiler)=|help-compiler)') flib_flags = [_m for _m in sys.argv[1:] if _reg3.match(_m)] sys.argv = [_m for _m in sys.argv if _m not in flib_flags] _reg4 = re.compile( - r'[-][-]((f(77|90)(flags|exec)|opt|arch)=|(debug|noopt|noarch|help[-]fcompiler))') + r'--((f(77|90)(flags|exec)|opt|arch)=|(debug|noopt|noarch|help-fcompiler))') fc_flags = [_m for _m in sys.argv[1:] if _reg4.match(_m)] sys.argv = [_m for _m in sys.argv if _m not in fc_flags] @@ -573,7 +569,7 @@ def run_compile(): del flib_flags[i] assert len(flib_flags) <= 2, repr(flib_flags) - _reg5 = re.compile(r'[-][-](verbose)') + _reg5 = re.compile(r'--(verbose)') setup_flags = [_m for _m in sys.argv[1:] if _reg5.match(_m)] sys.argv = [_m for _m in sys.argv if _m not in setup_flags] diff --git a/numpy/f2py/f90mod_rules.py b/numpy/f2py/f90mod_rules.py index 122fa89394b7..3e1c9674f8e2 100644 --- a/numpy/f2py/f90mod_rules.py +++ b/numpy/f2py/f90mod_rules.py @@ -192,7 +192,8 @@ def iadd(line, s=ihooks): if hasbody(m): for b in m['body']: if not isroutine(b): - print('Skipping', b['block'], b['name']) + outmess("f90mod_rules.buildhooks:" + f" skipping {b['block']} {b['name']}\n") continue modobjs.append('%s()' % (b['name'])) b['modulename'] = m['name'] diff --git a/numpy/f2py/func2subr.py b/numpy/f2py/func2subr.py index e9976f43cf41..21d4c009cc26 100644 --- a/numpy/f2py/func2subr.py +++ b/numpy/f2py/func2subr.py @@ -130,7 +130,7 @@ def add(line, ret=ret): l = l + ', ' + fortranname if need_interface: for line in rout['saved_interface'].split('\n'): - if line.lstrip().startswith('use '): + if line.lstrip().startswith('use ') and '__user__' not in line: add(line) args = args[1:] @@ -222,7 +222,7 @@ def add(line, ret=ret): if need_interface: for line in rout['saved_interface'].split('\n'): - if line.lstrip().startswith('use '): + if line.lstrip().startswith('use ') and '__user__' not in line: add(line) dumped_args = [] @@ -247,7 +247,10 @@ def add(line, ret=ret): pass else: add('interface') - add(rout['saved_interface'].lstrip()) + for line in rout['saved_interface'].split('\n'): + if line.lstrip().startswith('use ') and '__user__' in line: + continue + add(line) add('end interface') sargs = ', '.join([a for a in args if a not in extra_args]) diff --git a/numpy/f2py/rules.py b/numpy/f2py/rules.py index a14f60194408..63e47baa20ed 100755 --- a/numpy/f2py/rules.py +++ b/numpy/f2py/rules.py @@ -50,18 +50,15 @@ Pearu Peterson """ -__version__ = "$Revision: 1.129 $"[10:-1] - -from . import __version__ -f2py_version = __version__.version - -from .. import version as _numpy_version -numpy_version = _numpy_version.version - import os import time import copy +# __version__.version is now the same as the NumPy version +from . import __version__ +f2py_version = __version__.version +numpy_version = __version__.version + from .auxfuncs import ( applyrules, debugcapi, dictappend, errmess, gentitle, getargs2, hascallstatement, hasexternals, hasinitvalue, hasnote, hasresultnote, @@ -76,7 +73,7 @@ issubroutine, issubroutine_wrap, isthreadsafe, isunsigned, isunsigned_char, isunsigned_chararray, isunsigned_long_long, isunsigned_long_longarray, isunsigned_short, isunsigned_shortarray, - l_and, l_not, l_or, outmess, replace, stripcomma, + l_and, l_not, l_or, outmess, replace, stripcomma, requiresf90wrapper ) from . import capi_maps @@ -202,7 +199,7 @@ \tif (PyErr_Occurred()) \t\t{PyErr_SetString(PyExc_ImportError, \"can't initialize module #modulename# (failed to import numpy)\"); return m;} \td = PyModule_GetDict(m); -\ts = PyUnicode_FromString(\"$R""" + """evision: $\"); +\ts = PyUnicode_FromString(\"#f2py_version#\"); \tPyDict_SetItemString(d, \"__version__\", s); \tPy_DECREF(s); \ts = PyUnicode_FromString( @@ -812,7 +809,7 @@ """, {debugcapi: ["""\ fprintf(stderr,\"debug-capi:Assuming %d arguments; at most #maxnofargs#(-#nofoptargs#) is expected.\\n\",#varname#_cb.nofargs); - CFUNCSMESSPY(\"for #varname#=\",#cbname#_capi);""", + CFUNCSMESSPY(\"for #varname#=\",#varname#_cb.capi);""", {l_not(isintent_callback): """ fprintf(stderr,\"#vardebugshowvalue# (call-back in C).\\n\",#cbname#);"""}]}, """\ CFUNCSMESS(\"Saving callback variables for `#varname#`.\\n\"); @@ -1166,7 +1163,7 @@ def buildmodule(m, um): for n in m['interfaced']: nb = None for bi in m['body']: - if not bi['block'] == 'interface': + if bi['block'] not in ['interface', 'abstract interface']: errmess('buildmodule: Expected interface block. Skipping.\n') continue for b in bi['body']: @@ -1187,9 +1184,12 @@ def buildmodule(m, um): nb1['args'] = a nb_list.append(nb1) for nb in nb_list: + # requiresf90wrapper must be called before buildapi as it + # rewrites assumed shape arrays as automatic arrays. + isf90 = requiresf90wrapper(nb) api, wrap = buildapi(nb) if wrap: - if ismoduleroutine(nb): + if isf90: funcwrappers2.append(wrap) else: funcwrappers.append(wrap) @@ -1291,7 +1291,10 @@ def buildmodule(m, um): 'C It contains Fortran 77 wrappers to fortran functions.\n') lines = [] for l in ('\n\n'.join(funcwrappers) + '\n').split('\n'): - if l and l[0] == ' ': + if 0 <= l.find('!') < 66: + # don't split comment lines + lines.append(l + '\n') + elif l and l[0] == ' ': while len(l) >= 66: lines.append(l[:66] + '\n &') l = l[66:] @@ -1313,7 +1316,10 @@ def buildmodule(m, um): '! It contains Fortran 90 wrappers to fortran functions.\n') lines = [] for l in ('\n\n'.join(funcwrappers2) + '\n').split('\n'): - if len(l) > 72 and l[0] == ' ': + if 0 <= l.find('!') < 72: + # don't split comment lines + lines.append(l + '\n') + elif len(l) > 72 and l[0] == ' ': lines.append(l[:72] + '&\n &') l = l[72:] while len(l) > 66: diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index 3275f90ad2cb..b9ef18701ce3 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -799,7 +799,7 @@ PyArrayObject* array_from_pyobj(const int type_num, && ARRAY_ISCOMPATIBLE(arr,type_num) && F2PY_CHECK_ALIGNMENT(arr, intent) ) { - if ((intent & F2PY_INTENT_C)?PyArray_ISCARRAY(arr):PyArray_ISFARRAY(arr)) { + if ((intent & F2PY_INTENT_C)?PyArray_ISCARRAY_RO(arr):PyArray_ISFARRAY_RO(arr)) { if ((intent & F2PY_INTENT_OUT)) { Py_INCREF(arr); } @@ -807,9 +807,9 @@ PyArrayObject* array_from_pyobj(const int type_num, return arr; } } - if (intent & F2PY_INTENT_INOUT) { strcpy(mess, "failed to initialize intent(inout) array"); + /* Must use PyArray_IS*ARRAY because intent(inout) requires writable input */ if ((intent & F2PY_INTENT_C) && !PyArray_ISCARRAY(arr)) strcat(mess, " -- input not contiguous"); if (!(intent & F2PY_INTENT_C) && !PyArray_ISFARRAY(arr)) diff --git a/numpy/f2py/tests/test_abstract_interface.py b/numpy/f2py/tests/test_abstract_interface.py new file mode 100644 index 000000000000..936c1f7bc9ae --- /dev/null +++ b/numpy/f2py/tests/test_abstract_interface.py @@ -0,0 +1,66 @@ +import textwrap +from . import util +from numpy.f2py import crackfortran + + +class TestAbstractInterface(util.F2PyTest): + suffix = '.f90' + + skip = ['add1', 'add2'] + + code = textwrap.dedent(""" + module ops_module + + abstract interface + subroutine op(x, y, z) + integer, intent(in) :: x, y + integer, intent(out) :: z + end subroutine + end interface + + contains + + subroutine foo(x, y, r1, r2) + integer, intent(in) :: x, y + integer, intent(out) :: r1, r2 + procedure (op) add1, add2 + procedure (op), pointer::p + p=>add1 + call p(x, y, r1) + p=>add2 + call p(x, y, r2) + end subroutine + end module + + subroutine add1(x, y, z) + integer, intent(in) :: x, y + integer, intent(out) :: z + z = x + y + end subroutine + + subroutine add2(x, y, z) + integer, intent(in) :: x, y + integer, intent(out) :: z + z = x + 2 * y + end subroutine + """) + + def test_abstract_interface(self): + assert self.module.ops_module.foo(3, 5) == (8, 13) + + def test_parse_abstract_interface(self, tmp_path): + # Test gh18403 + f_path = tmp_path / "gh18403_mod.f90" + with f_path.open('w') as ff: + ff.write(textwrap.dedent("""\ + module test + abstract interface + subroutine foo() + end subroutine + end interface + end module test + """)) + mod = crackfortran.crackfortran([str(f_path)]) + assert len(mod) == 1 + assert len(mod[0]['body']) == 1 + assert mod[0]['body'][0]['block'] == 'abstract interface' diff --git a/numpy/f2py/tests/test_array_from_pyobj.py b/numpy/f2py/tests/test_array_from_pyobj.py index b719f2495ed4..649fd1c4863b 100644 --- a/numpy/f2py/tests/test_array_from_pyobj.py +++ b/numpy/f2py/tests/test_array_from_pyobj.py @@ -1,11 +1,11 @@ import os import sys import copy +import platform import pytest -from numpy import ( - array, alltrue, ndarray, zeros, dtype, intp, clongdouble - ) +import numpy as np + from numpy.testing import assert_, assert_equal from numpy.core.multiarray import typeinfo from . import util @@ -119,8 +119,11 @@ def is_intent_exact(self, *names): # 16 byte long double types this means the inout intent cannot be satisfied # and several tests fail as the alignment flag can be randomly true or fals # when numpy gains an aligned allocator the tests could be enabled again -if ((intp().dtype.itemsize != 4 or clongdouble().dtype.alignment <= 8) and - sys.platform != 'win32'): +# +# Furthermore, on macOS ARM64, LONGDOUBLE is an alias for DOUBLE. +if ((np.intp().dtype.itemsize != 4 or np.clongdouble().dtype.alignment <= 8) and + sys.platform != 'win32' and + (platform.system(), platform.processor()) != ('Darwin', 'arm')): _type_names.extend(['LONGDOUBLE', 'CDOUBLE', 'CLONGDOUBLE']) _cast_dict['LONGDOUBLE'] = _cast_dict['LONG'] + \ ['ULONG', 'FLOAT', 'DOUBLE', 'LONGDOUBLE'] @@ -133,7 +136,7 @@ class Type: _type_cache = {} def __new__(cls, name): - if isinstance(name, dtype): + if isinstance(name, np.dtype): dtype0 = name name = None for n, i in typeinfo.items(): @@ -153,7 +156,8 @@ def _init(self, name): info = typeinfo[self.NAME] self.type_num = getattr(wrap, 'NPY_' + self.NAME) assert_equal(self.type_num, info.num) - self.dtype = info.type + self.dtype = np.dtype(info.type) + self.type = info.type self.elsize = info.bits / 8 self.dtypechar = info.char @@ -202,7 +206,7 @@ def __init__(self, typ, dims, intent, obj): # arr.dtypechar may be different from typ.dtypechar self.arr = wrap.call(typ.type_num, dims, intent.flags, obj) - assert_(isinstance(self.arr, ndarray), repr(type(self.arr))) + assert_(isinstance(self.arr, np.ndarray), repr(type(self.arr))) self.arr_attr = wrap.array_attrs(self.arr) @@ -225,13 +229,15 @@ def __init__(self, typ, dims, intent, obj): return if intent.is_intent('cache'): - assert_(isinstance(obj, ndarray), repr(type(obj))) - self.pyarr = array(obj).reshape(*dims).copy() + assert_(isinstance(obj, np.ndarray), repr(type(obj))) + self.pyarr = np.array(obj).reshape(*dims).copy() else: - self.pyarr = array(array(obj, dtype=typ.dtypechar).reshape(*dims), - order=self.intent.is_intent('c') and 'C' or 'F') + self.pyarr = np.array( + np.array(obj, dtype=typ.dtypechar).reshape(*dims), + order=self.intent.is_intent('c') and 'C' or 'F') assert_(self.pyarr.dtype == typ, repr((self.pyarr.dtype, typ))) + self.pyarr.setflags(write=self.arr.flags['WRITEABLE']) assert_(self.pyarr.flags['OWNDATA'], (obj, intent)) self.pyarr_attr = wrap.array_attrs(self.pyarr) @@ -266,7 +272,7 @@ def __init__(self, typ, dims, intent, obj): repr((self.arr_attr[5][3], self.type.elsize))) assert_(self.arr_equal(self.pyarr, self.arr)) - if isinstance(self.obj, ndarray): + if isinstance(self.obj, np.ndarray): if typ.elsize == Type(obj.dtype).elsize: if not intent.is_intent('copy') and self.arr_attr[1] <= 1: assert_(self.has_shared_memory()) @@ -274,8 +280,7 @@ def __init__(self, typ, dims, intent, obj): def arr_equal(self, arr1, arr2): if arr1.shape != arr2.shape: return False - s = arr1 == arr2 - return alltrue(s.flatten()) + return (arr1 == arr2).all() def __str__(self): return str(self.arr) @@ -285,7 +290,7 @@ def has_shared_memory(self): """ if self.obj is self.arr: return True - if not isinstance(self.obj, ndarray): + if not isinstance(self.obj, np.ndarray): return False obj_attr = wrap.array_attrs(self.obj) return obj_attr[0] == self.arr_attr[0] @@ -318,7 +323,7 @@ def test_in_from_2seq(self): def test_in_from_2casttype(self): for t in self.type.cast_types(): - obj = array(self.num2seq, dtype=t.dtype) + obj = np.array(self.num2seq, dtype=t.dtype) a = self.array([len(self.num2seq)], intent.in_, obj) if t.elsize == self.type.elsize: assert_( @@ -326,8 +331,20 @@ def test_in_from_2casttype(self): else: assert_(not a.has_shared_memory(), repr(t.dtype)) + @pytest.mark.parametrize('write', ['w', 'ro']) + @pytest.mark.parametrize('order', ['C', 'F']) + @pytest.mark.parametrize('inp', ['2seq', '23seq']) + def test_in_nocopy(self, write, order, inp): + """Test if intent(in) array can be passed without copies + """ + seq = getattr(self, 'num' + inp) + obj = np.array(seq, dtype=self.type.dtype, order=order) + obj.setflags(write=(write == 'w')) + a = self.array(obj.shape, ((order=='C' and intent.in_.c) or intent.in_), obj) + assert a.has_shared_memory() + def test_inout_2seq(self): - obj = array(self.num2seq, dtype=self.type.dtype) + obj = np.array(self.num2seq, dtype=self.type.dtype) a = self.array([len(self.num2seq)], intent.inout, obj) assert_(a.has_shared_memory()) @@ -341,12 +358,12 @@ def test_inout_2seq(self): raise SystemError('intent(inout) should have failed on sequence') def test_f_inout_23seq(self): - obj = array(self.num23seq, dtype=self.type.dtype, order='F') + obj = np.array(self.num23seq, dtype=self.type.dtype, order='F') shape = (len(self.num23seq), len(self.num23seq[0])) a = self.array(shape, intent.in_.inout, obj) assert_(a.has_shared_memory()) - obj = array(self.num23seq, dtype=self.type.dtype, order='C') + obj = np.array(self.num23seq, dtype=self.type.dtype, order='C') shape = (len(self.num23seq), len(self.num23seq[0])) try: a = self.array(shape, intent.in_.inout, obj) @@ -359,14 +376,14 @@ def test_f_inout_23seq(self): 'intent(inout) should have failed on improper array') def test_c_inout_23seq(self): - obj = array(self.num23seq, dtype=self.type.dtype) + obj = np.array(self.num23seq, dtype=self.type.dtype) shape = (len(self.num23seq), len(self.num23seq[0])) a = self.array(shape, intent.in_.c.inout, obj) assert_(a.has_shared_memory()) def test_in_copy_from_2casttype(self): for t in self.type.cast_types(): - obj = array(self.num2seq, dtype=t.dtype) + obj = np.array(self.num2seq, dtype=t.dtype) a = self.array([len(self.num2seq)], intent.in_.copy, obj) assert_(not a.has_shared_memory(), repr(t.dtype)) @@ -377,14 +394,14 @@ def test_c_in_from_23seq(self): def test_in_from_23casttype(self): for t in self.type.cast_types(): - obj = array(self.num23seq, dtype=t.dtype) + obj = np.array(self.num23seq, dtype=t.dtype) a = self.array([len(self.num23seq), len(self.num23seq[0])], intent.in_, obj) assert_(not a.has_shared_memory(), repr(t.dtype)) def test_f_in_from_23casttype(self): for t in self.type.cast_types(): - obj = array(self.num23seq, dtype=t.dtype, order='F') + obj = np.array(self.num23seq, dtype=t.dtype, order='F') a = self.array([len(self.num23seq), len(self.num23seq[0])], intent.in_, obj) if t.elsize == self.type.elsize: @@ -394,7 +411,7 @@ def test_f_in_from_23casttype(self): def test_c_in_from_23casttype(self): for t in self.type.cast_types(): - obj = array(self.num23seq, dtype=t.dtype) + obj = np.array(self.num23seq, dtype=t.dtype) a = self.array([len(self.num23seq), len(self.num23seq[0])], intent.in_.c, obj) if t.elsize == self.type.elsize: @@ -404,14 +421,14 @@ def test_c_in_from_23casttype(self): def test_f_copy_in_from_23casttype(self): for t in self.type.cast_types(): - obj = array(self.num23seq, dtype=t.dtype, order='F') + obj = np.array(self.num23seq, dtype=t.dtype, order='F') a = self.array([len(self.num23seq), len(self.num23seq[0])], intent.in_.copy, obj) assert_(not a.has_shared_memory(), repr(t.dtype)) def test_c_copy_in_from_23casttype(self): for t in self.type.cast_types(): - obj = array(self.num23seq, dtype=t.dtype) + obj = np.array(self.num23seq, dtype=t.dtype) a = self.array([len(self.num23seq), len(self.num23seq[0])], intent.in_.c.copy, obj) assert_(not a.has_shared_memory(), repr(t.dtype)) @@ -420,7 +437,7 @@ def test_in_cache_from_2casttype(self): for t in self.type.all_types(): if t.elsize != self.type.elsize: continue - obj = array(self.num2seq, dtype=t.dtype) + obj = np.array(self.num2seq, dtype=t.dtype) shape = (len(self.num2seq),) a = self.array(shape, intent.in_.c.cache, obj) assert_(a.has_shared_memory(), repr(t.dtype)) @@ -428,7 +445,7 @@ def test_in_cache_from_2casttype(self): a = self.array(shape, intent.in_.cache, obj) assert_(a.has_shared_memory(), repr(t.dtype)) - obj = array(self.num2seq, dtype=t.dtype, order='F') + obj = np.array(self.num2seq, dtype=t.dtype, order='F') a = self.array(shape, intent.in_.c.cache, obj) assert_(a.has_shared_memory(), repr(t.dtype)) @@ -449,7 +466,7 @@ def test_in_cache_from_2casttype_failure(self): for t in self.type.all_types(): if t.elsize >= self.type.elsize: continue - obj = array(self.num2seq, dtype=t.dtype) + obj = np.array(self.num2seq, dtype=t.dtype) shape = (len(self.num2seq),) try: self.array(shape, intent.in_.cache, obj) # Should succeed @@ -485,18 +502,18 @@ def test_hidden(self): shape = (2,) a = self.array(shape, intent.hide, None) assert_(a.arr.shape == shape) - assert_(a.arr_equal(a.arr, zeros(shape, dtype=self.type.dtype))) + assert_(a.arr_equal(a.arr, np.zeros(shape, dtype=self.type.dtype))) shape = (2, 3) a = self.array(shape, intent.hide, None) assert_(a.arr.shape == shape) - assert_(a.arr_equal(a.arr, zeros(shape, dtype=self.type.dtype))) + assert_(a.arr_equal(a.arr, np.zeros(shape, dtype=self.type.dtype))) assert_(a.arr.flags['FORTRAN'] and not a.arr.flags['CONTIGUOUS']) shape = (2, 3) a = self.array(shape, intent.c.hide, None) assert_(a.arr.shape == shape) - assert_(a.arr_equal(a.arr, zeros(shape, dtype=self.type.dtype))) + assert_(a.arr_equal(a.arr, np.zeros(shape, dtype=self.type.dtype))) assert_(not a.arr.flags['FORTRAN'] and a.arr.flags['CONTIGUOUS']) shape = (-1, 3) @@ -514,18 +531,18 @@ def test_optional_none(self): shape = (2,) a = self.array(shape, intent.optional, None) assert_(a.arr.shape == shape) - assert_(a.arr_equal(a.arr, zeros(shape, dtype=self.type.dtype))) + assert_(a.arr_equal(a.arr, np.zeros(shape, dtype=self.type.dtype))) shape = (2, 3) a = self.array(shape, intent.optional, None) assert_(a.arr.shape == shape) - assert_(a.arr_equal(a.arr, zeros(shape, dtype=self.type.dtype))) + assert_(a.arr_equal(a.arr, np.zeros(shape, dtype=self.type.dtype))) assert_(a.arr.flags['FORTRAN'] and not a.arr.flags['CONTIGUOUS']) shape = (2, 3) a = self.array(shape, intent.c.optional, None) assert_(a.arr.shape == shape) - assert_(a.arr_equal(a.arr, zeros(shape, dtype=self.type.dtype))) + assert_(a.arr_equal(a.arr, np.zeros(shape, dtype=self.type.dtype))) assert_(not a.arr.flags['FORTRAN'] and a.arr.flags['CONTIGUOUS']) def test_optional_from_2seq(self): @@ -547,14 +564,14 @@ def test_optional_from_23seq(self): assert_(not a.has_shared_memory()) def test_inplace(self): - obj = array(self.num23seq, dtype=self.type.dtype) + obj = np.array(self.num23seq, dtype=self.type.dtype) assert_(not obj.flags['FORTRAN'] and obj.flags['CONTIGUOUS']) shape = obj.shape a = self.array(shape, intent.inplace, obj) assert_(obj[1][2] == a.arr[1][2], repr((obj, a.arr))) a.arr[1][2] = 54 assert_(obj[1][2] == a.arr[1][2] == - array(54, dtype=self.type.dtype), repr((obj, a.arr))) + np.array(54, dtype=self.type.dtype), repr((obj, a.arr))) assert_(a.arr is obj) assert_(obj.flags['FORTRAN']) # obj attributes are changed inplace! assert_(not obj.flags['CONTIGUOUS']) @@ -563,17 +580,17 @@ def test_inplace_from_casttype(self): for t in self.type.cast_types(): if t is self.type: continue - obj = array(self.num23seq, dtype=t.dtype) - assert_(obj.dtype.type == t.dtype) - assert_(obj.dtype.type is not self.type.dtype) + obj = np.array(self.num23seq, dtype=t.dtype) + assert_(obj.dtype.type == t.type) + assert_(obj.dtype.type is not self.type.type) assert_(not obj.flags['FORTRAN'] and obj.flags['CONTIGUOUS']) shape = obj.shape a = self.array(shape, intent.inplace, obj) assert_(obj[1][2] == a.arr[1][2], repr((obj, a.arr))) a.arr[1][2] = 54 assert_(obj[1][2] == a.arr[1][2] == - array(54, dtype=self.type.dtype), repr((obj, a.arr))) + np.array(54, dtype=self.type.dtype), repr((obj, a.arr))) assert_(a.arr is obj) assert_(obj.flags['FORTRAN']) # obj attributes changed inplace! assert_(not obj.flags['CONTIGUOUS']) - assert_(obj.dtype.type is self.type.dtype) # obj changed inplace! + assert_(obj.dtype.type is self.type.type) # obj changed inplace! diff --git a/numpy/f2py/tests/test_assumed_shape.py b/numpy/f2py/tests/test_assumed_shape.py index dfc252660c59..79e3ad138426 100644 --- a/numpy/f2py/tests/test_assumed_shape.py +++ b/numpy/f2py/tests/test_assumed_shape.py @@ -47,7 +47,7 @@ def setup(self): self.sources.append(self.f2cmap_file.name) self.options = ["--f2cmap", self.f2cmap_file.name] - super(TestF2cmapOption, self).setup() + super().setup() def teardown(self): os.unlink(self.f2cmap_file.name) diff --git a/numpy/f2py/tests/test_callback.py b/numpy/f2py/tests/test_callback.py index 81650a8191e8..2cb429ec21d5 100644 --- a/numpy/f2py/tests/test_callback.py +++ b/numpy/f2py/tests/test_callback.py @@ -61,6 +61,21 @@ class TestF77Callback(util.F2PyTest): a = callback(cu, lencu) end + + subroutine hidden_callback(a, r) + external global_f +cf2py intent(callback, hide) global_f + integer a, r, global_f +cf2py intent(out) r + r = global_f(a) + end + + subroutine hidden_callback2(a, r) + external global_f + integer a, r, global_f +cf2py intent(out) r + r = global_f(a) + end """ @pytest.mark.parametrize('name', 't,t2'.split(',')) @@ -204,6 +219,39 @@ def runner(name): if errors: raise AssertionError(errors) + def test_hidden_callback(self): + try: + self.module.hidden_callback(2) + except Exception as msg: + assert_(str(msg).startswith('Callback global_f not defined')) + + try: + self.module.hidden_callback2(2) + except Exception as msg: + assert_(str(msg).startswith('cb: Callback global_f not defined')) + + self.module.global_f = lambda x: x + 1 + r = self.module.hidden_callback(2) + assert_(r == 3) + + self.module.global_f = lambda x: x + 2 + r = self.module.hidden_callback(2) + assert_(r == 4) + + del self.module.global_f + try: + self.module.hidden_callback(2) + except Exception as msg: + assert_(str(msg).startswith('Callback global_f not defined')) + + self.module.global_f = lambda x=0: x + 3 + r = self.module.hidden_callback(2) + assert_(r == 5) + + # reproducer of gh18341 + r = self.module.hidden_callback2(2) + assert_(r == 3) + class TestF77CallbackPythonTLS(TestF77Callback): """ @@ -211,3 +259,68 @@ class TestF77CallbackPythonTLS(TestF77Callback): compiler-provided """ options = ["-DF2PY_USE_PYTHON_TLS"] + + +class TestF90Callback(util.F2PyTest): + + suffix = '.f90' + + code = textwrap.dedent( + """ + function gh17797(f, y) result(r) + external f + integer(8) :: r, f + integer(8), dimension(:) :: y + r = f(0) + r = r + sum(y) + end function gh17797 + """) + + def test_gh17797(self): + + def incr(x): + return x + 123 + + y = np.array([1, 2, 3], dtype=np.int64) + r = self.module.gh17797(incr, y) + assert r == 123 + 1 + 2 + 3 + + +class TestGH18335(util.F2PyTest): + """The reproduction of the reported issue requires specific input that + extensions may break the issue conditions, so the reproducer is + implemented as a separate test class. Do not extend this test with + other tests! + """ + + suffix = '.f90' + + code = textwrap.dedent( + """ + ! When gh18335_workaround is defined as an extension, + ! the issue cannot be reproduced. + !subroutine gh18335_workaround(f, y) + ! implicit none + ! external f + ! integer(kind=1) :: y(1) + ! call f(y) + !end subroutine gh18335_workaround + + function gh18335(f) result (r) + implicit none + external f + integer(kind=1) :: y(1), r + y(1) = 123 + call f(y) + r = y(1) + end function gh18335 + """) + + def test_gh18335(self): + + def foo(x): + x[0] += 1 + + y = np.array([1, 2, 3], dtype=np.int8) + r = self.module.gh18335(foo) + assert r == 123 + 1 diff --git a/numpy/f2py/tests/test_crackfortran.py b/numpy/f2py/tests/test_crackfortran.py index 735804024c9e..d26917f0c23f 100644 --- a/numpy/f2py/tests/test_crackfortran.py +++ b/numpy/f2py/tests/test_crackfortran.py @@ -86,3 +86,56 @@ def test_defaultPublic(self, tmp_path): assert 'public' not in mod['vars']['a']['attrspec'] assert 'private' not in mod['vars']['seta']['attrspec'] assert 'public' in mod['vars']['seta']['attrspec'] + +class TestExternal(util.F2PyTest): + # issue gh-17859: add external attribute support + code = """ + integer(8) function external_as_statement(fcn) + implicit none + external fcn + integer(8) :: fcn + external_as_statement = fcn(0) + end + + integer(8) function external_as_attribute(fcn) + implicit none + integer(8), external :: fcn + external_as_attribute = fcn(0) + end + """ + + def test_external_as_statement(self): + def incr(x): + return x + 123 + r = self.module.external_as_statement(incr) + assert r == 123 + + def test_external_as_attribute(self): + def incr(x): + return x + 123 + r = self.module.external_as_attribute(incr) + assert r == 123 + +class TestCrackFortran(util.F2PyTest): + + suffix = '.f90' + + code = textwrap.dedent(""" + subroutine gh2848( & + ! first 2 parameters + par1, par2,& + ! last 2 parameters + par3, par4) + + integer, intent(in) :: par1, par2 + integer, intent(out) :: par3, par4 + + par3 = par1 + par4 = par2 + + end subroutine gh2848 + """) + + def test_gh2848(self): + r = self.module.gh2848(1, 2) + assert r == (1, 2) diff --git a/numpy/fft/__init__.py b/numpy/fft/__init__.py index a86bb3ac0bad..fd5e47580a54 100644 --- a/numpy/fft/__init__.py +++ b/numpy/fft/__init__.py @@ -200,9 +200,13 @@ """ +from . import _pocketfft, helper from ._pocketfft import * from .helper import * +__all__ = _pocketfft.__all__.copy() +__all__ += helper.__all__ + from numpy._pytesttester import PytestTester test = PytestTester(__name__) del PytestTester diff --git a/numpy/fft/__init__.pyi b/numpy/fft/__init__.pyi index 45190517f2af..3c191a35f8f2 100644 --- a/numpy/fft/__init__.pyi +++ b/numpy/fft/__init__.pyi @@ -1,20 +1,22 @@ -from typing import Any +from typing import Any, List -fft: Any -ifft: Any -rfft: Any -irfft: Any -hfft: Any -ihfft: Any -rfftn: Any -irfftn: Any -rfft2: Any -irfft2: Any -fft2: Any -ifft2: Any -fftn: Any -ifftn: Any -fftshift: Any -ifftshift: Any -fftfreq: Any -rfftfreq: Any +__all__: List[str] + +def fft(a, n=..., axis=..., norm=...): ... +def ifft(a, n=..., axis=..., norm=...): ... +def rfft(a, n=..., axis=..., norm=...): ... +def irfft(a, n=..., axis=..., norm=...): ... +def hfft(a, n=..., axis=..., norm=...): ... +def ihfft(a, n=..., axis=..., norm=...): ... +def fftn(a, s=..., axes=..., norm=...): ... +def ifftn(a, s=..., axes=..., norm=...): ... +def rfftn(a, s=..., axes=..., norm=...): ... +def irfftn(a, s=..., axes=..., norm=...): ... +def fft2(a, s=..., axes=..., norm=...): ... +def ifft2(a, s=..., axes=..., norm=...): ... +def rfft2(a, s=..., axes=..., norm=...): ... +def irfft2(a, s=..., axes=..., norm=...): ... +def fftshift(x, axes=...): ... +def ifftshift(x, axes=...): ... +def fftfreq(n, d=...): ... +def rfftfreq(n, d=...): ... diff --git a/numpy/fft/_pocketfft.py b/numpy/fft/_pocketfft.py index 83ac86036bc0..ad69f7c837bb 100644 --- a/numpy/fft/_pocketfft.py +++ b/numpy/fft/_pocketfft.py @@ -112,7 +112,7 @@ def _swap_direction(norm): return _SWAP_DIRECTION_MAP[norm] except KeyError: raise ValueError(f'Invalid norm value {norm}; should be "backward", ' - '"ortho" or "forward".') + '"ortho" or "forward".') from None def _fft_dispatcher(a, n=None, axis=None, norm=None): @@ -160,7 +160,7 @@ def fft(a, n=None, axis=-1, norm=None): Raises ------ IndexError - if `axes` is larger than the last axis of `a`. + If `axis` is not a valid axis of `a`. See Also -------- @@ -272,7 +272,7 @@ def ifft(a, n=None, axis=-1, norm=None): Raises ------ IndexError - If `axes` is larger than the last axis of `a`. + If `axis` is not a valid axis of `a`. See Also -------- @@ -300,9 +300,11 @@ def ifft(a, n=None, axis=-1, norm=None): >>> n = np.zeros((400,), dtype=complex) >>> n[40:60] = np.exp(1j*np.random.uniform(0, 2*np.pi, (20,))) >>> s = np.fft.ifft(n) - >>> plt.plot(t, s.real, 'b-', t, s.imag, 'r--') - [, ] - >>> plt.legend(('real', 'imaginary')) + >>> plt.plot(t, s.real, label='real') + [] + >>> plt.plot(t, s.imag, '--', label='imaginary') + [] + >>> plt.legend() >>> plt.show() @@ -358,7 +360,7 @@ def rfft(a, n=None, axis=-1, norm=None): Raises ------ IndexError - If `axis` is larger than the last axis of `a`. + If `axis` is not a valid axis of `a`. See Also -------- @@ -461,7 +463,7 @@ def irfft(a, n=None, axis=-1, norm=None): Raises ------ IndexError - If `axis` is larger than the last axis of `a`. + If `axis` is not a valid axis of `a`. See Also -------- @@ -556,7 +558,7 @@ def hfft(a, n=None, axis=-1, norm=None): Raises ------ IndexError - If `axis` is larger than the last axis of `a`. + If `axis` is not a valid axis of `a`. See also -------- @@ -1242,6 +1244,15 @@ def rfft2(a, s=None, axes=(-2, -1), norm=None): This is really just `rfftn` with different default behavior. For more details see `rfftn`. + Examples + -------- + >>> a = np.mgrid[:5, :5][0] + >>> np.fft.rfft2(a) + array([[ 50. +0.j , 0. +0.j , 0. +0.j ], + [-12.5+17.20477401j, 0. +0.j , 0. +0.j ], + [-12.5 +4.0614962j , 0. +0.j , 0. +0.j ], + [-12.5 -4.0614962j , 0. +0.j , 0. +0.j ], + [-12.5-17.20477401j, 0. +0.j , 0. +0.j ]]) """ return rfftn(a, s, axes, norm) @@ -1399,5 +1410,15 @@ def irfft2(a, s=None, axes=(-2, -1), norm=None): This is really `irfftn` with different defaults. For more details see `irfftn`. + Examples + -------- + >>> a = np.mgrid[:5, :5][0] + >>> A = np.fft.rfft2(a) + >>> np.fft.irfft2(A, s=a.shape) + array([[0., 0., 0., 0., 0.], + [1., 1., 1., 1., 1.], + [2., 2., 2., 2., 2.], + [3., 3., 3., 3., 3.], + [4., 4., 4., 4., 4.]]) """ return irfftn(a, s, axes, norm) diff --git a/numpy/lib/__init__.pyi b/numpy/lib/__init__.pyi index 413e2ae1b5d0..2904b6a844dd 100644 --- a/numpy/lib/__init__.pyi +++ b/numpy/lib/__init__.pyi @@ -1,177 +1,233 @@ -from typing import Any - -emath: Any -math: Any -tracemalloc_domain: Any -Arrayterator: Any -iscomplexobj: Any -isrealobj: Any -imag: Any -iscomplex: Any -isreal: Any -nan_to_num: Any -real: Any -real_if_close: Any -typename: Any -asfarray: Any -mintypecode: Any -asscalar: Any -common_type: Any -ravel_multi_index: Any -unravel_index: Any -mgrid: Any -ogrid: Any -r_: Any -c_: Any -s_: Any -index_exp: Any -ix_: Any -ndenumerate: Any -ndindex: Any -fill_diagonal: Any -diag_indices: Any -diag_indices_from: Any -select: Any -piecewise: Any -trim_zeros: Any -copy: Any -iterable: Any -percentile: Any -diff: Any -gradient: Any -angle: Any -unwrap: Any -sort_complex: Any -disp: Any -flip: Any -rot90: Any -extract: Any -place: Any -vectorize: Any -asarray_chkfinite: Any -average: Any -bincount: Any -digitize: Any -cov: Any -corrcoef: Any -msort: Any -median: Any -sinc: Any -hamming: Any -hanning: Any -bartlett: Any -blackman: Any -kaiser: Any -trapz: Any -i0: Any -add_newdoc: Any -add_docstring: Any -meshgrid: Any -delete: Any -insert: Any -append: Any -interp: Any -add_newdoc_ufunc: Any -quantile: Any -column_stack: Any -row_stack: Any -dstack: Any -array_split: Any -split: Any -hsplit: Any -vsplit: Any -dsplit: Any -apply_over_axes: Any -expand_dims: Any -apply_along_axis: Any -kron: Any -tile: Any -get_array_wrap: Any -take_along_axis: Any -put_along_axis: Any -broadcast_to: Any -broadcast_arrays: Any -diag: Any -diagflat: Any -eye: Any -fliplr: Any -flipud: Any -tri: Any -triu: Any -tril: Any -vander: Any -histogram2d: Any -mask_indices: Any -tril_indices: Any -tril_indices_from: Any -triu_indices: Any -triu_indices_from: Any -fix: Any -isneginf: Any -isposinf: Any -pad: Any -poly: Any -roots: Any -polyint: Any -polyder: Any -polyadd: Any -polysub: Any -polymul: Any -polydiv: Any -polyval: Any -poly1d: Any -polyfit: Any -RankWarning: Any -issubclass_: Any -issubsctype: Any -issubdtype: Any -deprecate: Any -deprecate_with_doc: Any -get_include: Any -info: Any -source: Any -who: Any -lookfor: Any -byte_bounds: Any -safe_eval: Any -ediff1d: Any -intersect1d: Any -setxor1d: Any -union1d: Any -setdiff1d: Any -unique: Any -in1d: Any -isin: Any -savetxt: Any -loadtxt: Any -genfromtxt: Any -ndfromtxt: Any -mafromtxt: Any -recfromtxt: Any -recfromcsv: Any -load: Any -loads: Any -save: Any -savez: Any -savez_compressed: Any -packbits: Any -unpackbits: Any -fromregex: Any -DataSource: Any -nansum: Any -nanmax: Any -nanmin: Any -nanargmax: Any -nanargmin: Any -nanmean: Any -nanmedian: Any -nanpercentile: Any -nanvar: Any -nanstd: Any -nanprod: Any -nancumsum: Any -nancumprod: Any -nanquantile: Any -histogram: Any -histogramdd: Any -histogram_bin_edges: Any +import math as math +from typing import Any, List + +from numpy import ( + ndenumerate as ndenumerate, + ndindex as ndindex, +) + +from numpy.version import version + +from numpy.lib import ( + format as format, + mixins as mixins, + scimath as scimath, + stride_tricks as stride_stricks, +) + +from numpy.lib._version import ( + NumpyVersion as NumpyVersion, +) + +from numpy.lib.arraypad import ( + pad as pad, +) + +from numpy.lib.arraysetops import ( + ediff1d as ediff1d, + intersect1d as intersect1d, + setxor1d as setxor1d, + union1d as union1d, + setdiff1d as setdiff1d, + unique as unique, + in1d as in1d, + isin as isin, +) + +from numpy.lib.arrayterator import ( + Arrayterator as Arrayterator, +) + +from numpy.lib.function_base import ( + select as select, + piecewise as piecewise, + trim_zeros as trim_zeros, + copy as copy, + iterable as iterable, + percentile as percentile, + diff as diff, + gradient as gradient, + angle as angle, + unwrap as unwrap, + sort_complex as sort_complex, + disp as disp, + flip as flip, + rot90 as rot90, + extract as extract, + place as place, + vectorize as vectorize, + asarray_chkfinite as asarray_chkfinite, + average as average, + bincount as bincount, + digitize as digitize, + cov as cov, + corrcoef as corrcoef, + msort as msort, + median as median, + sinc as sinc, + hamming as hamming, + hanning as hanning, + bartlett as bartlett, + blackman as blackman, + kaiser as kaiser, + trapz as trapz, + i0 as i0, + add_newdoc as add_newdoc, + add_docstring as add_docstring, + meshgrid as meshgrid, + delete as delete, + insert as insert, + append as append, + interp as interp, + add_newdoc_ufunc as add_newdoc_ufunc, + quantile as quantile, +) + +from numpy.lib.index_tricks import ( + ravel_multi_index as ravel_multi_index, + unravel_index as unravel_index, + mgrid as mgrid, + ogrid as ogrid, + r_ as r_, + c_ as c_, + s_ as s_, + index_exp as index_exp, + ix_ as ix_, + fill_diagonal as fill_diagonal, + diag_indices as diag_indices, + diag_indices_from as diag_indices_from, +) + +from numpy.lib.nanfunctions import ( + nansum as nansum, + nanmax as nanmax, + nanmin as nanmin, + nanargmax as nanargmax, + nanargmin as nanargmin, + nanmean as nanmean, + nanmedian as nanmedian, + nanpercentile as nanpercentile, + nanvar as nanvar, + nanstd as nanstd, + nanprod as nanprod, + nancumsum as nancumsum, + nancumprod as nancumprod, + nanquantile as nanquantile, +) + +from numpy.lib.npyio import ( + savetxt as savetxt, + loadtxt as loadtxt, + genfromtxt as genfromtxt, + recfromtxt as recfromtxt, + recfromcsv as recfromcsv, + load as load, + loads as loads, + save as save, + savez as savez, + savez_compressed as savez_compressed, + packbits as packbits, + unpackbits as unpackbits, + fromregex as fromregex, + DataSource as DataSource, +) + +from numpy.lib.polynomial import ( + poly as poly, + roots as roots, + polyint as polyint, + polyder as polyder, + polyadd as polyadd, + polysub as polysub, + polymul as polymul, + polydiv as polydiv, + polyval as polyval, + polyfit as polyfit, + RankWarning as RankWarning, + poly1d as poly1d, +) + +from numpy.lib.shape_base import ( + column_stack as column_stack, + row_stack as row_stack, + dstack as dstack, + array_split as array_split, + split as split, + hsplit as hsplit, + vsplit as vsplit, + dsplit as dsplit, + apply_over_axes as apply_over_axes, + expand_dims as expand_dims, + apply_along_axis as apply_along_axis, + kron as kron, + tile as tile, + get_array_wrap as get_array_wrap, + take_along_axis as take_along_axis, + put_along_axis as put_along_axis, +) + +from numpy.lib.stride_tricks import ( + broadcast_to as broadcast_to, + broadcast_arrays as broadcast_arrays, + broadcast_shapes as broadcast_shapes, +) + +from numpy.lib.twodim_base import ( + diag as diag, + diagflat as diagflat, + eye as eye, + fliplr as fliplr, + flipud as flipud, + tri as tri, + triu as triu, + tril as tril, + vander as vander, + histogram2d as histogram2d, + mask_indices as mask_indices, + tril_indices as tril_indices, + tril_indices_from as tril_indices_from, + triu_indices as triu_indices, + triu_indices_from as triu_indices_from, +) + +from numpy.lib.type_check import ( + mintypecode as mintypecode, + asfarray as asfarray, + real as real, + imag as imag, + iscomplex as iscomplex, + isreal as isreal, + iscomplexobj as iscomplexobj, + isrealobj as isrealobj, + nan_to_num as nan_to_num, + real_if_close as real_if_close, + typename as typename, + common_type as common_type, +) + +from numpy.lib.ufunclike import ( + fix as fix, + isposinf as isposinf, + isneginf as isneginf, +) + +from numpy.lib.utils import ( + issubclass_ as issubclass_, + issubsctype as issubsctype, + issubdtype as issubdtype, + deprecate as deprecate, + deprecate_with_doc as deprecate_with_doc, + get_include as get_include, + info as info, + source as source, + who as who, + lookfor as lookfor, + byte_bounds as byte_bounds, + safe_eval as safe_eval, +) + +__all__: List[str] + +__version__ = version +emath = scimath +tracemalloc_domain: int diff --git a/numpy/lib/_datasource.py b/numpy/lib/_datasource.py index 7a23b165158c..c790a6462a28 100644 --- a/numpy/lib/_datasource.py +++ b/numpy/lib/_datasource.py @@ -35,7 +35,6 @@ """ import os -import shutil import io from numpy.core.overrides import set_module @@ -257,6 +256,8 @@ def __init__(self, destpath=os.curdir): def __del__(self): # Remove temp directories if hasattr(self, '_istmpdest') and self._istmpdest: + import shutil + shutil.rmtree(self._destpath) def _iszip(self, filename): @@ -319,8 +320,9 @@ def _cache(self, path): Creates a copy of the file in the datasource cache. """ - # We import these here because importing urllib is slow and + # We import these here because importing them is slow and # a significant fraction of numpy's total import time. + import shutil from urllib.request import urlopen from urllib.error import URLError diff --git a/numpy/lib/_version.py b/numpy/lib/_version.py index d4098acb5aea..bfac5f814501 100644 --- a/numpy/lib/_version.py +++ b/numpy/lib/_version.py @@ -15,7 +15,7 @@ class NumpyVersion(): """Parse and compare numpy version strings. NumPy has the following versioning scheme (numbers given are examples; they - can be > 9) in principle): + can be > 9 in principle): - Released version: '1.8.0', '1.8.1', etc. - Alpha: '1.8.0a1', '1.8.0a2', etc. @@ -54,7 +54,7 @@ class NumpyVersion(): def __init__(self, vstring): self.vstring = vstring - ver_main = re.match(r'\d[.]\d+[.]\d+', vstring) + ver_main = re.match(r'\d+\.\d+\.\d+', vstring) if not ver_main: raise ValueError("Not a valid numpy version string") @@ -151,5 +151,5 @@ def __gt__(self, other): def __ge__(self, other): return self._compare(other) >= 0 - def __repr(self): + def __repr__(self): return "NumpyVersion(%s)" % self.vstring diff --git a/numpy/lib/_version.pyi b/numpy/lib/_version.pyi new file mode 100644 index 000000000000..3581d639bcdd --- /dev/null +++ b/numpy/lib/_version.pyi @@ -0,0 +1,19 @@ +from typing import Union, List + +__all__: List[str] + +class NumpyVersion: + vstring: str + version: str + major: int + minor: int + bugfix: int + pre_release: str + is_devversion: bool + def __init__(self, vstring: str) -> None: ... + def __lt__(self, other: Union[str, NumpyVersion]) -> bool: ... + def __le__(self, other: Union[str, NumpyVersion]) -> bool: ... + def __eq__(self, other: Union[str, NumpyVersion]) -> bool: ... # type: ignore[override] + def __ne__(self, other: Union[str, NumpyVersion]) -> bool: ... # type: ignore[override] + def __gt__(self, other: Union[str, NumpyVersion]) -> bool: ... + def __ge__(self, other: Union[str, NumpyVersion]) -> bool: ... diff --git a/numpy/lib/arraypad.pyi b/numpy/lib/arraypad.pyi new file mode 100644 index 000000000000..64e3e133117a --- /dev/null +++ b/numpy/lib/arraypad.pyi @@ -0,0 +1,5 @@ +from typing import List + +__all__: List[str] + +def pad(array, pad_width, mode=..., **kwargs): ... diff --git a/numpy/lib/arraysetops.py b/numpy/lib/arraysetops.py index 6c6c1ff809ce..7600e17be88b 100644 --- a/numpy/lib/arraysetops.py +++ b/numpy/lib/arraysetops.py @@ -209,6 +209,16 @@ def unique(ar, return_index=False, return_inverse=False, flattened subarrays are sorted in lexicographic order starting with the first element. + .. versionchanged: NumPy 1.21 + If nan values are in the input array, a single nan is put + to the end of the sorted unique values. + + Also for complex arrays all NaN values are considered equivalent + (no matter whether the NaN is in the real or imaginary part). + As the representant for the returned array the smallest one in the + lexicographical order is chosen - see np.sort for how the lexicographical + order is defined for complex arrays. + Examples -------- >>> np.unique([1, 1, 2, 2, 3, 3]) @@ -324,7 +334,16 @@ def _unique1d(ar, return_index=False, return_inverse=False, aux = ar mask = np.empty(aux.shape, dtype=np.bool_) mask[:1] = True - mask[1:] = aux[1:] != aux[:-1] + if aux.shape[0] > 0 and aux.dtype.kind in "cfmM" and np.isnan(aux[-1]): + if aux.dtype.kind == "c": # for complex all NaNs are considered equivalent + aux_firstnan = np.searchsorted(np.isnan(aux), True, side='left') + else: + aux_firstnan = np.searchsorted(aux, aux[-1], side='left') + mask[1:aux_firstnan] = (aux[1:aux_firstnan] != aux[:aux_firstnan - 1]) + mask[aux_firstnan] = True + mask[aux_firstnan + 1:] = False + else: + mask[1:] = aux[1:] != aux[:-1] ret = (aux[mask],) if return_index: @@ -565,6 +584,10 @@ def in1d(ar1, ar2, assume_unique=False, invert=False): ar1 = np.asarray(ar1).ravel() ar2 = np.asarray(ar2).ravel() + # Ensure that iteration through object arrays yields size-1 arrays + if ar2.dtype == object: + ar2 = ar2.reshape(-1, 1) + # Check if one of the arrays may contain arbitrary objects contains_object = ar1.dtype.hasobject or ar2.dtype.hasobject diff --git a/numpy/lib/arraysetops.pyi b/numpy/lib/arraysetops.pyi new file mode 100644 index 000000000000..029aa147492f --- /dev/null +++ b/numpy/lib/arraysetops.pyi @@ -0,0 +1,12 @@ +from typing import List + +__all__: List[str] + +def ediff1d(ary, to_end=..., to_begin=...): ... +def unique(ar, return_index=..., return_inverse=..., return_counts=..., axis=...): ... +def intersect1d(ar1, ar2, assume_unique=..., return_indices=...): ... +def setxor1d(ar1, ar2, assume_unique=...): ... +def in1d(ar1, ar2, assume_unique=..., invert=...): ... +def isin(element, test_elements, assume_unique=..., invert=...): ... +def union1d(ar1, ar2): ... +def setdiff1d(ar1, ar2, assume_unique=...): ... diff --git a/numpy/lib/arrayterator.pyi b/numpy/lib/arrayterator.pyi new file mode 100644 index 000000000000..39d6fd84354e --- /dev/null +++ b/numpy/lib/arrayterator.pyi @@ -0,0 +1,53 @@ +import sys +from typing import ( + List, + Any, + TypeVar, + Generator, + List, + Union, + Tuple, + overload, +) + +from numpy import ndarray, dtype, generic +from numpy.typing import DTypeLike + +# TODO: Set a shape bound once we've got proper shape support +_Shape = TypeVar("_Shape", bound=Any) +_DType = TypeVar("_DType", bound=dtype[Any]) +_ScalarType = TypeVar("_ScalarType", bound=generic) + +_Index = Union[ + Union[ellipsis, int, slice], + Tuple[Union[ellipsis, int, slice], ...], +] + +__all__: List[str] + +# NOTE: In reality `Arrayterator` does not actually inherit from `ndarray`, +# but its ``__getattr__` method does wrap around the former and thus has +# access to all its methods + +class Arrayterator(ndarray[_Shape, _DType]): + var: ndarray[_Shape, _DType] # type: ignore[assignment] + buf_size: None | int + start: List[int] + stop: List[int] + step: List[int] + + @property # type: ignore[misc] + def shape(self) -> Tuple[int, ...]: ... + @property + def flat( # type: ignore[override] + self: ndarray[Any, dtype[_ScalarType]] + ) -> Generator[_ScalarType, None, None]: ... + def __init__( + self, var: ndarray[_Shape, _DType], buf_size: None | int = ... + ) -> None: ... + @overload + def __array__(self, dtype: None = ...) -> ndarray[Any, _DType]: ... + @overload + def __array__(self, dtype: DTypeLike) -> ndarray[Any, dtype[Any]]: ... + def __getitem__(self, index: _Index) -> Arrayterator[Any, _DType]: ... + def __iter__(self) -> Generator[ndarray[Any, _DType], None, None]: ... diff --git a/numpy/lib/format.py b/numpy/lib/format.py index 5d951e262570..ead6a0420251 100644 --- a/numpy/lib/format.py +++ b/numpy/lib/format.py @@ -173,6 +173,7 @@ __all__ = [] +EXPECTED_KEYS = {'descr', 'fortran_order', 'shape'} MAGIC_PREFIX = b'\x93NUMPY' MAGIC_LEN = len(MAGIC_PREFIX) + 2 ARRAY_ALIGN = 64 # plausible values are powers of 2 between 16 and 4096 @@ -378,7 +379,7 @@ def _wrap_header(header, version): header_prefix = magic(*version) + struct.pack(fmt, hlen + padlen) except struct.error: msg = "Header length {} too big for version={}".format(hlen, version) - raise ValueError(msg) + raise ValueError(msg) from None # Pad the header with spaces and a final newline such that the magic # string, the header-length short and the header are aligned on a @@ -432,7 +433,6 @@ def _write_array_header(fp, d, version=None): header.append("'%s': %s, " % (key, repr(value))) header.append("}") header = "".join(header) - header = _filter_header(header) if version is None: header = _wrap_header_guess_version(header) else: @@ -590,23 +590,27 @@ def _read_array_header(fp, version): # "shape" : tuple of int # "fortran_order" : bool # "descr" : dtype.descr - header = _filter_header(header) + # Versions (2, 0) and (1, 0) could have been created by a Python 2 + # implementation before header filtering was implemented. + if version <= (2, 0): + header = _filter_header(header) try: d = safe_eval(header) except SyntaxError as e: - msg = "Cannot parse header: {!r}\nException: {!r}" - raise ValueError(msg.format(header, e)) + msg = "Cannot parse header: {!r}" + raise ValueError(msg.format(header)) from e if not isinstance(d, dict): msg = "Header is not a dictionary: {!r}" raise ValueError(msg.format(d)) - keys = sorted(d.keys()) - if keys != ['descr', 'fortran_order', 'shape']: + + if EXPECTED_KEYS != d.keys(): + keys = sorted(d.keys()) msg = "Header does not contain the correct keys: {!r}" - raise ValueError(msg.format(keys)) + raise ValueError(msg.format(d.keys())) # Sanity-check the values. if (not isinstance(d['shape'], tuple) or - not numpy.all([isinstance(x, int) for x in d['shape']])): + not all(isinstance(x, int) for x in d['shape'])): msg = "shape is not valid: {!r}" raise ValueError(msg.format(d['shape'])) if not isinstance(d['fortran_order'], bool): @@ -614,9 +618,9 @@ def _read_array_header(fp, version): raise ValueError(msg.format(d['fortran_order'])) try: dtype = descr_to_dtype(d['descr']) - except TypeError: + except TypeError as e: msg = "descr is not a valid dtype descriptor: {!r}" - raise ValueError(msg.format(d['descr'])) + raise ValueError(msg.format(d['descr'])) from e return d['shape'], d['fortran_order'], dtype diff --git a/numpy/lib/format.pyi b/numpy/lib/format.pyi new file mode 100644 index 000000000000..4c44d57bf827 --- /dev/null +++ b/numpy/lib/format.pyi @@ -0,0 +1,28 @@ +import sys +from typing import Any, List, Set + +if sys.version_info >= (3, 8): + from typing import Literal, Final +else: + from typing_extensions import Literal, Final + +__all__: List[str] + +EXPECTED_KEYS: Final[Set[str]] +MAGIC_PREFIX: Final[bytes] +MAGIC_LEN: Literal[8] +ARRAY_ALIGN: Literal[64] +BUFFER_SIZE: Literal[262144] # 2**18 + +def magic(major, minor): ... +def read_magic(fp): ... +def dtype_to_descr(dtype): ... +def descr_to_dtype(descr): ... +def header_data_from_array_1_0(array): ... +def write_array_header_1_0(fp, d): ... +def write_array_header_2_0(fp, d): ... +def read_array_header_1_0(fp): ... +def read_array_header_2_0(fp): ... +def write_array(fp, array, version=..., allow_pickle=..., pickle_kwargs=...): ... +def read_array(fp, allow_pickle=..., pickle_kwargs=...): ... +def open_memmap(filename, mode=..., dtype=..., shape=..., fortran_order=..., version=...): ... diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py index 984f3086e5b7..783d45c2fbfc 100644 --- a/numpy/lib/function_base.py +++ b/numpy/lib/function_base.py @@ -8,7 +8,7 @@ import numpy.core.numeric as _nx from numpy.core import transpose from numpy.core.numeric import ( - ones, zeros, arange, concatenate, array, asarray, asanyarray, empty, + ones, zeros_like, arange, concatenate, array, asarray, asanyarray, empty, ndarray, around, floor, ceil, take, dot, where, intp, integer, isscalar, absolute ) @@ -593,7 +593,7 @@ def piecewise(x, condlist, funclist, *args, **kw): not isinstance(condlist[0], (list, ndarray)) and x.ndim != 0): condlist = [condlist] - condlist = array(condlist, dtype=bool) + condlist = asarray(condlist, dtype=bool) n = len(condlist) if n == n2 - 1: # compute the "otherwise" condition. @@ -606,7 +606,7 @@ def piecewise(x, condlist, funclist, *args, **kw): .format(n, n, n+1) ) - y = zeros(x.shape, x.dtype) + y = zeros_like(x) for cond, func in zip(condlist, funclist): if not isinstance(func, collections.abc.Callable): y[cond] = func @@ -671,11 +671,22 @@ def select(condlist, choicelist, default=0): raise ValueError("select with an empty condition list is not possible") choicelist = [np.asarray(choice) for choice in choicelist] - choicelist.append(np.asarray(default)) + + try: + intermediate_dtype = np.result_type(*choicelist) + except TypeError as e: + msg = f'Choicelist elements do not have a common dtype: {e}' + raise TypeError(msg) from None + default_array = np.asarray(default) + choicelist.append(default_array) # need to get the result type before broadcasting for correct scalar # behaviour - dtype = np.result_type(*choicelist) + try: + dtype = np.result_type(intermediate_dtype, default_array) + except TypeError as e: + msg = f'Choicelists and default value do not have a common dtype: {e}' + raise TypeError(msg) from None # Convert conditions to arrays and broadcast conditions and choices # as the shape is needed for the result. Doing it separately optimizes @@ -846,7 +857,7 @@ def gradient(f, *varargs, axis=None, edge_order=1): Returns ------- gradient : ndarray or list of ndarray - A set of ndarrays (or a single ndarray if there is only one dimension) + A list of ndarrays (or a single ndarray if there is only one dimension) corresponding to the derivatives of f with respect to each dimension. Each derivative has the same shape as f. @@ -1290,7 +1301,7 @@ def _interp_dispatcher(x, xp, fp, left=None, right=None, period=None): @array_function_dispatch(_interp_dispatcher) def interp(x, xp, fp, left=None, right=None, period=None): """ - One-dimensional linear interpolation. + One-dimensional linear interpolation for monotonically increasing sample points. Returns the one-dimensional piecewise linear interpolant to a function with given discrete data points (`xp`, `fp`), evaluated at `x`. @@ -1337,8 +1348,8 @@ def interp(x, xp, fp, left=None, right=None, period=None): -------- scipy.interpolate - Notes - ----- + Warnings + -------- The x-coordinate sequence is expected to be increasing, but this is not explicitly enforced. However, if the sequence `xp` is non-increasing, interpolation results are meaningless. @@ -1485,26 +1496,40 @@ def angle(z, deg=False): return a -def _unwrap_dispatcher(p, discont=None, axis=None): +def _unwrap_dispatcher(p, discont=None, axis=None, *, period=None): return (p,) @array_function_dispatch(_unwrap_dispatcher) -def unwrap(p, discont=pi, axis=-1): - """ - Unwrap by changing deltas between values to 2*pi complement. +def unwrap(p, discont=None, axis=-1, *, period=2*pi): + r""" + Unwrap by taking the complement of large deltas with respect to the period. + + This unwraps a signal `p` by changing elements which have an absolute + difference from their predecessor of more than ``max(discont, period/2)`` + to their `period`-complementary values. - Unwrap radian phase `p` by changing absolute jumps greater than - `discont` to their 2*pi complement along the given axis. + For the default case where `period` is :math:`2\pi` and is `discont` is + :math:`\pi`, this unwraps a radian phase `p` such that adjacent differences + are never greater than :math:`\pi` by adding :math:`2k\pi` for some + integer :math:`k`. Parameters ---------- p : array_like Input array. discont : float, optional - Maximum discontinuity between values, default is ``pi``. + Maximum discontinuity between values, default is ``period/2``. + Values below ``period/2`` are treated as if they were ``period/2``. + To have an effect different from the default, `discont` should be + larger than ``period/2``. axis : int, optional Axis along which unwrap will operate, default is the last axis. + period: float, optional + Size of the range over which the input wraps. By default, it is + ``2 pi``. + + .. versionadded:: 1.21.0 Returns ------- @@ -1517,9 +1542,9 @@ def unwrap(p, discont=pi, axis=-1): Notes ----- - If the discontinuity in `p` is smaller than ``pi``, but larger than - `discont`, no unwrapping is done because taking the 2*pi complement - would only make the discontinuity larger. + If the discontinuity in `p` is smaller than ``period/2``, + but larger than `discont`, no unwrapping is done because taking + the complement would only make the discontinuity larger. Examples -------- @@ -1529,19 +1554,44 @@ def unwrap(p, discont=pi, axis=-1): array([ 0. , 0.78539816, 1.57079633, 5.49778714, 6.28318531]) # may vary >>> np.unwrap(phase) array([ 0. , 0.78539816, 1.57079633, -0.78539816, 0. ]) # may vary - + >>> np.unwrap([0, 1, 2, -1, 0], period=4) + array([0, 1, 2, 3, 4]) + >>> np.unwrap([ 1, 2, 3, 4, 5, 6, 1, 2, 3], period=6) + array([1, 2, 3, 4, 5, 6, 7, 8, 9]) + >>> np.unwrap([2, 3, 4, 5, 2, 3, 4, 5], period=4) + array([2, 3, 4, 5, 6, 7, 8, 9]) + >>> phase_deg = np.mod(np.linspace(0 ,720, 19), 360) - 180 + >>> np.unwrap(phase_deg, period=360) + array([-180., -140., -100., -60., -20., 20., 60., 100., 140., + 180., 220., 260., 300., 340., 380., 420., 460., 500., + 540.]) """ p = asarray(p) nd = p.ndim dd = diff(p, axis=axis) + if discont is None: + discont = period/2 slice1 = [slice(None, None)]*nd # full slices slice1[axis] = slice(1, None) slice1 = tuple(slice1) - ddmod = mod(dd + pi, 2*pi) - pi - _nx.copyto(ddmod, pi, where=(ddmod == -pi) & (dd > 0)) + dtype = np.result_type(dd, period) + if _nx.issubdtype(dtype, _nx.integer): + interval_high, rem = divmod(period, 2) + boundary_ambiguous = rem == 0 + else: + interval_high = period / 2 + boundary_ambiguous = True + interval_low = -interval_high + ddmod = mod(dd - interval_low, period) + interval_low + if boundary_ambiguous: + # for `mask = (abs(dd) == period/2)`, the above line made + # `ddmod[mask] == -period/2`. correct these such that + # `ddmod[mask] == sign(dd[mask])*period/2`. + _nx.copyto(ddmod, interval_high, + where=(ddmod == interval_low) & (dd > 0)) ph_correct = ddmod - dd _nx.copyto(ph_correct, 0, where=abs(dd) < discont) - up = array(p, copy=True, dtype='d') + up = array(p, copy=True, dtype=dtype) up[slice1] = p[slice1] + ph_correct.cumsum(axis) return up @@ -2191,15 +2241,14 @@ def _vectorize_call(self, func, args): ufunc, otypes = self._get_ufunc_and_otypes(func=func, args=args) # Convert args to object arrays first - inputs = [array(a, copy=False, subok=True, dtype=object) - for a in args] + inputs = [asanyarray(a, dtype=object) for a in args] outputs = ufunc(*inputs) if ufunc.nout == 1: - res = array(outputs, copy=False, subok=True, dtype=otypes[0]) + res = asanyarray(outputs, dtype=otypes[0]) else: - res = tuple([array(x, copy=False, subok=True, dtype=t) + res = tuple([asanyarray(x, dtype=t) for x, t in zip(outputs, otypes)]) return res @@ -3947,11 +3996,10 @@ def _quantile_is_valid(q): # avoid expensive reductions, relevant for arrays with < O(1000) elements if q.ndim == 1 and q.size < 10: for i in range(q.size): - if q[i] < 0.0 or q[i] > 1.0: + if not (0.0 <= q[i] <= 1.0): return False else: - # faster than any() - if np.count_nonzero(q < 0.0) or np.count_nonzero(q > 1.0): + if not (np.all(0 <= q) and np.all(q <= 1)): return False return True @@ -4080,11 +4128,18 @@ def _trapz_dispatcher(y, x=None, dx=None, axis=None): @array_function_dispatch(_trapz_dispatcher) def trapz(y, x=None, dx=1.0, axis=-1): - """ + r""" Integrate along the given axis using the composite trapezoidal rule. - Integrate `y` (`x`) along given axis. - + If `x` is provided, the integration happens in sequence along its + elements - they are not sorted. + + Integrate `y` (`x`) along each 1d slice on the given axis, compute + :math:`\int y(x) dx`. + When `x` is specified, this integrates along the parametric curve, + computing :math:`\int_t y(t) dt = + \int_t y(t) \left.\frac{dx}{dt}\right|_{x=x(t)} dt`. + Parameters ---------- y : array_like @@ -4100,9 +4155,12 @@ def trapz(y, x=None, dx=1.0, axis=-1): Returns ------- - trapz : float - Definite integral as approximated by trapezoidal rule. - + trapz : float or ndarray + Definite integral of 'y' = n-dimensional array as approximated along + a single axis by the trapezoidal rule. If 'y' is a 1-dimensional array, + then the result is a float. If 'n' is greater than 1, then the result + is an 'n-1' dimensional array. + See Also -------- sum, cumsum @@ -4131,6 +4189,20 @@ def trapz(y, x=None, dx=1.0, axis=-1): 8.0 >>> np.trapz([1,2,3], dx=2) 8.0 + + Using a decreasing `x` corresponds to integrating in reverse: + + >>> np.trapz([1,2,3], x=[8,6,4]) + -8.0 + + More generally `x` is used to integrate along a parametric curve. + This finds the area of a circle, noting we repeat the sample which closes + the curve: + + >>> theta = np.linspace(0, 2 * np.pi, num=1000, endpoint=True) + >>> np.trapz(np.cos(theta), x=np.sin(theta)) + 3.141571941375841 + >>> a = np.arange(6).reshape(2, 3) >>> a array([[0, 1, 2], @@ -4139,7 +4211,6 @@ def trapz(y, x=None, dx=1.0, axis=-1): array([1.5, 2.5, 3.5]) >>> np.trapz(a, axis=1) array([2., 8.]) - """ y = asanyarray(y) if x is None: @@ -4245,7 +4316,7 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): See Also -------- mgrid : Construct a multi-dimensional "meshgrid" using indexing notation. - ogrid : Construct an open multi-dimensional "meshgrid" using indexing + ogrid : Construct an open multi-dimensional "meshgrid" using indexing notation. Examples @@ -4274,7 +4345,8 @@ def meshgrid(*xi, copy=True, sparse=False, indexing='xy'): >>> y = np.arange(-5, 5, 0.1) >>> xx, yy = np.meshgrid(x, y, sparse=True) >>> z = np.sin(xx**2 + yy**2) / (xx**2 + yy**2) - >>> h = plt.contourf(x,y,z) + >>> h = plt.contourf(x, y, z) + >>> plt.axis('scaled') >>> plt.show() """ diff --git a/numpy/lib/function_base.pyi b/numpy/lib/function_base.pyi new file mode 100644 index 000000000000..69c615c9c603 --- /dev/null +++ b/numpy/lib/function_base.pyi @@ -0,0 +1,57 @@ +from typing import List + +from numpy import ( + vectorize as vectorize, +) + +from numpy.core.function_base import ( + add_newdoc as add_newdoc, +) + +from numpy.core.multiarray import ( + add_docstring as add_docstring, + bincount as bincount, +) +from numpy.core.umath import _add_newdoc_ufunc + +__all__: List[str] + +add_newdoc_ufunc = _add_newdoc_ufunc + +def rot90(m, k=..., axes = ...): ... +def flip(m, axis=...): ... +def iterable(y): ... +def average(a, axis=..., weights=..., returned=...): ... +def asarray_chkfinite(a, dtype=..., order=...): ... +def piecewise(x, condlist, funclist, *args, **kw): ... +def select(condlist, choicelist, default=...): ... +def copy(a, order=..., subok=...): ... +def gradient(f, *varargs, axis=..., edge_order=...): ... +def diff(a, n=..., axis=..., prepend = ..., append = ...): ... +def interp(x, xp, fp, left=..., right=..., period=...): ... +def angle(z, deg=...): ... +def unwrap(p, discont = ..., axis=..., *, period=...): ... +def sort_complex(a): ... +def trim_zeros(filt, trim=...): ... +def extract(condition, arr): ... +def place(arr, mask, vals): ... +def disp(mesg, device=..., linefeed=...): ... +def cov(m, y=..., rowvar=..., bias=..., ddof=..., fweights=..., aweights=..., *, dtype=...): ... +def corrcoef(x, y=..., rowvar=..., bias = ..., ddof = ..., *, dtype=...): ... +def blackman(M): ... +def bartlett(M): ... +def hanning(M): ... +def hamming(M): ... +def i0(x): ... +def kaiser(M, beta): ... +def sinc(x): ... +def msort(a): ... +def median(a, axis=..., out=..., overwrite_input=..., keepdims=...): ... +def percentile(a, q, axis=..., out=..., overwrite_input=..., interpolation=..., keepdims=...): ... +def quantile(a, q, axis=..., out=..., overwrite_input=..., interpolation=..., keepdims=...): ... +def trapz(y, x=..., dx=..., axis=...): ... +def meshgrid(*xi, copy=..., sparse=..., indexing=...): ... +def delete(arr, obj, axis=...): ... +def insert(arr, obj, values, axis=...): ... +def append(arr, values, axis=...): ... +def digitize(x, bins, right=...): ... diff --git a/numpy/lib/histograms.py b/numpy/lib/histograms.py index 1a9b41ceda6b..b6909bc1d9e0 100644 --- a/numpy/lib/histograms.py +++ b/numpy/lib/histograms.py @@ -562,7 +562,8 @@ def histogram_bin_edges(a, bins=10, range=None, weights=None): below, :math:`h` is the binwidth and :math:`n_h` is the number of bins. All estimators that compute bin counts are recast to bin width using the `ptp` of the data. The final bin count is obtained from - ``np.round(np.ceil(range / h))``. + ``np.round(np.ceil(range / h))``. The final bin width is often less + than what is returned by the estimators below. 'auto' (maximum of the 'sturges' and 'fd' estimators) A compromise to get a good value. For small datasets the Sturges @@ -678,7 +679,7 @@ def _histogram_dispatcher( def histogram(a, bins=10, range=None, normed=None, weights=None, density=None): r""" - Compute the histogram of a set of data. + Compute the histogram of a dataset. Parameters ---------- diff --git a/numpy/lib/histograms.pyi b/numpy/lib/histograms.pyi new file mode 100644 index 000000000000..25a33e3aea90 --- /dev/null +++ b/numpy/lib/histograms.pyi @@ -0,0 +1,7 @@ +from typing import List + +__all__: List[str] + +def histogram_bin_edges(a, bins=..., range=..., weights=...): ... +def histogram(a, bins=..., range=..., normed=..., weights=..., density=...): ... +def histogramdd(sample, bins=..., range=..., normed=..., weights=..., density=...): ... diff --git a/numpy/lib/index_tricks.py b/numpy/lib/index_tricks.py index 9d3de69ddc5d..72d8e9de4f75 100644 --- a/numpy/lib/index_tricks.py +++ b/numpy/lib/index_tricks.py @@ -6,7 +6,7 @@ import numpy.core.numeric as _nx from numpy.core.numeric import ( asarray, ScalarType, array, alltrue, cumprod, arange, ndim - ) +) from numpy.core.numerictypes import find_common_type, issubdtype import numpy.matrixlib as matrixlib @@ -25,7 +25,7 @@ 'ravel_multi_index', 'unravel_index', 'mgrid', 'ogrid', 'r_', 'c_', 's_', 'index_exp', 'ix_', 'ndenumerate', 'ndindex', 'fill_diagonal', 'diag_indices', 'diag_indices_from' - ] +] def _ix__dispatcher(*args): @@ -106,6 +106,7 @@ def ix_(*args): out.append(new) return tuple(out) + class nd_grid: """ Construct a multi-dimensional "meshgrid". @@ -167,7 +168,7 @@ def __getitem__(self, key): typ = float if self.sparse: nn = [_nx.arange(_x, dtype=_t) - for _x, _t in zip(size, (typ,)*len(size))] + for _x, _t in zip(size, (typ,)*len(size))] else: nn = _nx.indices(size, typ) for k in range(len(size)): @@ -248,11 +249,14 @@ class MGridClass(nd_grid): array([-1. , -0.5, 0. , 0.5, 1. ]) """ + def __init__(self): - super(MGridClass, self).__init__(sparse=False) + super().__init__(sparse=False) + mgrid = MGridClass() + class OGridClass(nd_grid): """ `nd_grid` instance which returns an open multi-dimensional "meshgrid". @@ -292,8 +296,10 @@ class OGridClass(nd_grid): [4]]), array([[0, 1, 2, 3, 4]])] """ + def __init__(self): - super(OGridClass, self).__init__(sparse=True) + super().__init__(sparse=True) + ogrid = OGridClass() @@ -357,7 +363,7 @@ def __getitem__(self, key): elif isinstance(item, str): if k != 0: raise ValueError("special directives must be the " - "first entry.") + "first entry.") if item in ('r', 'c'): matrix = True col = (item == 'c') @@ -376,8 +382,8 @@ def __getitem__(self, key): try: axis = int(item) continue - except (ValueError, TypeError): - raise ValueError("unknown special directive") + except (ValueError, TypeError) as e: + raise ValueError("unknown special directive") from e elif type(item) in ScalarType: newobj = array(item, ndmin=ndmin) scalars.append(len(objs)) @@ -420,6 +426,7 @@ def __len__(self): # etc. because otherwise we couldn't get the doc string to come out right # in help(r_) + class RClass(AxisConcatenator): """ Translates slice objects to concatenation along the first axis. @@ -518,8 +525,10 @@ class RClass(AxisConcatenator): def __init__(self): AxisConcatenator.__init__(self, 0) + r_ = RClass() + class CClass(AxisConcatenator): """ Translates slice objects to concatenation along the second axis. @@ -528,7 +537,7 @@ class CClass(AxisConcatenator): useful because of its common occurrence. In particular, arrays will be stacked along their last axis after being upgraded to at least 2-D with 1's post-pended to the shape (column vectors made out of 1-D arrays). - + See Also -------- column_stack : Stack 1-D arrays as columns into a 2-D array. @@ -751,6 +760,7 @@ def __getitem__(self, item): else: return item + index_exp = IndexExpression(maketuple=True) s_ = IndexExpression(maketuple=False) @@ -885,7 +895,7 @@ def fill_diagonal(a, val, wrap=False): # Explicit, fast formula for the common case. For 2-d arrays, we # accept rectangular ones. step = a.shape[1] + 1 - #This is needed to don't have tall matrix have the diagonal wrap. + # This is needed to don't have tall matrix have the diagonal wrap. if not wrap: end = a.shape[1] * a.shape[1] else: diff --git a/numpy/lib/index_tricks.pyi b/numpy/lib/index_tricks.pyi new file mode 100644 index 000000000000..a3bfef6b6406 --- /dev/null +++ b/numpy/lib/index_tricks.pyi @@ -0,0 +1,194 @@ +import sys +from typing import ( + Any, + Tuple, + TypeVar, + Generic, + overload, + List, + Union, + Sequence, +) + +from numpy import ( + # Circumvent a naming conflict with `AxisConcatenator.matrix` + matrix as _Matrix, + ndenumerate as ndenumerate, + ndindex as ndindex, + ndarray, + dtype, + integer, + str_, + bytes_, + bool_, + int_, + float_, + complex_, + intp, + _OrderCF, + _ModeKind, +) +from numpy.typing import ( + # Arrays + ArrayLike, + _NestedSequence, + _RecursiveSequence, + NDArray, + _ArrayLikeInt, + + # DTypes + DTypeLike, + _SupportsDType, + + # Shapes + _ShapeLike, +) + +if sys.version_info >= (3, 8): + from typing import Literal, SupportsIndex +else: + from typing_extensions import Literal, SupportsIndex + +_T = TypeVar("_T") +_DType = TypeVar("_DType", bound=dtype[Any]) +_BoolType = TypeVar("_BoolType", Literal[True], Literal[False]) +_TupType = TypeVar("_TupType", bound=Tuple[Any, ...]) +_ArrayType = TypeVar("_ArrayType", bound=ndarray[Any, Any]) + +__all__: List[str] + +@overload +def unravel_index( # type: ignore[misc] + indices: Union[int, integer[Any]], + shape: _ShapeLike, + order: _OrderCF = ... +) -> Tuple[intp, ...]: ... +@overload +def unravel_index( + indices: _ArrayLikeInt, + shape: _ShapeLike, + order: _OrderCF = ... +) -> Tuple[NDArray[intp], ...]: ... + +@overload +def ravel_multi_index( # type: ignore[misc] + multi_index: Sequence[Union[int, integer[Any]]], + dims: _ShapeLike, + mode: Union[_ModeKind, Tuple[_ModeKind, ...]] = ..., + order: _OrderCF = ... +) -> intp: ... +@overload +def ravel_multi_index( + multi_index: Sequence[_ArrayLikeInt], + dims: _ShapeLike, + mode: Union[_ModeKind, Tuple[_ModeKind, ...]] = ..., + order: _OrderCF = ... +) -> NDArray[intp]: ... + +@overload +def ix_(*args: _NestedSequence[_SupportsDType[_DType]]) -> Tuple[ndarray[Any, _DType], ...]: ... +@overload +def ix_(*args: _NestedSequence[str]) -> Tuple[NDArray[str_], ...]: ... +@overload +def ix_(*args: _NestedSequence[bytes]) -> Tuple[NDArray[bytes_], ...]: ... +@overload +def ix_(*args: _NestedSequence[bool]) -> Tuple[NDArray[bool_], ...]: ... +@overload +def ix_(*args: _NestedSequence[int]) -> Tuple[NDArray[int_], ...]: ... +@overload +def ix_(*args: _NestedSequence[float]) -> Tuple[NDArray[float_], ...]: ... +@overload +def ix_(*args: _NestedSequence[complex]) -> Tuple[NDArray[complex_], ...]: ... +@overload +def ix_(*args: _RecursiveSequence) -> Tuple[NDArray[Any], ...]: ... + +class nd_grid(Generic[_BoolType]): + sparse: _BoolType + def __init__(self, sparse: _BoolType = ...) -> None: ... + @overload + def __getitem__( + self: nd_grid[Literal[False]], + key: Union[slice, Sequence[slice]], + ) -> NDArray[Any]: ... + @overload + def __getitem__( + self: nd_grid[Literal[True]], + key: Union[slice, Sequence[slice]], + ) -> List[NDArray[Any]]: ... + +class MGridClass(nd_grid[Literal[False]]): + def __init__(self) -> None: ... + +mgrid: MGridClass + +class OGridClass(nd_grid[Literal[True]]): + def __init__(self) -> None: ... + +ogrid: OGridClass + +class AxisConcatenator: + axis: int + matrix: bool + ndmin: int + trans1d: int + def __init__( + self, + axis: int = ..., + matrix: bool = ..., + ndmin: int = ..., + trans1d: int = ..., + ) -> None: ... + @staticmethod + @overload + def concatenate( # type: ignore[misc] + *a: ArrayLike, axis: SupportsIndex = ..., out: None = ... + ) -> NDArray[Any]: ... + @staticmethod + @overload + def concatenate( + *a: ArrayLike, axis: SupportsIndex = ..., out: _ArrayType = ... + ) -> _ArrayType: ... + @staticmethod + def makemat( + data: ArrayLike, dtype: DTypeLike = ..., copy: bool = ... + ) -> _Matrix: ... + + # TODO: Sort out this `__getitem__` method + def __getitem__(self, key: Any) -> Any: ... + +class RClass(AxisConcatenator): + axis: Literal[0] + matrix: Literal[False] + ndmin: Literal[1] + trans1d: Literal[-1] + def __init__(self) -> None: ... + +r_: RClass + +class CClass(AxisConcatenator): + axis: Literal[-1] + matrix: Literal[False] + ndmin: Literal[2] + trans1d: Literal[0] + def __init__(self) -> None: ... + +c_: CClass + +class IndexExpression(Generic[_BoolType]): + maketuple: _BoolType + def __init__(self, maketuple: _BoolType) -> None: ... + @overload + def __getitem__(self, item: _TupType) -> _TupType: ... # type: ignore[misc] + @overload + def __getitem__(self: IndexExpression[Literal[True]], item: _T) -> Tuple[_T]: ... + @overload + def __getitem__(self: IndexExpression[Literal[False]], item: _T) -> _T: ... + +index_exp: IndexExpression[Literal[True]] +s_: IndexExpression[Literal[False]] + +def fill_diagonal(a: ndarray[Any, Any], val: Any, wrap: bool = ...) -> None: ... +def diag_indices(n: int, ndim: int = ...) -> Tuple[NDArray[int_], ...]: ... +def diag_indices_from(arr: ArrayLike) -> Tuple[NDArray[int_], ...]: ... + +# NOTE: see `numpy/__init__.pyi` for `ndenumerate` and `ndindex` diff --git a/numpy/lib/mixins.pyi b/numpy/lib/mixins.pyi new file mode 100644 index 000000000000..f137bb5bcf4b --- /dev/null +++ b/numpy/lib/mixins.pyi @@ -0,0 +1,62 @@ +from typing import List +from abc import ABCMeta, abstractmethod + +__all__: List[str] + +# NOTE: `NDArrayOperatorsMixin` is not formally an abstract baseclass, +# even though it's reliant on subclasses implementing `__array_ufunc__` + +class NDArrayOperatorsMixin(metaclass=ABCMeta): + @abstractmethod + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): ... + def __lt__(self, other): ... + def __le__(self, other): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def __gt__(self, other): ... + def __ge__(self, other): ... + def __add__(self, other): ... + def __radd__(self, other): ... + def __iadd__(self, other): ... + def __sub__(self, other): ... + def __rsub__(self, other): ... + def __isub__(self, other): ... + def __mul__(self, other): ... + def __rmul__(self, other): ... + def __imul__(self, other): ... + def __matmul__(self, other): ... + def __rmatmul__(self, other): ... + def __imatmul__(self, other): ... + def __truediv__(self, other): ... + def __rtruediv__(self, other): ... + def __itruediv__(self, other): ... + def __floordiv__(self, other): ... + def __rfloordiv__(self, other): ... + def __ifloordiv__(self, other): ... + def __mod__(self, other): ... + def __rmod__(self, other): ... + def __imod__(self, other): ... + def __divmod__(self, other): ... + def __rdivmod__(self, other): ... + def __pow__(self, other): ... + def __rpow__(self, other): ... + def __ipow__(self, other): ... + def __lshift__(self, other): ... + def __rlshift__(self, other): ... + def __ilshift__(self, other): ... + def __rshift__(self, other): ... + def __rrshift__(self, other): ... + def __irshift__(self, other): ... + def __and__(self, other): ... + def __rand__(self, other): ... + def __iand__(self, other): ... + def __xor__(self, other): ... + def __rxor__(self, other): ... + def __ixor__(self, other): ... + def __or__(self, other): ... + def __ror__(self, other): ... + def __ior__(self, other): ... + def __neg__(self): ... + def __pos__(self): ... + def __abs__(self): ... + def __invert__(self): ... diff --git a/numpy/lib/nanfunctions.py b/numpy/lib/nanfunctions.py index 409016adb7dd..2c2c3435bc1f 100644 --- a/numpy/lib/nanfunctions.py +++ b/numpy/lib/nanfunctions.py @@ -613,7 +613,7 @@ def nansum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue): -------- numpy.sum : Sum across array propagating NaNs. isnan : Show which elements are NaN. - isfinite: Show which elements are not NaN or +/-inf. + isfinite : Show which elements are not NaN or +/-inf. Notes ----- @@ -962,12 +962,16 @@ def _nanmedian1d(arr1d, overwrite_input=False): Private function for rank 1 arrays. Compute the median ignoring NaNs. See nanmedian for parameter usage """ - arr1d, overwrite_input = _remove_nan_1d(arr1d, - overwrite_input=overwrite_input) - if arr1d.size == 0: - return np.nan + arr1d_parsed, overwrite_input = _remove_nan_1d( + arr1d, overwrite_input=overwrite_input, + ) - return np.median(arr1d, overwrite_input=overwrite_input) + if arr1d_parsed.size == 0: + # Ensure that a nan-esque scalar of the appropiate type (and unit) + # is returned for `timedelta64` and `complexfloating` + return arr1d[-1] + + return np.median(arr1d_parsed, overwrite_input=overwrite_input) def _nanmedian(a, axis=None, out=None, overwrite_input=False): @@ -1008,10 +1012,12 @@ def _nanmedian_small(a, axis=None, out=None, overwrite_input=False): for i in range(np.count_nonzero(m.mask.ravel())): warnings.warn("All-NaN slice encountered", RuntimeWarning, stacklevel=4) + + fill_value = np.timedelta64("NaT") if m.dtype.kind == "m" else np.nan if out is not None: - out[...] = m.filled(np.nan) + out[...] = m.filled(fill_value) return out - return m.filled(np.nan) + return m.filled(fill_value) def _nanmedian_dispatcher( diff --git a/numpy/lib/nanfunctions.pyi b/numpy/lib/nanfunctions.pyi new file mode 100644 index 000000000000..447770a54494 --- /dev/null +++ b/numpy/lib/nanfunctions.pyi @@ -0,0 +1,54 @@ +from typing import List + +__all__: List[str] + +def nanmin(a, axis=..., out=..., keepdims=...): ... +def nanmax(a, axis=..., out=..., keepdims=...): ... +def nanargmin(a, axis=...): ... +def nanargmax(a, axis=...): ... +def nansum(a, axis=..., dtype=..., out=..., keepdims=...): ... +def nanprod(a, axis=..., dtype=..., out=..., keepdims=...): ... +def nancumsum(a, axis=..., dtype=..., out=...): ... +def nancumprod(a, axis=..., dtype=..., out=...): ... +def nanmean(a, axis=..., dtype=..., out=..., keepdims=...): ... +def nanmedian( + a, + axis=..., + out=..., + overwrite_input=..., + keepdims=..., +): ... +def nanpercentile( + a, + q, + axis=..., + out=..., + overwrite_input=..., + interpolation=..., + keepdims=..., +): ... +def nanquantile( + a, + q, + axis=..., + out=..., + overwrite_input=..., + interpolation=..., + keepdims=..., +): ... +def nanvar( + a, + axis=..., + dtype=..., + out=..., + ddof=..., + keepdims=..., +): ... +def nanstd( + a, + axis=..., + dtype=..., + out=..., + ddof=..., + keepdims=..., +): ... diff --git a/numpy/lib/npyio.py b/numpy/lib/npyio.py index 3b2de3e61e1f..9552579cf4e6 100644 --- a/numpy/lib/npyio.py +++ b/numpy/lib/npyio.py @@ -24,7 +24,7 @@ from numpy.compat import ( asbytes, asstr, asunicode, os_fspath, os_PathLike, - pickle, contextlib_nullcontext + pickle ) @@ -517,7 +517,7 @@ def save(file, arr, allow_pickle=True, fix_imports=True): # [1 2] [1 3] """ if hasattr(file, 'write'): - file_ctx = contextlib_nullcontext(file) + file_ctx = contextlib.nullcontext(file) else: file = os_fspath(file) if not file.endswith('.npy'): @@ -539,10 +539,11 @@ def _savez_dispatcher(file, *args, **kwds): def savez(file, *args, **kwds): """Save several arrays into a single file in uncompressed ``.npz`` format. - If arguments are passed in with no keywords, the corresponding variable - names, in the ``.npz`` file, are 'arr_0', 'arr_1', etc. If keyword - arguments are given, the corresponding variable names, in the ``.npz`` - file will match the keyword names. + Provide arrays as keyword arguments to store them under the + corresponding name in the output file: ``savez(fn, x=x, y=y)``. + + If arrays are specified as positional arguments, i.e., ``savez(fn, + x, y)``, their names will be `arr_0`, `arr_1`, etc. Parameters ---------- @@ -552,13 +553,12 @@ def savez(file, *args, **kwds): ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional - Arrays to save to the file. Since it is not possible for Python to - know the names of the arrays outside `savez`, the arrays will be saved - with names "arr_0", "arr_1", and so on. These arguments can be any - expression. + Arrays to save to the file. Please use keyword arguments (see + `kwds` below) to assign names to arrays. Arrays specified as + args will be named "arr_0", "arr_1", and so on. kwds : Keyword arguments, optional - Arrays to save to the file. Arrays will be saved in the file with the - keyword names. + Arrays to save to the file. Each array will be saved to the + output file with its corresponding keyword name. Returns ------- @@ -613,6 +613,7 @@ def savez(file, *args, **kwds): ['x', 'y'] >>> npzfile['x'] array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + """ _savez(file, args, kwds, False) @@ -627,9 +628,11 @@ def savez_compressed(file, *args, **kwds): """ Save several arrays into a single file in compressed ``.npz`` format. - If keyword arguments are given, then filenames are taken from the keywords. - If arguments are passed in with no keywords, then stored filenames are - arr_0, arr_1, etc. + Provide arrays as keyword arguments to store them under the + corresponding name in the output file: ``savez(fn, x=x, y=y)``. + + If arrays are specified as positional arguments, i.e., ``savez(fn, + x, y)``, their names will be `arr_0`, `arr_1`, etc. Parameters ---------- @@ -639,13 +642,12 @@ def savez_compressed(file, *args, **kwds): ``.npz`` extension will be appended to the filename if it is not already there. args : Arguments, optional - Arrays to save to the file. Since it is not possible for Python to - know the names of the arrays outside `savez`, the arrays will be saved - with names "arr_0", "arr_1", and so on. These arguments can be any - expression. + Arrays to save to the file. Please use keyword arguments (see + `kwds` below) to assign names to arrays. Arrays specified as + args will be named "arr_0", "arr_1", and so on. kwds : Keyword arguments, optional - Arrays to save to the file. Arrays will be saved in the file with the - keyword names. + Arrays to save to the file. Each array will be saved to the + output file with its corresponding keyword name. Returns ------- @@ -1585,7 +1587,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, column, individually. comments : str, optional The character used to indicate the start of a comment. - All the characters occurring on a line after a comment are discarded + All the characters occurring on a line after a comment are discarded. delimiter : str, int, or sequence, optional The string used to separate values. By default, any consecutive whitespaces act as delimiter. An integer or sequence of integers @@ -1612,15 +1614,15 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, ``usecols = (1, 4, 5)`` will extract the 2nd, 5th and 6th columns. names : {None, True, str, sequence}, optional If `names` is True, the field names are read from the first line after - the first `skip_header` lines. This line can optionally be proceeded + the first `skip_header` lines. This line can optionally be preceeded by a comment delimiter. If `names` is a sequence or a single-string of comma-separated names, the names will be used to define the field names in a structured dtype. If `names` is None, the names of the dtype fields will be used, if any. excludelist : sequence, optional A list of names to exclude. This list is appended to the default list - ['return','file','print']. Excluded names are appended an underscore: - for example, `file` would become `file_`. + ['return','file','print']. Excluded names are appended with an + underscore: for example, `file` would become `file_`. deletechars : str, optional A string combining invalid characters that must be deleted from the names. @@ -1629,7 +1631,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, autostrip : bool, optional Whether to automatically strip white spaces from the variables. replace_space : char, optional - Character(s) used in replacement of white spaces in the variables + Character(s) used in replacement of white spaces in the variable names. By default, use a '_'. case_sensitive : {True, False, 'upper', 'lower'}, optional If True, field names are case sensitive. @@ -1792,7 +1794,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None, fid_ctx = contextlib.closing(fid) else: fid = fname - fid_ctx = contextlib_nullcontext(fid) + fid_ctx = contextlib.nullcontext(fid) fhd = iter(fid) except TypeError as e: raise TypeError( diff --git a/numpy/lib/npyio.pyi b/numpy/lib/npyio.pyi new file mode 100644 index 000000000000..508357927b72 --- /dev/null +++ b/numpy/lib/npyio.pyi @@ -0,0 +1,104 @@ +from typing import Mapping, List, Any + +from numpy import ( + DataSource as DataSource, +) + +from numpy.core.multiarray import ( + packbits as packbits, + unpackbits as unpackbits, +) + +__all__: List[str] + +def loads(*args, **kwargs): ... + +class BagObj: + def __init__(self, obj): ... + def __getattribute__(self, key): ... + def __dir__(self): ... + +def zipfile_factory(file, *args, **kwargs): ... + +class NpzFile(Mapping[Any, Any]): + zip: Any + fid: Any + files: Any + allow_pickle: Any + pickle_kwargs: Any + f: Any + def __init__(self, fid, own_fid=..., allow_pickle=..., pickle_kwargs=...): ... + def __enter__(self): ... + def __exit__(self, exc_type, exc_value, traceback): ... + def close(self): ... + def __del__(self): ... + def __iter__(self): ... + def __len__(self): ... + def __getitem__(self, key): ... + def iteritems(self): ... + def iterkeys(self): ... + +def load(file, mmap_mode=..., allow_pickle=..., fix_imports=..., encoding=...): ... +def save(file, arr, allow_pickle=..., fix_imports=...): ... +def savez(file, *args, **kwds): ... +def savez_compressed(file, *args, **kwds): ... +def loadtxt( + fname, + dtype=..., + comments=..., + delimiter=..., + converters=..., + skiprows=..., + usecols=..., + unpack=..., + ndmin=..., + encoding=..., + max_rows=..., + *, + like=..., +): ... +def savetxt( + fname, + X, + fmt=..., + delimiter=..., + newline=..., + header=..., + footer=..., + comments=..., + encoding=..., +): ... +def fromregex(file, regexp, dtype, encoding=...): ... +def genfromtxt( + fname, + dtype=..., + comments=..., + delimiter=..., + skip_header=..., + skip_footer=..., + converters=..., + missing_values=..., + filling_values=..., + usecols=..., + names=..., + excludelist=..., + deletechars=..., + replace_space=..., + autostrip=..., + case_sensitive=..., + defaultfmt=..., + unpack=..., + usemask=..., + loose=..., + invalid_raise=..., + max_rows=..., + encoding=..., + *, + like=..., +): ... +def recfromtxt(fname, **kwargs): ... +def recfromcsv(fname, **kwargs): ... + +# NOTE: Deprecated +# def ndfromtxt(fname, **kwargs): ... +# def mafromtxt(fname, **kwargs): ... diff --git a/numpy/lib/polynomial.py b/numpy/lib/polynomial.py index 0fd9bbd79c6a..e9df783b4b76 100644 --- a/numpy/lib/polynomial.py +++ b/numpy/lib/polynomial.py @@ -708,8 +708,8 @@ def polyval(p, x): ``p[0]*x**(N-1) + p[1]*x**(N-2) + ... + p[N-2]*x + p[N-1]`` - If `x` is a sequence, then `p(x)` is returned for each element of `x`. - If `x` is another polynomial then the composite polynomial `p(x(t))` + If `x` is a sequence, then ``p(x)`` is returned for each element of ``x``. + If `x` is another polynomial then the composite polynomial ``p(x(t))`` is returned. Parameters @@ -1037,7 +1037,7 @@ def polydiv(u, v): return poly1d(q), poly1d(r) return q, r -_poly_mat = re.compile(r"[*][*]([0-9]*)") +_poly_mat = re.compile(r"\*\*([0-9]*)") def _raise_power(astr, wrap=70): n = 0 line1 = '' diff --git a/numpy/lib/polynomial.pyi b/numpy/lib/polynomial.pyi new file mode 100644 index 000000000000..7d38658d0202 --- /dev/null +++ b/numpy/lib/polynomial.pyi @@ -0,0 +1,19 @@ +from typing import List + +from numpy import ( + RankWarning as RankWarning, + poly1d as poly1d, +) + +__all__: List[str] + +def poly(seq_of_zeros): ... +def roots(p): ... +def polyint(p, m=..., k=...): ... +def polyder(p, m=...): ... +def polyfit(x, y, deg, rcond=..., full=..., w=..., cov=...): ... +def polyval(p, x): ... +def polyadd(a1, a2): ... +def polysub(a1, a2): ... +def polymul(a1, a2): ... +def polydiv(u, v): ... diff --git a/numpy/lib/scimath.py b/numpy/lib/scimath.py index 2b0d38c37f2b..ed9ffd295b90 100644 --- a/numpy/lib/scimath.py +++ b/numpy/lib/scimath.py @@ -572,10 +572,10 @@ def arctanh(x): Compute the inverse hyperbolic tangent of `x`. Return the "principal value" (for a description of this, see - `numpy.arctanh`) of `arctanh(x)`. For real `x` such that - `abs(x) < 1`, this is a real number. If `abs(x) > 1`, or if `x` is + `numpy.arctanh`) of ``arctanh(x)``. For real `x` such that + ``abs(x) < 1``, this is a real number. If `abs(x) > 1`, or if `x` is complex, the result is complex. Finally, `x = 1` returns``inf`` and - `x=-1` returns ``-inf``. + ``x=-1`` returns ``-inf``. Parameters ---------- @@ -597,7 +597,7 @@ def arctanh(x): ----- For an arctanh() that returns ``NAN`` when real `x` is not in the interval ``(-1,1)``, use `numpy.arctanh` (this latter, however, does - return +/-inf for `x = +/-1`). + return +/-inf for ``x = +/-1``). Examples -------- diff --git a/numpy/lib/scimath.pyi b/numpy/lib/scimath.pyi new file mode 100644 index 000000000000..d0d4af41eb0c --- /dev/null +++ b/numpy/lib/scimath.pyi @@ -0,0 +1,13 @@ +from typing import List + +__all__: List[str] + +def sqrt(x): ... +def log(x): ... +def log10(x): ... +def logn(n, x): ... +def log2(x): ... +def power(x, p): ... +def arccos(x): ... +def arcsin(x): ... +def arctanh(x): ... diff --git a/numpy/lib/shape_base.py b/numpy/lib/shape_base.py index cbc4641d83d3..a3fbee3d5fb3 100644 --- a/numpy/lib/shape_base.py +++ b/numpy/lib/shape_base.py @@ -69,13 +69,13 @@ def take_along_axis(arr, indices, axis): Parameters ---------- - arr: ndarray (Ni..., M, Nk...) + arr : ndarray (Ni..., M, Nk...) Source array - indices: ndarray (Ni..., J, Nk...) + indices : ndarray (Ni..., J, Nk...) Indices to take along each 1d slice of `arr`. This must match the dimension of arr, but dimensions Ni and Nj only need to broadcast against `arr`. - axis: int + axis : int The axis to take 1d slices along. If axis is None, the input array is treated as if it had first been flattened to 1d, for consistency with `sort` and `argsort`. @@ -190,16 +190,16 @@ def put_along_axis(arr, indices, values, axis): Parameters ---------- - arr: ndarray (Ni..., M, Nk...) + arr : ndarray (Ni..., M, Nk...) Destination array. - indices: ndarray (Ni..., J, Nk...) + indices : ndarray (Ni..., J, Nk...) Indices to change along each 1d slice of `arr`. This must match the dimension of arr, but dimensions in Ni and Nj may be 1 to broadcast against `arr`. - values: array_like (Ni..., J, Nk...) + values : array_like (Ni..., J, Nk...) values to insert at those indices. Its shape and dimension are broadcast to match that of `indices`. - axis: int + axis : int The axis to take 1d slices along. If axis is None, the destination array is treated as if a flattened 1d view had been created of it. @@ -649,7 +649,7 @@ def column_stack(tup): arrays = [] for v in tup: - arr = array(v, copy=False, subok=True) + arr = asanyarray(v) if arr.ndim < 2: arr = array(arr, copy=False, subok=True, ndmin=2).T arrays.append(arr) @@ -775,7 +775,7 @@ def array_split(ary, indices_or_sections, axis=0): # indices_or_sections is a scalar, not an array. Nsections = int(indices_or_sections) if Nsections <= 0: - raise ValueError('number sections must be larger than 0.') + raise ValueError('number sections must be larger than 0.') from None Neach_section, extras = divmod(Ntotal, Nsections) section_sizes = ([0] + extras * [Neach_section+1] + @@ -1088,8 +1088,8 @@ def kron(a, b): ----- The function assumes that the number of dimensions of `a` and `b` are the same, if necessary prepending the smallest with ones. - If `a.shape = (r0,r1,..,rN)` and `b.shape = (s0,s1,...,sN)`, - the Kronecker product has shape `(r0*s0, r1*s1, ..., rN*SN)`. + If ``a.shape = (r0,r1,..,rN)`` and ``b.shape = (s0,s1,...,sN)``, + the Kronecker product has shape ``(r0*s0, r1*s1, ..., rN*SN)``. The elements are products of elements from `a` and `b`, organized explicitly by:: diff --git a/numpy/lib/shape_base.pyi b/numpy/lib/shape_base.pyi new file mode 100644 index 000000000000..09edbcb6cab4 --- /dev/null +++ b/numpy/lib/shape_base.pyi @@ -0,0 +1,24 @@ +from typing import List + +from numpy.core.shape_base import vstack + +__all__: List[str] + +row_stack = vstack + +def take_along_axis(arr, indices, axis): ... +def put_along_axis(arr, indices, values, axis): ... +def apply_along_axis(func1d, axis, arr, *args, **kwargs): ... +def apply_over_axes(func, a, axes): ... +def expand_dims(a, axis): ... +def column_stack(tup): ... +def dstack(tup): ... +def array_split(ary, indices_or_sections, axis=...): ... +def split(ary, indices_or_sections, axis=...): ... +def hsplit(ary, indices_or_sections): ... +def vsplit(ary, indices_or_sections): ... +def dsplit(ary, indices_or_sections): ... +def get_array_prepare(*args): ... +def get_array_wrap(*args): ... +def kron(a, b): ... +def tile(A, reps): ... diff --git a/numpy/lib/stride_tricks.pyi b/numpy/lib/stride_tricks.pyi new file mode 100644 index 000000000000..d2e744b5a7b1 --- /dev/null +++ b/numpy/lib/stride_tricks.pyi @@ -0,0 +1,16 @@ +from typing import Any, List + +from numpy.typing import _ShapeLike, _Shape + +__all__: List[str] + +class DummyArray: + __array_interface__: Any + base: Any + def __init__(self, interface, base=...): ... + +def as_strided(x, shape=..., strides=..., subok=..., writeable=...): ... +def sliding_window_view(x, window_shape, axis=..., *, subok=..., writeable=...): ... +def broadcast_to(array, shape, subok=...): ... +def broadcast_shapes(*args: _ShapeLike) -> _Shape: ... +def broadcast_arrays(*args, subok=...): ... diff --git a/numpy/lib/tests/test__version.py b/numpy/lib/tests/test__version.py index 1825046316a9..e6d41ad93932 100644 --- a/numpy/lib/tests/test__version.py +++ b/numpy/lib/tests/test__version.py @@ -7,7 +7,7 @@ def test_main_versions(): assert_(NumpyVersion('1.8.0') == '1.8.0') - for ver in ['1.9.0', '2.0.0', '1.8.1']: + for ver in ['1.9.0', '2.0.0', '1.8.1', '10.0.1']: assert_(NumpyVersion('1.8.0') < ver) for ver in ['1.7.0', '1.7.1', '0.9.9']: diff --git a/numpy/lib/tests/test_arraysetops.py b/numpy/lib/tests/test_arraysetops.py index 847e6cb8a3e7..d62da9efba0b 100644 --- a/numpy/lib/tests/test_arraysetops.py +++ b/numpy/lib/tests/test_arraysetops.py @@ -358,6 +358,39 @@ def test_in1d_both_arrays_have_structured_dtype(self): result = np.in1d(ar1, ar2) assert_array_equal(result, expected) + def test_in1d_with_arrays_containing_tuples(self): + ar1 = np.array([(1,), 2], dtype=object) + ar2 = np.array([(1,), 2], dtype=object) + expected = np.array([True, True]) + result = np.in1d(ar1, ar2) + assert_array_equal(result, expected) + result = np.in1d(ar1, ar2, invert=True) + assert_array_equal(result, np.invert(expected)) + + # An integer is added at the end of the array to make sure + # that the array builder will create the array with tuples + # and after it's created the integer is removed. + # There's a bug in the array constructor that doesn't handle + # tuples properly and adding the integer fixes that. + ar1 = np.array([(1,), (2, 1), 1], dtype=object) + ar1 = ar1[:-1] + ar2 = np.array([(1,), (2, 1), 1], dtype=object) + ar2 = ar2[:-1] + expected = np.array([True, True]) + result = np.in1d(ar1, ar2) + assert_array_equal(result, expected) + result = np.in1d(ar1, ar2, invert=True) + assert_array_equal(result, np.invert(expected)) + + ar1 = np.array([(1,), (2, 3), 1], dtype=object) + ar1 = ar1[:-1] + ar2 = np.array([(1,), 2], dtype=object) + expected = np.array([True, False]) + result = np.in1d(ar1, ar2) + assert_array_equal(result, expected) + result = np.in1d(ar1, ar2, invert=True) + assert_array_equal(result, np.invert(expected)) + def test_union1d(self): a = np.array([5, 4, 7, 1, 2]) b = np.array([2, 4, 3, 3, 2, 1, 5]) @@ -531,6 +564,52 @@ def check_all(a, b, i1, i2, c, dt): assert_equal(a3_idx.dtype, np.intp) assert_equal(a3_inv.dtype, np.intp) + # test for ticket 2111 - float + a = [2.0, np.nan, 1.0, np.nan] + ua = [1.0, 2.0, np.nan] + ua_idx = [2, 0, 1] + ua_inv = [1, 2, 0, 2] + ua_cnt = [1, 1, 2] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for ticket 2111 - complex + a = [2.0-1j, np.nan, 1.0+1j, complex(0.0, np.nan), complex(1.0, np.nan)] + ua = [1.0+1j, 2.0-1j, complex(0.0, np.nan)] + ua_idx = [2, 0, 3] + ua_inv = [1, 2, 0, 2, 2] + ua_cnt = [1, 1, 3] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for ticket 2111 - datetime64 + nat = np.datetime64('nat') + a = [np.datetime64('2020-12-26'), nat, np.datetime64('2020-12-24'), nat] + ua = [np.datetime64('2020-12-24'), np.datetime64('2020-12-26'), nat] + ua_idx = [2, 0, 1] + ua_inv = [1, 2, 0, 2] + ua_cnt = [1, 1, 2] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + + # test for ticket 2111 - timedelta + nat = np.timedelta64('nat') + a = [np.timedelta64(1, 'D'), nat, np.timedelta64(1, 'h'), nat] + ua = [np.timedelta64(1, 'h'), np.timedelta64(1, 'D'), nat] + ua_idx = [2, 0, 1] + ua_inv = [1, 2, 0, 2] + ua_cnt = [1, 1, 2] + assert_equal(np.unique(a), ua) + assert_equal(np.unique(a, return_index=True), (ua, ua_idx)) + assert_equal(np.unique(a, return_inverse=True), (ua, ua_inv)) + assert_equal(np.unique(a, return_counts=True), (ua, ua_cnt)) + def test_unique_axis_errors(self): assert_raises(TypeError, self._run_axis_tests, object) assert_raises(TypeError, self._run_axis_tests, diff --git a/numpy/lib/tests/test_format.py b/numpy/lib/tests/test_format.py index bac42fad3ec2..10656a23322b 100644 --- a/numpy/lib/tests/test_format.py +++ b/numpy/lib/tests/test_format.py @@ -402,7 +402,7 @@ class BytesIOSRandomSize(BytesIO): def read(self, size=None): import random size = random.randint(1, size) - return super(BytesIOSRandomSize, self).read(size) + return super().read(size) def roundtrip(arr): diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py index 4c7c0480c106..a4f49a78b09d 100644 --- a/numpy/lib/tests/test_function_base.py +++ b/numpy/lib/tests/test_function_base.py @@ -1798,6 +1798,24 @@ def test_simple(self): assert_array_equal(unwrap([1, 1 + 2 * np.pi]), [1, 1]) # check that unwrap maintains continuity assert_(np.all(diff(unwrap(rand(10) * 100)) < np.pi)) + + def test_period(self): + # check that unwrap removes jumps greater that 255 + assert_array_equal(unwrap([1, 1 + 256], period=255), [1, 2]) + # check that unwrap maintains continuity + assert_(np.all(diff(unwrap(rand(10) * 1000, period=255)) < 255)) + # check simple case + simple_seq = np.array([0, 75, 150, 225, 300]) + wrap_seq = np.mod(simple_seq, 255) + assert_array_equal(unwrap(wrap_seq, period=255), simple_seq) + # check custom discont value + uneven_seq = np.array([0, 75, 150, 225, 300, 430]) + wrap_uneven = np.mod(uneven_seq, 250) + no_discont = unwrap(wrap_uneven, period=250) + assert_array_equal(no_discont, [0, 75, 150, 225, 300, 180]) + sm_discont = unwrap(wrap_uneven, period=250, discont=140) + assert_array_equal(sm_discont, [0, 75, 150, 225, 300, 430]) + assert sm_discont.dtype == wrap_uneven.dtype class TestFilterwindows: @@ -2307,6 +2325,27 @@ def test_writeback(self): assert_equal(x[0, :], 0) assert_equal(x[1, :], X) + def test_nd_shape(self): + a, b, c, d, e = np.meshgrid(*([0] * i for i in range(1, 6))) + expected_shape = (2, 1, 3, 4, 5) + assert_equal(a.shape, expected_shape) + assert_equal(b.shape, expected_shape) + assert_equal(c.shape, expected_shape) + assert_equal(d.shape, expected_shape) + assert_equal(e.shape, expected_shape) + + def test_nd_values(self): + a, b, c = np.meshgrid([0], [1, 2], [3, 4, 5]) + assert_equal(a, [[[0, 0, 0]], [[0, 0, 0]]]) + assert_equal(b, [[[1, 1, 1]], [[2, 2, 2]]]) + assert_equal(c, [[[3, 4, 5]], [[3, 4, 5]]]) + + def test_nd_indexing(self): + a, b, c = np.meshgrid([0], [1, 2], [3, 4, 5], indexing='ij') + assert_equal(a, [[[0, 0, 0], [0, 0, 0]]]) + assert_equal(b, [[[1, 1, 1], [2, 2, 2]]]) + assert_equal(c, [[[3, 4, 5], [3, 4, 5]]]) + class TestPiecewise: @@ -2399,6 +2438,14 @@ def test_multidimensional_extrafunc(self): assert_array_equal(y, np.array([[-1., -1., -1.], [3., 3., 1.]])) + def test_subclasses(self): + class subclass(np.ndarray): + pass + x = np.arange(5.).view(subclass) + r = piecewise(x, [x<2., x>=4], [-1., 1., 0.]) + assert_equal(type(r), subclass) + assert_equal(r, [-1., -1., 0., 0., 1.]) + class TestBincount: @@ -2721,6 +2768,10 @@ def test_fraction(self): assert_equal(p, Fraction(7, 4)) assert_equal(type(p), Fraction) + p = np.percentile(x, [Fraction(50)]) + assert_equal(p, np.array([Fraction(7, 4)])) + assert_equal(type(p), np.ndarray) + def test_api(self): d = np.ones(5) np.percentile(d, 5, None, None, False) @@ -3115,6 +3166,16 @@ def test_nan_behavior(self): assert_equal(np.percentile( a, [0.3, 0.6], (0, 2), interpolation='nearest'), b) + def test_nan_q(self): + # GH18830 + with pytest.raises(ValueError, match="Percentiles must be in"): + np.percentile([1, 2, 3, 4.0], np.nan) + with pytest.raises(ValueError, match="Percentiles must be in"): + np.percentile([1, 2, 3, 4.0], [np.nan]) + q = np.linspace(1.0, 99.0, 16) + q[0] = np.nan + with pytest.raises(ValueError, match="Percentiles must be in"): + np.percentile([1, 2, 3, 4.0], q) class TestQuantile: # most of this is already tested by TestPercentile @@ -3151,6 +3212,14 @@ def test_fraction(self): assert_equal(q, Fraction(7, 4)) assert_equal(type(q), Fraction) + q = np.quantile(x, [Fraction(1, 2)]) + assert_equal(q, np.array([Fraction(7, 4)])) + assert_equal(type(q), np.ndarray) + + q = np.quantile(x, [[Fraction(1, 2)]]) + assert_equal(q, np.array([[Fraction(7, 4)]])) + assert_equal(type(q), np.ndarray) + # repeat with integral input but fractional quantile x = np.arange(8) assert_equal(np.quantile(x, Fraction(1, 2)), Fraction(7, 2)) diff --git a/numpy/lib/tests/test_index_tricks.py b/numpy/lib/tests/test_index_tricks.py index 843e27cefc91..c21aefd1a66b 100644 --- a/numpy/lib/tests/test_index_tricks.py +++ b/numpy/lib/tests/test_index_tricks.py @@ -16,23 +16,13 @@ class TestRavelUnravelIndex: def test_basic(self): assert_equal(np.unravel_index(2, (2, 2)), (1, 0)) - # test backwards compatibility with older dims - # keyword argument; see Issue #10586 - with assert_warns(DeprecationWarning): - # we should achieve the correct result - # AND raise the appropriate warning - # when using older "dims" kw argument - assert_equal(np.unravel_index(indices=2, - dims=(2, 2)), - (1, 0)) - # test that new shape argument works properly assert_equal(np.unravel_index(indices=2, shape=(2, 2)), (1, 0)) # test that an invalid second keyword argument - # is properly handled + # is properly handled, including the old name `dims`. with assert_raises(TypeError): np.unravel_index(indices=2, hape=(2, 2)) @@ -42,6 +32,9 @@ def test_basic(self): with assert_raises(TypeError): np.unravel_index(254, ims=(17, 94)) + with assert_raises(TypeError): + np.unravel_index(254, dims=(17, 94)) + assert_equal(np.ravel_multi_index((1, 0), (2, 2)), 2) assert_equal(np.unravel_index(254, (17, 94)), (2, 66)) assert_equal(np.ravel_multi_index((2, 66), (17, 94)), 254) diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py index aa449976435a..534ab683c584 100644 --- a/numpy/lib/tests/test_io.py +++ b/numpy/lib/tests/test_io.py @@ -580,7 +580,7 @@ def check_large_zip(memoryerror_raised): memoryerror_raised.value = False try: # The test takes at least 6GB of memory, writes a file larger - # than 4GB + # than 4GB. This tests the ``allowZip64`` kwarg to ``zipfile`` test_data = np.asarray([np.random.rand( np.random.randint(50,100),4) for i in range(800000)], dtype=object) @@ -599,6 +599,9 @@ def check_large_zip(memoryerror_raised): p.join() if memoryerror_raised.value: raise MemoryError("Child process raised a MemoryError exception") + # -9 indicates a SIGKILL, probably an OOM. + if p.exitcode == -9: + pytest.xfail("subprocess got a SIGKILL, apparently free memory was not sufficient") assert p.exitcode == 0 class LoadTxtBase: diff --git a/numpy/lib/tests/test_nanfunctions.py b/numpy/lib/tests/test_nanfunctions.py index e0f723a3c22f..1f1f5601b19c 100644 --- a/numpy/lib/tests/test_nanfunctions.py +++ b/numpy/lib/tests/test_nanfunctions.py @@ -588,6 +588,15 @@ def test_empty(self): assert_(len(w) == 0) +_TIME_UNITS = ( + "Y", "M", "W", "D", "h", "m", "s", "ms", "us", "ns", "ps", "fs", "as" +) + +# All `inexact` + `timdelta64` type codes +_TYPE_CODES = list(np.typecodes["AllFloat"]) +_TYPE_CODES += [f"m8[{unit}]" for unit in _TIME_UNITS] + + class TestNanFunctions_Median: def test_mutation(self): @@ -662,23 +671,32 @@ def test_result_values(self): res = np.nanmedian(_ndat, axis=1) assert_almost_equal(res, tgt) - def test_allnans(self): - mat = np.array([np.nan]*9).reshape(3, 3) - for axis in [None, 0, 1]: - with suppress_warnings() as sup: - sup.record(RuntimeWarning) + @pytest.mark.parametrize("axis", [None, 0, 1]) + @pytest.mark.parametrize("dtype", _TYPE_CODES) + def test_allnans(self, dtype, axis): + mat = np.full((3, 3), np.nan).astype(dtype) + with suppress_warnings() as sup: + sup.record(RuntimeWarning) - assert_(np.isnan(np.nanmedian(mat, axis=axis)).all()) - if axis is None: - assert_(len(sup.log) == 1) - else: - assert_(len(sup.log) == 3) - # Check scalar - assert_(np.isnan(np.nanmedian(np.nan))) - if axis is None: - assert_(len(sup.log) == 2) - else: - assert_(len(sup.log) == 4) + output = np.nanmedian(mat, axis=axis) + assert output.dtype == mat.dtype + assert np.isnan(output).all() + + if axis is None: + assert_(len(sup.log) == 1) + else: + assert_(len(sup.log) == 3) + + # Check scalar + scalar = np.array(np.nan).astype(dtype)[()] + output_scalar = np.nanmedian(scalar) + assert output_scalar.dtype == scalar.dtype + assert np.isnan(output_scalar) + + if axis is None: + assert_(len(sup.log) == 2) + else: + assert_(len(sup.log) == 4) def test_empty(self): mat = np.zeros((0, 3)) diff --git a/numpy/lib/tests/test_regression.py b/numpy/lib/tests/test_regression.py index 55df2a6752c2..373226277acd 100644 --- a/numpy/lib/tests/test_regression.py +++ b/numpy/lib/tests/test_regression.py @@ -1,3 +1,5 @@ +import pytest + import os import numpy as np diff --git a/numpy/lib/tests/test_utils.py b/numpy/lib/tests/test_utils.py index 33951b92ade9..8a877ae69854 100644 --- a/numpy/lib/tests/test_utils.py +++ b/numpy/lib/tests/test_utils.py @@ -4,7 +4,7 @@ from numpy.core import arange from numpy.testing import assert_, assert_equal, assert_raises_regex -from numpy.lib import deprecate +from numpy.lib import deprecate, deprecate_with_doc import numpy.lib.utils as utils from io import StringIO @@ -60,6 +60,11 @@ def old_func6(self, x): new_func6 = deprecate(old_func6) +@deprecate_with_doc(msg="Rather use new_func7") +def old_func7(self,x): + return x + + def test_deprecate_decorator(): assert_('deprecated' in old_func.__doc__) @@ -73,6 +78,10 @@ def test_deprecate_fn(): assert_('new_func3' in new_func3.__doc__) +def test_deprecate_with_doc_decorator_message(): + assert_('Rather use new_func7' in old_func7.__doc__) + + @pytest.mark.skipif(sys.flags.optimize == 2, reason="-OO discards docstrings") @pytest.mark.parametrize('old_func, new_func', [ (old_func4, new_func4), diff --git a/numpy/lib/twodim_base.py b/numpy/lib/twodim_base.py index 2b4cbdfbbc11..fd8c73573290 100644 --- a/numpy/lib/twodim_base.py +++ b/numpy/lib/twodim_base.py @@ -5,12 +5,13 @@ from numpy.core.numeric import ( asanyarray, arange, zeros, greater_equal, multiply, ones, - asarray, where, int8, int16, int32, int64, empty, promote_types, diagonal, - nonzero + asarray, where, int8, int16, int32, int64, intp, empty, promote_types, + diagonal, nonzero, indices ) from numpy.core.overrides import set_array_function_like_doc, set_module from numpy.core import overrides from numpy.core import iinfo +from numpy.lib.stride_tricks import broadcast_to __all__ = [ @@ -46,10 +47,11 @@ def _flip_dispatcher(m): @array_function_dispatch(_flip_dispatcher) def fliplr(m): """ - Flip array in the left/right direction. + Reverse the order of elements along axis 1 (left/right). - Flip the entries in each row in the left/right direction. - Columns are preserved, but appear in a different order than before. + For a 2-D array, this flips the entries in each row in the left/right + direction. Columns are preserved, but appear in a different order than + before. Parameters ---------- @@ -65,11 +67,13 @@ def fliplr(m): See Also -------- flipud : Flip array in the up/down direction. + flip : Flip array in one or more dimesions. rot90 : Rotate array counterclockwise. Notes ----- - Equivalent to m[:,::-1]. Requires the array to be at least 2-D. + Equivalent to ``m[:,::-1]`` or ``np.flip(m, axis=1)``. + Requires the array to be at least 2-D. Examples -------- @@ -97,10 +101,10 @@ def fliplr(m): @array_function_dispatch(_flip_dispatcher) def flipud(m): """ - Flip array in the up/down direction. + Reverse the order of elements along axis 0 (up/down). - Flip the entries in each column in the up/down direction. - Rows are preserved, but appear in a different order than before. + For a 2-D array, this flips the entries in each column in the up/down + direction. Rows are preserved, but appear in a different order than before. Parameters ---------- @@ -116,12 +120,13 @@ def flipud(m): See Also -------- fliplr : Flip array in the left/right direction. + flip : Flip array in one or more dimesions. rot90 : Rotate array counterclockwise. Notes ----- - Equivalent to ``m[::-1,...]``. - Does not require the array to be two-dimensional. + Equivalent to ``m[::-1, ...]`` or ``np.flip(m, axis=0)``. + Requires the array to be at least 1-D. Examples -------- @@ -347,10 +352,10 @@ def diagflat(v, k=0): n = s + abs(k) res = zeros((n, n), v.dtype) if (k >= 0): - i = arange(0, n-k) + i = arange(0, n-k, dtype=intp) fi = i+k+i*n else: - i = arange(0, n+k) + i = arange(0, n+k, dtype=intp) fi = i+(i-k)*n res.flat[fi] = v if not wrap: @@ -700,7 +705,9 @@ def histogram2d(x, y, bins=10, range=None, normed=None, weights=None, >>> x = np.random.normal(2, 1, 100) >>> y = np.random.normal(1, 1, 100) >>> H, xedges, yedges = np.histogram2d(x, y, bins=(xedges, yedges)) - >>> H = H.T # Let each row list bins with common y range. + >>> # Histogram does not follow Cartesian convention (see Notes), + >>> # therefore transpose H for visualization purposes. + >>> H = H.T :func:`imshow ` can only display square bins: @@ -894,7 +901,10 @@ def tril_indices(n, k=0, m=None): [-10, -10, -10, -10]]) """ - return nonzero(tri(n, m, k=k, dtype=bool)) + tri_ = tri(n, m, k=k, dtype=bool) + + return tuple(broadcast_to(inds, tri_.shape)[tri_] + for inds in indices(tri_.shape, sparse=True)) def _trilu_indices_form_dispatcher(arr, k=None): @@ -1010,7 +1020,10 @@ def triu_indices(n, k=0, m=None): [ 12, 13, 14, -1]]) """ - return nonzero(~tri(n, m, k=k-1, dtype=bool)) + tri_ = ~tri(n, m, k=k - 1, dtype=bool) + + return tuple(broadcast_to(inds, tri_.shape)[tri_] + for inds in indices(tri_.shape, sparse=True)) @array_function_dispatch(_trilu_indices_form_dispatcher) diff --git a/numpy/lib/twodim_base.pyi b/numpy/lib/twodim_base.pyi new file mode 100644 index 000000000000..79b9511b88ab --- /dev/null +++ b/numpy/lib/twodim_base.pyi @@ -0,0 +1,32 @@ +from typing import List, Optional, Any + +from numpy import ndarray, _OrderCF +from numpy.typing import ArrayLike, DTypeLike + +__all__: List[str] + +def fliplr(m): ... +def flipud(m): ... + +def eye( + N: int, + M: Optional[int] = ..., + k: int = ..., + dtype: DTypeLike = ..., + order: _OrderCF = ..., + *, + like: Optional[ArrayLike] = ... +) -> ndarray[Any, Any]: ... + +def diag(v, k=...): ... +def diagflat(v, k=...): ... +def tri(N, M=..., k=..., dtype = ..., *, like=...): ... +def tril(m, k=...): ... +def triu(m, k=...): ... +def vander(x, N=..., increasing=...): ... +def histogram2d(x, y, bins=..., range=..., normed=..., weights=..., density=...): ... +def mask_indices(n, mask_func, k=...): ... +def tril_indices(n, k=..., m=...): ... +def tril_indices_from(arr, k=...): ... +def triu_indices(n, k=..., m=...): ... +def triu_indices_from(arr, k=...): ... diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index 2a2982ab348e..56afd83ce335 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -262,6 +262,10 @@ def isreal(x): out : ndarray, bool Boolean array of same shape as `x`. + Notes + ----- + `isreal` may behave unexpectedly for string or object arrays (see examples) + See Also -------- iscomplex @@ -269,8 +273,28 @@ def isreal(x): Examples -------- - >>> np.isreal([1+1j, 1+0j, 4.5, 3, 2, 2j]) + >>> a = np.array([1+1j, 1+0j, 4.5, 3, 2, 2j], dtype=complex) + >>> np.isreal(a) array([False, True, True, True, True, False]) + + The function does not work on string arrays. + + >>> a = np.array([2j, "a"], dtype="U") + >>> np.isreal(a) # Warns about non-elementwise comparison + False + + Returns True for all elements in input array of ``dtype=object`` even if + any of the elements is complex. + + >>> a = np.array([1, "2", 3+4j], dtype=object) + >>> np.isreal(a) + array([ True, True, True]) + + isreal should not be used with object arrays + + >>> a = np.array([1+2j, 2+1j], dtype=object) + >>> np.isreal(a) + array([ True, True]) """ return imag(x) == 0 @@ -340,6 +364,19 @@ def isrealobj(x): -------- iscomplexobj, isreal + Notes + ----- + The function is only meant for arrays with numerical values but it + accepts all other objects. Since it assumes array input, the return + value of other objects may be True. + + >>> np.isrealobj('A string') + True + >>> np.isrealobj(False) + True + >>> np.isrealobj(None) + True + Examples -------- >>> np.isrealobj(1) diff --git a/numpy/lib/type_check.pyi b/numpy/lib/type_check.pyi new file mode 100644 index 000000000000..7da02bb9f196 --- /dev/null +++ b/numpy/lib/type_check.pyi @@ -0,0 +1,19 @@ +from typing import List + +__all__: List[str] + +def mintypecode(typechars, typeset=..., default=...): ... +def asfarray(a, dtype = ...): ... +def real(val): ... +def imag(val): ... +def iscomplex(x): ... +def isreal(x): ... +def iscomplexobj(x): ... +def isrealobj(x): ... +def nan_to_num(x, copy=..., nan=..., posinf=..., neginf=...): ... +def real_if_close(a, tol=...): ... +def typename(char): ... +def common_type(*arrays): ... + +# NOTE: Deprecated +# def asscalar(a): ... diff --git a/numpy/lib/ufunclike.py b/numpy/lib/ufunclike.py index 1f26a18459f0..a93c4773bc4e 100644 --- a/numpy/lib/ufunclike.py +++ b/numpy/lib/ufunclike.py @@ -100,7 +100,7 @@ def fix(x, out=None): See Also -------- - trunc, floor, ceil + rint, trunc, floor, ceil around : Round to given number of decimals Examples @@ -189,7 +189,8 @@ def isposinf(x, out=None): try: signbit = ~nx.signbit(x) except TypeError as e: - raise TypeError('This operation is not supported for complex values ' + dtype = nx.asanyarray(x).dtype + raise TypeError(f'This operation is not supported for {dtype} values ' 'because it would be ambiguous.') from e else: return nx.logical_and(is_inf, signbit, out) @@ -260,7 +261,8 @@ def isneginf(x, out=None): try: signbit = nx.signbit(x) except TypeError as e: - raise TypeError('This operation is not supported for complex values ' + dtype = nx.asanyarray(x).dtype + raise TypeError(f'This operation is not supported for {dtype} values ' 'because it would be ambiguous.') from e else: return nx.logical_and(is_inf, signbit, out) diff --git a/numpy/lib/ufunclike.pyi b/numpy/lib/ufunclike.pyi new file mode 100644 index 000000000000..03f08ebffea3 --- /dev/null +++ b/numpy/lib/ufunclike.pyi @@ -0,0 +1,66 @@ +from typing import Any, overload, TypeVar, List, Union + +from numpy import floating, bool_, object_, ndarray +from numpy.typing import ( + NDArray, + _FloatLike_co, + _ArrayLikeFloat_co, + _ArrayLikeObject_co, +) + +_ArrayType = TypeVar("_ArrayType", bound=ndarray[Any, Any]) + +__all__: List[str] + +@overload +def fix( # type: ignore[misc] + x: _FloatLike_co, + out: None = ..., +) -> floating[Any]: ... +@overload +def fix( + x: _ArrayLikeFloat_co, + out: None = ..., +) -> NDArray[floating[Any]]: ... +@overload +def fix( + x: _ArrayLikeObject_co, + out: None = ..., +) -> NDArray[object_]: ... +@overload +def fix( + x: Union[_ArrayLikeFloat_co, _ArrayLikeObject_co], + out: _ArrayType, +) -> _ArrayType: ... + +@overload +def isposinf( # type: ignore[misc] + x: _FloatLike_co, + out: None = ..., +) -> bool_: ... +@overload +def isposinf( + x: _ArrayLikeFloat_co, + out: None = ..., +) -> NDArray[bool_]: ... +@overload +def isposinf( + x: _ArrayLikeFloat_co, + out: _ArrayType, +) -> _ArrayType: ... + +@overload +def isneginf( # type: ignore[misc] + x: _FloatLike_co, + out: None = ..., +) -> bool_: ... +@overload +def isneginf( + x: _ArrayLikeFloat_co, + out: None = ..., +) -> NDArray[bool_]: ... +@overload +def isneginf( + x: _ArrayLikeFloat_co, + out: _ArrayType, +) -> _ArrayType: ... diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 5447608bf5bb..12a7cacdce87 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -193,7 +193,32 @@ def deprecate(*args, **kwargs): else: return _Deprecate(*args, **kwargs) -deprecate_with_doc = lambda msg: _Deprecate(message=msg) + +def deprecate_with_doc(msg): + """ + Deprecates a function and includes the deprecation in its docstring. + + This function is used as a decorator. It returns an object that can be + used to issue a DeprecationWarning, by passing the to-be decorated + function as argument, this adds warning to the to-be decorated function's + docstring and returns the new function object. + + See Also + -------- + deprecate : Decorate a function such that it issues a `DeprecationWarning` + + Parameters + ---------- + msg : str + Additional explanation of the deprecation. Displayed in the + docstring after the warning. + + Returns + ------- + obj : object + + """ + return _Deprecate(message=msg) #-------------------------------------------- @@ -990,10 +1015,11 @@ def _median_nancheck(data, result, axis, out): Input data to median function result : Array or MaskedArray Result of median function - axis : {int, sequence of int, None}, optional - Axis or axes along which the median was computed. + axis : int + Axis along which the median was computed. out : ndarray, optional Output array in which to place the result. + Returns ------- median : scalar or ndarray @@ -1001,8 +1027,7 @@ def _median_nancheck(data, result, axis, out): """ if data.size == 0: return result - data = np.moveaxis(data, axis, -1) - n = np.isnan(data[..., -1]) + n = np.isnan(data.take(-1, axis=axis)) # masked NaN values are ok if np.ma.isMaskedArray(n): n = n.filled(False) @@ -1017,4 +1042,30 @@ def _median_nancheck(data, result, axis, out): result[n] = np.nan return result +def _opt_info(): + """ + Returns a string contains the supported CPU features by the current build. + + The string format can be explained as follows: + - dispatched features that are supported by the running machine + end with `*`. + - dispatched features that are "not" supported by the running machine + end with `?`. + - remained features are representing the baseline. + """ + from numpy.core._multiarray_umath import ( + __cpu_features__, __cpu_baseline__, __cpu_dispatch__ + ) + + if len(__cpu_baseline__) == 0 and len(__cpu_dispatch__) == 0: + return '' + + enabled_features = ' '.join(__cpu_baseline__) + for feature in __cpu_dispatch__: + if __cpu_features__[feature]: + enabled_features += f" {feature}*" + else: + enabled_features += f" {feature}?" + + return enabled_features #----------------------------------------------------------------------------- diff --git a/numpy/lib/utils.pyi b/numpy/lib/utils.pyi new file mode 100644 index 000000000000..0518655c6ce5 --- /dev/null +++ b/numpy/lib/utils.pyi @@ -0,0 +1,99 @@ +import sys +from ast import AST +from typing import ( + Any, + Callable, + List, + Mapping, + Optional, + overload, + Sequence, + Tuple, + TypeVar, + Union, +) + +from numpy import ndarray, generic + +from numpy.core.numerictypes import ( + issubclass_ as issubclass_, + issubdtype as issubdtype, + issubsctype as issubsctype, +) + +if sys.version_info >= (3, 8): + from typing import Protocol +else: + from typing_extensions import Protocol + +_T_contra = TypeVar("_T_contra", contravariant=True) +_FuncType = TypeVar("_FuncType", bound=Callable[..., Any]) + +# A file-like object opened in `w` mode +class _SupportsWrite(Protocol[_T_contra]): + def write(self, __s: _T_contra) -> Any: ... + +__all__: List[str] + +class _Deprecate: + old_name: Optional[str] + new_name: Optional[str] + message: Optional[str] + def __init__( + self, + old_name: Optional[str] = ..., + new_name: Optional[str] = ..., + message: Optional[str] = ..., + ) -> None: ... + # NOTE: `__call__` can in principle take arbitrary `*args` and `**kwargs`, + # even though they aren't used for anything + def __call__(self, func: _FuncType) -> _FuncType: ... + +def get_include() -> str: ... + +@overload +def deprecate( + *, + old_name: Optional[str] = ..., + new_name: Optional[str] = ..., + message: Optional[str] = ..., +) -> _Deprecate: ... +@overload +def deprecate( + __func: _FuncType, + old_name: Optional[str] = ..., + new_name: Optional[str] = ..., + message: Optional[str] = ..., +) -> _FuncType: ... + +def deprecate_with_doc(msg: Optional[str]) -> _Deprecate: ... + +# NOTE: In practice `byte_bounds` can (potentially) take any object +# implementing the `__array_interface__` protocol. The caveat is +# that certain keys, marked as optional in the spec, must be present for +# `byte_bounds`. This concerns `"strides"` and `"data"`. +def byte_bounds(a: Union[generic, ndarray[Any, Any]]) -> Tuple[int, int]: ... + +def who(vardict: Optional[Mapping[str, ndarray[Any, Any]]] = ...) -> None: ... + +def info( + object: object = ..., + maxwidth: int = ..., + output: Optional[_SupportsWrite[str]] = ..., + toplevel: str = ..., +) -> None: ... + +def source( + object: object, + output: Optional[_SupportsWrite[str]] = ..., +) -> None: ... + +def lookfor( + what: str, + module: Union[None, str, Sequence[str]] = ..., + import_modules: bool = ..., + regenerate: bool = ..., + output: Optional[_SupportsWrite[str]] =..., +) -> None: ... + +def safe_eval(source: Union[str, AST]) -> Any: ... diff --git a/numpy/linalg/__init__.py b/numpy/linalg/__init__.py index 3a53ac6ecac6..93943de3896c 100644 --- a/numpy/linalg/__init__.py +++ b/numpy/linalg/__init__.py @@ -70,8 +70,11 @@ """ # To get sub-modules +from . import linalg from .linalg import * +__all__ = linalg.__all__.copy() + from numpy._pytesttester import PytestTester test = PytestTester(__name__) del PytestTester diff --git a/numpy/linalg/__init__.pyi b/numpy/linalg/__init__.pyi index ffb05bb8130a..5080019f4de4 100644 --- a/numpy/linalg/__init__.pyi +++ b/numpy/linalg/__init__.pyi @@ -1,23 +1,26 @@ -from typing import Any +from typing import Any, List -matrix_power: Any -solve: Any -tensorsolve: Any -tensorinv: Any -inv: Any -cholesky: Any -eigvals: Any -eigvalsh: Any -pinv: Any -slogdet: Any -det: Any -svd: Any -eig: Any -eigh: Any -lstsq: Any -norm: Any -qr: Any -cond: Any -matrix_rank: Any -LinAlgError: Any -multi_dot: Any +__all__: List[str] + +class LinAlgError(Exception): ... + +def tensorsolve(a, b, axes=...): ... +def solve(a, b): ... +def tensorinv(a, ind=...): ... +def inv(a): ... +def matrix_power(a, n): ... +def cholesky(a): ... +def qr(a, mode=...): ... +def eigvals(a): ... +def eigvalsh(a, UPLO=...): ... +def eig(a): ... +def eigh(a, UPLO=...): ... +def svd(a, full_matrices=..., compute_uv=..., hermitian=...): ... +def cond(x, p=...): ... +def matrix_rank(M, tol=..., hermitian=...): ... +def pinv(a, rcond=..., hermitian=...): ... +def slogdet(a): ... +def det(a): ... +def lstsq(a, b, rcond=...): ... +def norm(x, ord=..., axis=..., keepdims=...): ... +def multi_dot(arrays, *, out=...): ... diff --git a/numpy/linalg/lapack_lite/clapack_scrub.py b/numpy/linalg/lapack_lite/clapack_scrub.py index f3e7d25d2eab..738fad7fe1ee 100644 --- a/numpy/linalg/lapack_lite/clapack_scrub.py +++ b/numpy/linalg/lapack_lite/clapack_scrub.py @@ -224,7 +224,7 @@ def OutOfHeader(line): def removeSubroutinePrototypes(source): expression = re.compile( - r'/[*] Subroutine [*]/^\s*(?:(?:inline|static)\s+){0,2}(?!else|typedef|return)\w+\s+\*?\s*(\w+)\s*\([^0]+\)\s*;?' + r'/\* Subroutine \*/^\s*(?:(?:inline|static)\s+){0,2}(?!else|typedef|return)\w+\s+\*?\s*(\w+)\s*\([^0]+\)\s*;?' ) lines = LineQueue() for line in StringIO(source): diff --git a/numpy/linalg/lapack_lite/make_lite.py b/numpy/linalg/lapack_lite/make_lite.py index cf15b2541361..b145f6c4f8b5 100755 --- a/numpy/linalg/lapack_lite/make_lite.py +++ b/numpy/linalg/lapack_lite/make_lite.py @@ -261,8 +261,8 @@ def runF2C(fortran_filename, output_dir): subprocess.check_call( ["f2c"] + F2C_ARGS + ['-d', output_dir, fortran_filename] ) - except subprocess.CalledProcessError: - raise F2CError + except subprocess.CalledProcessError as e: + raise F2CError from e def scrubF2CSource(c_file): with open(c_file) as fo: diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 7775e4c32fcd..46fb2502e5cc 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -2171,13 +2171,14 @@ def lstsq(a, b, rcond="warn"): r""" Return the least-squares solution to a linear matrix equation. - Computes the vector x that approximatively solves the equation + Computes the vector `x` that approximatively solves the equation ``a @ x = b``. The equation may be under-, well-, or over-determined (i.e., the number of linearly independent rows of `a` can be less than, equal to, or greater than its number of linearly independent columns). If `a` is square and of full rank, then `x` (but for round-off error) is the "exact" solution of the equation. Else, `x` minimizes the - Euclidean 2-norm :math:`|| b - a x ||`. + Euclidean 2-norm :math:`||b - ax||`. If there are multiple minimizing + solutions, the one with the smallest 2-norm :math:`||x||` is returned. Parameters ---------- diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index 5c9f2a4cb56c..e2944f38c33d 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -9,11 +9,6 @@ def configuration(parent_package='', top_path=None): config.add_subpackage('tests') - # Accelerate is buggy, disallow it. See also numpy/core/setup.py - for opt_order in (blas_opt_info.blas_order, lapack_opt_info.lapack_order): - if 'accelerate' in opt_order: - opt_order.remove('accelerate') - # Configure lapack_lite src_dir = 'lapack_lite' diff --git a/numpy/linalg/tests/test_build.py b/numpy/linalg/tests/test_build.py index 4859226d99a4..868341ff298c 100644 --- a/numpy/linalg/tests/test_build.py +++ b/numpy/linalg/tests/test_build.py @@ -15,8 +15,8 @@ def __init__(self): try: p = Popen(self.cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() - except OSError: - raise RuntimeError(f'command {self.cmd} cannot be run') + except OSError as e: + raise RuntimeError(f'command {self.cmd} cannot be run') from e def get_dependencies(self, lfile): p = Popen(self.cmd + [lfile], stdout=PIPE, stderr=PIPE) diff --git a/numpy/linalg/tests/test_linalg.py b/numpy/linalg/tests/test_linalg.py index 21fab58e1d42..c6e8cdd039f1 100644 --- a/numpy/linalg/tests/test_linalg.py +++ b/numpy/linalg/tests/test_linalg.py @@ -348,10 +348,10 @@ def check_cases(self, require=set(), exclude=set()): try: case.check(self.do) - except Exception: + except Exception as e: msg = f'In test case: {case!r}\n\n' msg += traceback.format_exc() - raise AssertionError(msg) + raise AssertionError(msg) from e class LinalgSquareTestCase(LinalgTestCase): @@ -684,7 +684,7 @@ def hermitian(mat): axes = list(range(mat.ndim)) axes[-1], axes[-2] = axes[-2], axes[-1] return np.conj(np.transpose(mat, axes=axes)) - + assert_almost_equal(np.matmul(u, hermitian(u)), np.broadcast_to(np.eye(u.shape[-1]), u.shape)) assert_almost_equal(np.matmul(vt, hermitian(vt)), np.broadcast_to(np.eye(vt.shape[-1]), vt.shape)) assert_equal(np.sort(s)[..., ::-1], s) @@ -766,6 +766,9 @@ def test_singular(self): for A, p in itertools.product(As, p_neg): linalg.cond(A, p) + @pytest.mark.xfail(True, run=False, + reason="Platform/LAPACK-dependent failure, " + "see gh-18914") def test_nan(self): # nans should be passed through, not converted to infs ps = [None, 1, -1, 2, -2, 'fro'] @@ -981,7 +984,7 @@ def test_incompatible_dims(self): linalg.lstsq(A, y, rcond=None) -@pytest.mark.parametrize('dt', [np.dtype(c) for c in '?bBhHiIqQefdgFDGO']) +@pytest.mark.parametrize('dt', [np.dtype(c) for c in '?bBhHiIqQefdgFDGO']) class TestMatrixPower: rshft_0 = np.eye(4) @@ -1010,7 +1013,7 @@ def tz(M): mz = matrix_power(M, 0) assert_equal(mz, identity_like_generalized(M)) assert_equal(mz.dtype, M.dtype) - + for mat in self.rshft_all: tz(mat.astype(dt)) if dt != object: diff --git a/numpy/ma/__init__.pyi b/numpy/ma/__init__.pyi index d1259abcce73..a9a833e520f2 100644 --- a/numpy/ma/__init__.pyi +++ b/numpy/ma/__init__.pyi @@ -1,225 +1,232 @@ -from typing import Any +from typing import Any, List -core: Any -extras: Any -MAError: Any -MaskError: Any -MaskType: Any -MaskedArray: Any -abs: Any -absolute: Any -add: Any -all: Any -allclose: Any -allequal: Any -alltrue: Any -amax: Any -amin: Any -angle: Any -anom: Any -anomalies: Any -any: Any -append: Any -arange: Any -arccos: Any -arccosh: Any -arcsin: Any -arcsinh: Any -arctan: Any -arctan2: Any -arctanh: Any -argmax: Any -argmin: Any -argsort: Any -around: Any -array: Any -asanyarray: Any -asarray: Any -bitwise_and: Any -bitwise_or: Any -bitwise_xor: Any -bool_: Any -ceil: Any -choose: Any -clip: Any -common_fill_value: Any -compress: Any -compressed: Any -concatenate: Any -conjugate: Any -convolve: Any -copy: Any -correlate: Any -cos: Any -cosh: Any -count: Any -cumprod: Any -cumsum: Any -default_fill_value: Any -diag: Any -diagonal: Any -diff: Any -divide: Any -empty: Any -empty_like: Any -equal: Any -exp: Any -expand_dims: Any -fabs: Any -filled: Any -fix_invalid: Any -flatten_mask: Any -flatten_structured_array: Any -floor: Any -floor_divide: Any -fmod: Any -frombuffer: Any -fromflex: Any -fromfunction: Any -getdata: Any -getmask: Any -getmaskarray: Any -greater: Any -greater_equal: Any -harden_mask: Any -hypot: Any -identity: Any -ids: Any -indices: Any -inner: Any -innerproduct: Any -isMA: Any -isMaskedArray: Any -is_mask: Any -is_masked: Any -isarray: Any -left_shift: Any -less: Any -less_equal: Any -log: Any -log10: Any -log2: Any -logical_and: Any -logical_not: Any -logical_or: Any -logical_xor: Any -make_mask: Any -make_mask_descr: Any -make_mask_none: Any -mask_or: Any -masked: Any -masked_array: Any -masked_equal: Any -masked_greater: Any -masked_greater_equal: Any -masked_inside: Any -masked_invalid: Any -masked_less: Any -masked_less_equal: Any -masked_not_equal: Any -masked_object: Any -masked_outside: Any -masked_print_option: Any -masked_singleton: Any -masked_values: Any -masked_where: Any -max: Any -maximum: Any -maximum_fill_value: Any -mean: Any -min: Any -minimum: Any -minimum_fill_value: Any -mod: Any -multiply: Any -mvoid: Any -ndim: Any -negative: Any -nomask: Any -nonzero: Any -not_equal: Any -ones: Any -outer: Any -outerproduct: Any -power: Any -prod: Any -product: Any -ptp: Any -put: Any -putmask: Any -ravel: Any -remainder: Any -repeat: Any -reshape: Any -resize: Any -right_shift: Any -round: Any -round_: Any -set_fill_value: Any -shape: Any -sin: Any -sinh: Any -size: Any -soften_mask: Any -sometrue: Any -sort: Any -sqrt: Any -squeeze: Any -std: Any -subtract: Any -sum: Any -swapaxes: Any -take: Any -tan: Any -tanh: Any -trace: Any -transpose: Any -true_divide: Any -var: Any -where: Any -zeros: Any -apply_along_axis: Any -apply_over_axes: Any -atleast_1d: Any -atleast_2d: Any -atleast_3d: Any -average: Any -clump_masked: Any -clump_unmasked: Any -column_stack: Any -compress_cols: Any -compress_nd: Any -compress_rowcols: Any -compress_rows: Any -count_masked: Any -corrcoef: Any -cov: Any -diagflat: Any -dot: Any -dstack: Any -ediff1d: Any -flatnotmasked_contiguous: Any -flatnotmasked_edges: Any -hsplit: Any -hstack: Any -isin: Any -in1d: Any -intersect1d: Any -mask_cols: Any -mask_rowcols: Any -mask_rows: Any -masked_all: Any -masked_all_like: Any -median: Any -mr_: Any -notmasked_contiguous: Any -notmasked_edges: Any -polyfit: Any -row_stack: Any -setdiff1d: Any -setxor1d: Any -stack: Any -unique: Any -union1d: Any -vander: Any -vstack: Any +from numpy.ma import extras as extras + +from numpy.ma.core import ( + MAError as MAError, + MaskError as MaskError, + MaskType as MaskType, + MaskedArray as MaskedArray, + abs as abs, + absolute as absolute, + add as add, + all as all, + allclose as allclose, + allequal as allequal, + alltrue as alltrue, + amax as amax, + amin as amin, + angle as angle, + anom as anom, + anomalies as anomalies, + any as any, + append as append, + arange as arange, + arccos as arccos, + arccosh as arccosh, + arcsin as arcsin, + arcsinh as arcsinh, + arctan as arctan, + arctan2 as arctan2, + arctanh as arctanh, + argmax as argmax, + argmin as argmin, + argsort as argsort, + around as around, + array as array, + asanyarray as asanyarray, + asarray as asarray, + bitwise_and as bitwise_and, + bitwise_or as bitwise_or, + bitwise_xor as bitwise_xor, + bool_ as bool_, + ceil as ceil, + choose as choose, + clip as clip, + common_fill_value as common_fill_value, + compress as compress, + compressed as compressed, + concatenate as concatenate, + conjugate as conjugate, + convolve as convolve, + copy as copy, + correlate as correlate, + cos as cos, + cosh as cosh, + count as count, + cumprod as cumprod, + cumsum as cumsum, + default_fill_value as default_fill_value, + diag as diag, + diagonal as diagonal, + diff as diff, + divide as divide, + empty as empty, + empty_like as empty_like, + equal as equal, + exp as exp, + expand_dims as expand_dims, + fabs as fabs, + filled as filled, + fix_invalid as fix_invalid, + flatten_mask as flatten_mask, + flatten_structured_array as flatten_structured_array, + floor as floor, + floor_divide as floor_divide, + fmod as fmod, + frombuffer as frombuffer, + fromflex as fromflex, + fromfunction as fromfunction, + getdata as getdata, + getmask as getmask, + getmaskarray as getmaskarray, + greater as greater, + greater_equal as greater_equal, + harden_mask as harden_mask, + hypot as hypot, + identity as identity, + ids as ids, + indices as indices, + inner as inner, + innerproduct as innerproduct, + isMA as isMA, + isMaskedArray as isMaskedArray, + is_mask as is_mask, + is_masked as is_masked, + isarray as isarray, + left_shift as left_shift, + less as less, + less_equal as less_equal, + log as log, + log10 as log10, + log2 as log2, + logical_and as logical_and, + logical_not as logical_not, + logical_or as logical_or, + logical_xor as logical_xor, + make_mask as make_mask, + make_mask_descr as make_mask_descr, + make_mask_none as make_mask_none, + mask_or as mask_or, + masked as masked, + masked_array as masked_array, + masked_equal as masked_equal, + masked_greater as masked_greater, + masked_greater_equal as masked_greater_equal, + masked_inside as masked_inside, + masked_invalid as masked_invalid, + masked_less as masked_less, + masked_less_equal as masked_less_equal, + masked_not_equal as masked_not_equal, + masked_object as masked_object, + masked_outside as masked_outside, + masked_print_option as masked_print_option, + masked_singleton as masked_singleton, + masked_values as masked_values, + masked_where as masked_where, + max as max, + maximum as maximum, + maximum_fill_value as maximum_fill_value, + mean as mean, + min as min, + minimum as minimum, + minimum_fill_value as minimum_fill_value, + mod as mod, + multiply as multiply, + mvoid as mvoid, + ndim as ndim, + negative as negative, + nomask as nomask, + nonzero as nonzero, + not_equal as not_equal, + ones as ones, + outer as outer, + outerproduct as outerproduct, + power as power, + prod as prod, + product as product, + ptp as ptp, + put as put, + putmask as putmask, + ravel as ravel, + remainder as remainder, + repeat as repeat, + reshape as reshape, + resize as resize, + right_shift as right_shift, + round as round, + round_ as round_, + set_fill_value as set_fill_value, + shape as shape, + sin as sin, + sinh as sinh, + size as size, + soften_mask as soften_mask, + sometrue as sometrue, + sort as sort, + sqrt as sqrt, + squeeze as squeeze, + std as std, + subtract as subtract, + sum as sum, + swapaxes as swapaxes, + take as take, + tan as tan, + tanh as tanh, + trace as trace, + transpose as transpose, + true_divide as true_divide, + var as var, + where as where, + zeros as zeros, +) + +from numpy.ma.extras import ( + apply_along_axis as apply_along_axis, + apply_over_axes as apply_over_axes, + atleast_1d as atleast_1d, + atleast_2d as atleast_2d, + atleast_3d as atleast_3d, + average as average, + clump_masked as clump_masked, + clump_unmasked as clump_unmasked, + column_stack as column_stack, + compress_cols as compress_cols, + compress_nd as compress_nd, + compress_rowcols as compress_rowcols, + compress_rows as compress_rows, + count_masked as count_masked, + corrcoef as corrcoef, + cov as cov, + diagflat as diagflat, + dot as dot, + dstack as dstack, + ediff1d as ediff1d, + flatnotmasked_contiguous as flatnotmasked_contiguous, + flatnotmasked_edges as flatnotmasked_edges, + hsplit as hsplit, + hstack as hstack, + isin as isin, + in1d as in1d, + intersect1d as intersect1d, + mask_cols as mask_cols, + mask_rowcols as mask_rowcols, + mask_rows as mask_rows, + masked_all as masked_all, + masked_all_like as masked_all_like, + median as median, + mr_ as mr_, + notmasked_contiguous as notmasked_contiguous, + notmasked_edges as notmasked_edges, + polyfit as polyfit, + row_stack as row_stack, + setdiff1d as setdiff1d, + setxor1d as setxor1d, + stack as stack, + unique as unique, + union1d as union1d, + vander as vander, + vstack as vstack, +) + +__all__: List[str] diff --git a/numpy/ma/core.py b/numpy/ma/core.py index 4fb7d8c28224..63d42eccad08 100644 --- a/numpy/ma/core.py +++ b/numpy/ma/core.py @@ -399,10 +399,10 @@ def _recursive_set_fill_value(fillvalue, dt): Parameters ---------- - fillvalue: scalar or array_like + fillvalue : scalar or array_like Scalar or array representing the fill value. If it is of shorter length than the number of fields in dt, it will be resized. - dt: dtype + dt : dtype The structured dtype for which to create the fill value. Returns @@ -914,7 +914,7 @@ class _MaskedUnaryOperation(_MaskedUFunc): """ def __init__(self, mufunc, fill=0, domain=None): - super(_MaskedUnaryOperation, self).__init__(mufunc) + super().__init__(mufunc) self.fill = fill self.domain = domain ufunc_domain[mufunc] = domain @@ -996,7 +996,7 @@ def __init__(self, mbfunc, fillx=0, filly=0): abfunc(x, filly) = x for all x to enable reduce. """ - super(_MaskedBinaryOperation, self).__init__(mbfunc) + super().__init__(mbfunc) self.fillx = fillx self.filly = filly ufunc_domain[mbfunc] = None @@ -1142,7 +1142,7 @@ def __init__(self, dbfunc, domain, fillx=0, filly=0): """abfunc(fillx, filly) must be defined. abfunc(x, filly) = x for all x to enable reduce. """ - super(_DomainedBinaryOperation, self).__init__(dbfunc) + super().__init__(dbfunc) self.domain = domain self.fillx = fillx self.filly = filly @@ -1936,6 +1936,10 @@ def masked_where(condition, a, copy=True): result = a.view(cls) # Assign to *.mask so that structured masks are handled correctly. result.mask = _shrink_mask(cond) + # There is no view of a boolean so when 'a' is a MaskedArray with nomask + # the update to the result's mask has no effect. + if not copy and hasattr(a, '_mask') and getmask(a) is nomask: + a._mask = result._mask.view() return result @@ -2859,8 +2863,9 @@ def __new__(cls, data=None, mask=nomask, dtype=None, copy=False, elif isinstance(data, (tuple, list)): try: # If data is a sequence of masked array - mask = np.array([getmaskarray(np.asanyarray(m, dtype=mdtype)) - for m in data], dtype=mdtype) + mask = np.array( + [getmaskarray(np.asanyarray(m, dtype=_data.dtype)) + for m in data], dtype=mdtype) except ValueError: # If data is nested mask = nomask @@ -3399,7 +3404,7 @@ def __setitem__(self, indx, value): # Define so that we can overwrite the setter. @property def dtype(self): - return super(MaskedArray, self).dtype + return super().dtype @dtype.setter def dtype(self, dtype): @@ -3415,7 +3420,7 @@ def dtype(self, dtype): @property def shape(self): - return super(MaskedArray, self).shape + return super().shape @shape.setter def shape(self, shape): @@ -3894,7 +3899,7 @@ def compress(self, condition, axis=None, out=None): # Force the condition to a regular ndarray and forget the missing # values. - condition = np.array(condition, copy=False, subok=False) + condition = np.asarray(condition) _new = _data.compress(condition, axis=axis, out=out).view(type(self)) _new._update_from(self) @@ -4984,8 +4989,8 @@ def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None): #!!!: implement out + test! m = self._mask if m is nomask: - result = super(MaskedArray, self).trace(offset=offset, axis1=axis1, - axis2=axis2, out=out) + result = super().trace(offset=offset, axis1=axis1, axis2=axis2, + out=out) return result.astype(dtype) else: D = self.diagonal(offset=offset, axis1=axis1, axis2=axis2) @@ -5220,7 +5225,7 @@ def mean(self, axis=None, dtype=None, out=None, keepdims=np._NoValue): -------- numpy.ndarray.mean : corresponding function for ndarrays numpy.mean : Equivalent function - numpy.ma.average: Weighted average. + numpy.ma.average : Weighted average. Examples -------- @@ -5236,8 +5241,7 @@ def mean(self, axis=None, dtype=None, out=None, keepdims=np._NoValue): kwargs = {} if keepdims is np._NoValue else {'keepdims': keepdims} if self._mask is nomask: - result = super(MaskedArray, self).mean(axis=axis, - dtype=dtype, **kwargs)[()] + result = super().mean(axis=axis, dtype=dtype, **kwargs)[()] else: dsum = self.sum(axis=axis, dtype=dtype, **kwargs) cnt = self.count(axis=axis, **kwargs) @@ -5314,8 +5318,8 @@ def var(self, axis=None, dtype=None, out=None, ddof=0, # Easy case: nomask, business as usual if self._mask is nomask: - ret = super(MaskedArray, self).var(axis=axis, dtype=dtype, out=out, - ddof=ddof, **kwargs)[()] + ret = super().var(axis=axis, dtype=dtype, out=out, ddof=ddof, + **kwargs)[()] if out is not None: if isinstance(out, MaskedArray): out.__setmask__(nomask) @@ -5439,7 +5443,7 @@ def argsort(self, axis=np._NoValue, kind=None, order=None, When the array contains unmasked values at the same extremes of the datatype, the ordering of these values and the masked values is undefined. - fill_value : {var}, optional + fill_value : scalar or None, optional Value used internally for the masked values. If ``fill_value`` is not None, it supersedes ``endwith``. @@ -5497,7 +5501,7 @@ def argmin(self, axis=None, fill_value=None, out=None): axis : {None, integer} If None, the index is into the flattened array, otherwise along the specified axis - fill_value : {var}, optional + fill_value : scalar or None, optional Value used to fill in the masked values. If None, the output of minimum_fill_value(self._data) is used instead. out : {None, array}, optional @@ -5543,7 +5547,7 @@ def argmax(self, axis=None, fill_value=None, out=None): axis : {None, integer} If None, the index is into the flattened array, otherwise along the specified axis - fill_value : {var}, optional + fill_value : scalar or None, optional Value used to fill in the masked values. If None, the output of maximum_fill_value(self._data) is used instead. out : {None, array}, optional @@ -5594,7 +5598,7 @@ def sort(self, axis=-1, kind=None, order=None, When the array contains unmasked values sorting at the same extremes of the datatype, the ordering of these values and the masked values is undefined. - fill_value : {var}, optional + fill_value : scalar or None, optional Value used internally for the masked values. If ``fill_value`` is not None, it supersedes ``endwith``. @@ -5665,7 +5669,7 @@ def min(self, axis=None, out=None, fill_value=None, keepdims=np._NoValue): out : array_like, optional Alternative output array in which to place the result. Must be of the same shape and buffer length as the expected output. - fill_value : {var}, optional + fill_value : scalar or None, optional Value used to fill in the masked values. If None, use the output of `minimum_fill_value`. keepdims : bool, optional @@ -5799,7 +5803,7 @@ def max(self, axis=None, out=None, fill_value=None, keepdims=np._NoValue): out : array_like, optional Alternative output array in which to place the result. Must be of the same shape and buffer length as the expected output. - fill_value : {var}, optional + fill_value : scalar or None, optional Value used to fill in the masked values. If None, use the output of maximum_fill_value(). keepdims : bool, optional @@ -5876,7 +5880,7 @@ def ptp(self, axis=None, out=None, fill_value=None, keepdims=False): Alternative output array in which to place the result. It must have the same shape and buffer length as the expected output but the type will be cast if necessary. - fill_value : {var}, optional + fill_value : scalar or None, optional Value used to fill in the masked values. keepdims : bool, optional If this is set to True, the axes which are reduced are left @@ -5946,13 +5950,13 @@ def partition(self, *args, **kwargs): warnings.warn("Warning: 'partition' will ignore the 'mask' " f"of the {self.__class__.__name__}.", stacklevel=2) - return super(MaskedArray, self).partition(*args, **kwargs) + return super().partition(*args, **kwargs) def argpartition(self, *args, **kwargs): warnings.warn("Warning: 'argpartition' will ignore the 'mask' " f"of the {self.__class__.__name__}.", stacklevel=2) - return super(MaskedArray, self).argpartition(*args, **kwargs) + return super().argpartition(*args, **kwargs) def take(self, indices, axis=None, out=None, mode='raise'): """ @@ -6178,7 +6182,7 @@ def __getstate__(self): """ cf = 'CF'[self.flags.fnc] - data_state = super(MaskedArray, self).__reduce__()[2] + data_state = super().__reduce__()[2] return data_state + (getmaskarray(self).tobytes(cf), self._fill_value) def __setstate__(self, state): @@ -6194,7 +6198,7 @@ def __setstate__(self, state): """ (_, shp, typ, isf, raw, msk, flv) = state - super(MaskedArray, self).__setstate__((shp, typ, isf, raw)) + super().__setstate__((shp, typ, isf, raw)) self._mask.__setstate__((shp, make_mask_descr(typ), isf, msk)) self.fill_value = flv @@ -6255,7 +6259,7 @@ def __new__(self, data, mask=nomask, dtype=None, fill_value=None, @property def _data(self): # Make sure that the _data part is a np.void - return super(mvoid, self)._data[()] + return super()._data[()] def __getitem__(self, indx): """ @@ -6292,7 +6296,7 @@ def __str__(self): return str(self._data) rdtype = _replace_dtype_fields(self._data.dtype, "O") - data_arr = super(mvoid, self)._data + data_arr = super()._data res = data_arr.astype(rdtype) _recursive_printoption(res, self._mask, masked_print_option) return str(res) @@ -6454,7 +6458,7 @@ def __array_finalize__(self, obj): if not self.__has_singleton(): # this handles the `.view` in __new__, which we want to copy across # properties normally - return super(MaskedConstant, self).__array_finalize__(obj) + return super().__array_finalize__(obj) elif self is self.__singleton: # not clear how this can happen, play it safe pass @@ -6528,14 +6532,14 @@ def __deepcopy__(self, memo): def __setattr__(self, attr, value): if not self.__has_singleton(): # allow the singleton to be initialized - return super(MaskedConstant, self).__setattr__(attr, value) + return super().__setattr__(attr, value) elif self is self.__singleton: raise AttributeError( f"attributes of {self!r} are not writeable") else: # duplicate instance - we can end up here from __array_finalize__, # where we set the __class__ attribute - return super(MaskedConstant, self).__setattr__(attr, value) + return super().__setattr__(attr, value) masked = masked_singleton = MaskedConstant() @@ -6627,7 +6631,7 @@ class _extrema_operation(_MaskedUFunc): """ def __init__(self, ufunc, compare, fill_value): - super(_extrema_operation, self).__init__(ufunc) + super().__init__(ufunc) self.compare = compare self.fill_value_func = fill_value @@ -6913,8 +6917,7 @@ def compressed(x): See Also -------- - ma.MaskedArray.compressed - Equivalent method. + ma.MaskedArray.compressed : Equivalent method. """ return asanyarray(x).compressed() @@ -7339,16 +7342,16 @@ def where(condition, x=_NoValue, y=_NoValue): def choose(indices, choices, out=None, mode='raise'): """ - Use an index array to construct a new array from a set of choices. + Use an index array to construct a new array from a list of choices. - Given an array of integers and a set of n choice arrays, this method + Given an array of integers and a list of n choice arrays, this method will create a new array that merges each of the choice arrays. Where a - value in `a` is i, the new array will have the value that choices[i] + value in `index` is i, the new array will have the value that choices[i] contains in the same place. Parameters ---------- - a : ndarray of ints + indices : ndarray of ints This array must contain integers in ``[0, n-1]``, where n is the number of choices. choices : sequence of arrays @@ -7870,9 +7873,14 @@ def allclose(a, b, masked_equal=True, rtol=1e-5, atol=1e-8): # make sure y is an inexact type to avoid abs(MIN_INT); will cause # casting of x later. - dtype = np.result_type(y, 1.) - if y.dtype != dtype: - y = masked_array(y, dtype=dtype, copy=False) + # NOTE: We explicitly allow timedelta, which used to work. This could + # possibly be deprecated. See also gh-18286. + # timedelta works if `atol` is an integer or also a timedelta. + # Although, the default tolerances are unlikely to be useful + if y.dtype.kind != "m": + dtype = np.result_type(y, 1.) + if y.dtype != dtype: + y = masked_array(y, dtype=dtype, copy=False) m = mask_or(getmask(x), getmask(y)) xinf = np.isinf(masked_array(x, copy=False, mask=m)).filled(False) diff --git a/numpy/ma/core.pyi b/numpy/ma/core.pyi new file mode 100644 index 000000000000..e7e3f1f36818 --- /dev/null +++ b/numpy/ma/core.pyi @@ -0,0 +1,468 @@ +from typing import Any, List, TypeVar, Callable +from numpy import ndarray, dtype, float64 + +from numpy import ( + amax as amax, + amin as amin, + bool_ as bool_, + expand_dims as expand_dims, + diff as diff, + clip as clip, + indices as indices, + ones_like as ones_like, + squeeze as squeeze, + zeros_like as zeros_like, +) + +from numpy.lib.function_base import ( + angle as angle, +) + +# TODO: Set the `bound` to something more suitable once we +# have proper shape support +_ShapeType = TypeVar("_ShapeType", bound=Any) +_DType_co = TypeVar("_DType_co", bound=dtype[Any], covariant=True) + +__all__: List[str] + +MaskType = bool_ +nomask: bool_ + +class MaskedArrayFutureWarning(FutureWarning): ... +class MAError(Exception): ... +class MaskError(MAError): ... + +def default_fill_value(obj): ... +def minimum_fill_value(obj): ... +def maximum_fill_value(obj): ... +def set_fill_value(a, fill_value): ... +def common_fill_value(a, b): ... +def filled(a, fill_value=...): ... +def getdata(a, subok=...): ... +get_data = getdata + +def fix_invalid(a, mask=..., copy=..., fill_value=...): ... + +class _MaskedUFunc: + f: Any + __doc__: Any + __name__: Any + def __init__(self, ufunc): ... + +class _MaskedUnaryOperation(_MaskedUFunc): + fill: Any + domain: Any + def __init__(self, mufunc, fill=..., domain=...): ... + def __call__(self, a, *args, **kwargs): ... + +class _MaskedBinaryOperation(_MaskedUFunc): + fillx: Any + filly: Any + def __init__(self, mbfunc, fillx=..., filly=...): ... + def __call__(self, a, b, *args, **kwargs): ... + def reduce(self, target, axis=..., dtype=...): ... + def outer(self, a, b): ... + def accumulate(self, target, axis=...): ... + +class _DomainedBinaryOperation(_MaskedUFunc): + domain: Any + fillx: Any + filly: Any + def __init__(self, dbfunc, domain, fillx=..., filly=...): ... + def __call__(self, a, b, *args, **kwargs): ... + +exp: _MaskedUnaryOperation +conjugate: _MaskedUnaryOperation +sin: _MaskedUnaryOperation +cos: _MaskedUnaryOperation +arctan: _MaskedUnaryOperation +arcsinh: _MaskedUnaryOperation +sinh: _MaskedUnaryOperation +cosh: _MaskedUnaryOperation +tanh: _MaskedUnaryOperation +abs: _MaskedUnaryOperation +absolute: _MaskedUnaryOperation +fabs: _MaskedUnaryOperation +negative: _MaskedUnaryOperation +floor: _MaskedUnaryOperation +ceil: _MaskedUnaryOperation +around: _MaskedUnaryOperation +logical_not: _MaskedUnaryOperation +sqrt: _MaskedUnaryOperation +log: _MaskedUnaryOperation +log2: _MaskedUnaryOperation +log10: _MaskedUnaryOperation +tan: _MaskedUnaryOperation +arcsin: _MaskedUnaryOperation +arccos: _MaskedUnaryOperation +arccosh: _MaskedUnaryOperation +arctanh: _MaskedUnaryOperation + +add: _MaskedBinaryOperation +subtract: _MaskedBinaryOperation +multiply: _MaskedBinaryOperation +arctan2: _MaskedBinaryOperation +equal: _MaskedBinaryOperation +not_equal: _MaskedBinaryOperation +less_equal: _MaskedBinaryOperation +greater_equal: _MaskedBinaryOperation +less: _MaskedBinaryOperation +greater: _MaskedBinaryOperation +logical_and: _MaskedBinaryOperation +alltrue: _MaskedBinaryOperation +logical_or: _MaskedBinaryOperation +sometrue: Callable[..., Any] +logical_xor: _MaskedBinaryOperation +bitwise_and: _MaskedBinaryOperation +bitwise_or: _MaskedBinaryOperation +bitwise_xor: _MaskedBinaryOperation +hypot: _MaskedBinaryOperation +divide: _MaskedBinaryOperation +true_divide: _MaskedBinaryOperation +floor_divide: _MaskedBinaryOperation +remainder: _MaskedBinaryOperation +fmod: _MaskedBinaryOperation +mod: _MaskedBinaryOperation + +def make_mask_descr(ndtype): ... +def getmask(a): ... +get_mask = getmask + +def getmaskarray(arr): ... +def is_mask(m): ... +def make_mask(m, copy=..., shrink=..., dtype=...): ... +def make_mask_none(newshape, dtype=...): ... +def mask_or(m1, m2, copy=..., shrink=...): ... +def flatten_mask(mask): ... +def masked_where(condition, a, copy=...): ... +def masked_greater(x, value, copy=...): ... +def masked_greater_equal(x, value, copy=...): ... +def masked_less(x, value, copy=...): ... +def masked_less_equal(x, value, copy=...): ... +def masked_not_equal(x, value, copy=...): ... +def masked_equal(x, value, copy=...): ... +def masked_inside(x, v1, v2, copy=...): ... +def masked_outside(x, v1, v2, copy=...): ... +def masked_object(x, value, copy=..., shrink=...): ... +def masked_values(x, value, rtol=..., atol=..., copy=..., shrink=...): ... +def masked_invalid(a, copy=...): ... + +class _MaskedPrintOption: + def __init__(self, display): ... + def display(self): ... + def set_display(self, s): ... + def enabled(self): ... + def enable(self, shrink=...): ... + +masked_print_option: _MaskedPrintOption + +def flatten_structured_array(a): ... + +class MaskedIterator: + ma: Any + dataiter: Any + maskiter: Any + def __init__(self, ma): ... + def __iter__(self): ... + def __getitem__(self, indx): ... + def __setitem__(self, index, value): ... + def __next__(self): ... + +class MaskedArray(ndarray[_ShapeType, _DType_co]): + __array_priority__: Any + def __new__(cls, data=..., mask=..., dtype=..., copy=..., subok=..., ndmin=..., fill_value=..., keep_mask=..., hard_mask=..., shrink=..., order=...): ... + def __array_finalize__(self, obj): ... + def __array_wrap__(self, obj, context=...): ... + def view(self, dtype=..., type=..., fill_value=...): ... + def __getitem__(self, indx): ... + def __setitem__(self, indx, value): ... + @property + def dtype(self): ... + @dtype.setter + def dtype(self, dtype): ... + @property + def shape(self): ... + @shape.setter + def shape(self, shape): ... + def __setmask__(self, mask, copy=...): ... + @property + def mask(self): ... + @mask.setter + def mask(self, value): ... + @property + def recordmask(self): ... + @recordmask.setter + def recordmask(self, mask): ... + def harden_mask(self): ... + def soften_mask(self): ... + @property + def hardmask(self): ... + def unshare_mask(self): ... + @property + def sharedmask(self): ... + def shrink_mask(self): ... + @property + def baseclass(self): ... + data: Any + @property + def flat(self): ... + @flat.setter + def flat(self, value): ... + @property + def fill_value(self): ... + @fill_value.setter + def fill_value(self, value=...): ... + get_fill_value: Any + set_fill_value: Any + def filled(self, fill_value=...): ... + def compressed(self): ... + def compress(self, condition, axis=..., out=...): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def __add__(self, other): ... + def __radd__(self, other): ... + def __sub__(self, other): ... + def __rsub__(self, other): ... + def __mul__(self, other): ... + def __rmul__(self, other): ... + def __div__(self, other): ... + def __truediv__(self, other): ... + def __rtruediv__(self, other): ... + def __floordiv__(self, other): ... + def __rfloordiv__(self, other): ... + def __pow__(self, other): ... + def __rpow__(self, other): ... + def __iadd__(self, other): ... + def __isub__(self, other): ... + def __imul__(self, other): ... + def __idiv__(self, other): ... + def __ifloordiv__(self, other): ... + def __itruediv__(self, other): ... + def __ipow__(self, other): ... + def __float__(self): ... + def __int__(self): ... + @property # type: ignore[misc] + def imag(self): ... + get_imag: Any + @property # type: ignore[misc] + def real(self): ... + get_real: Any + def count(self, axis=..., keepdims=...): ... + def ravel(self, order=...): ... + def reshape(self, *s, **kwargs): ... + def resize(self, newshape, refcheck=..., order=...): ... + def put(self, indices, values, mode=...): ... + def ids(self): ... + def iscontiguous(self): ... + def all(self, axis=..., out=..., keepdims=...): ... + def any(self, axis=..., out=..., keepdims=...): ... + def nonzero(self): ... + def trace(self, offset=..., axis1=..., axis2=..., dtype=..., out=...): ... + def dot(self, b, out=..., strict=...): ... + def sum(self, axis=..., dtype=..., out=..., keepdims=...): ... + def cumsum(self, axis=..., dtype=..., out=...): ... + def prod(self, axis=..., dtype=..., out=..., keepdims=...): ... + product: Any + def cumprod(self, axis=..., dtype=..., out=...): ... + def mean(self, axis=..., dtype=..., out=..., keepdims=...): ... + def anom(self, axis=..., dtype=...): ... + def var(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ... + def std(self, axis=..., dtype=..., out=..., ddof=..., keepdims=...): ... + def round(self, decimals=..., out=...): ... + def argsort(self, axis=..., kind=..., order=..., endwith=..., fill_value=...): ... + def argmin(self, axis=..., fill_value=..., out=...): ... + def argmax(self, axis=..., fill_value=..., out=...): ... + def sort(self, axis=..., kind=..., order=..., endwith=..., fill_value=...): ... + def min(self, axis=..., out=..., fill_value=..., keepdims=...): ... + # NOTE: deprecated + # def mini(self, axis=...): ... + # def tostring(self, fill_value=..., order=...): ... + def max(self, axis=..., out=..., fill_value=..., keepdims=...): ... + def ptp(self, axis=..., out=..., fill_value=..., keepdims=...): ... + def partition(self, *args, **kwargs): ... + def argpartition(self, *args, **kwargs): ... + def take(self, indices, axis=..., out=..., mode=...): ... + copy: Any + diagonal: Any + flatten: Any + repeat: Any + squeeze: Any + swapaxes: Any + T: Any + transpose: Any + def tolist(self, fill_value=...): ... + def tobytes(self, fill_value=..., order=...): ... + def tofile(self, fid, sep=..., format=...): ... + def toflex(self): ... + torecords: Any + def __reduce__(self): ... + def __deepcopy__(self, memo=...): ... + +class mvoid(MaskedArray[_ShapeType, _DType_co]): + def __new__( + self, + data, + mask=..., + dtype=..., + fill_value=..., + hardmask=..., + copy=..., + subok=..., + ): ... + def __getitem__(self, indx): ... + def __setitem__(self, indx, value): ... + def __iter__(self): ... + def __len__(self): ... + def filled(self, fill_value=...): ... + def tolist(self): ... + +def isMaskedArray(x): ... +isarray = isMaskedArray +isMA = isMaskedArray + +# 0D float64 array +class MaskedConstant(MaskedArray[Any, dtype[float64]]): + def __new__(cls): ... + __class__: Any + def __array_finalize__(self, obj): ... + def __array_prepare__(self, obj, context=...): ... + def __array_wrap__(self, obj, context=...): ... + def __format__(self, format_spec): ... + def __reduce__(self): ... + def __iop__(self, other): ... + __iadd__: Any + __isub__: Any + __imul__: Any + __ifloordiv__: Any + __itruediv__: Any + __ipow__: Any + def copy(self, *args, **kwargs): ... + def __copy__(self): ... + def __deepcopy__(self, memo): ... + def __setattr__(self, attr, value): ... + +masked: MaskedConstant +masked_singleton: MaskedConstant +masked_array = MaskedArray + +def array( + data, + dtype=..., + copy=..., + order=..., + mask=..., + fill_value=..., + keep_mask=..., + hard_mask=..., + shrink=..., + subok=..., + ndmin=..., +): ... +def is_masked(x): ... + +class _extrema_operation(_MaskedUFunc): + compare: Any + fill_value_func: Any + def __init__(self, ufunc, compare, fill_value): ... + # NOTE: in practice `b` has a default value, but users should + # explicitly provide a value here as the default is deprecated + def __call__(self, a, b): ... + def reduce(self, target, axis=...): ... + def outer(self, a, b): ... + +def min(obj, axis=..., out=..., fill_value=..., keepdims=...): ... +def max(obj, axis=..., out=..., fill_value=..., keepdims=...): ... +def ptp(obj, axis=..., out=..., fill_value=..., keepdims=...): ... + +class _frommethod: + __name__: Any + __doc__: Any + reversed: Any + def __init__(self, methodname, reversed=...): ... + def getdoc(self): ... + def __call__(self, a, *args, **params): ... + +all: _frommethod +anomalies: _frommethod +anom: _frommethod +any: _frommethod +compress: _frommethod +cumprod: _frommethod +cumsum: _frommethod +copy: _frommethod +diagonal: _frommethod +harden_mask: _frommethod +ids: _frommethod +mean: _frommethod +nonzero: _frommethod +prod: _frommethod +product: _frommethod +ravel: _frommethod +repeat: _frommethod +soften_mask: _frommethod +std: _frommethod +sum: _frommethod +swapaxes: _frommethod +trace: _frommethod +var: _frommethod +count: _frommethod +argmin: _frommethod +argmax: _frommethod + +minimum: _extrema_operation +maximum: _extrema_operation + +def take(a, indices, axis=..., out=..., mode=...): ... +def power(a, b, third=...): ... +def argsort(a, axis=..., kind=..., order=..., endwith=..., fill_value=...): ... +def sort(a, axis=..., kind=..., order=..., endwith=..., fill_value=...): ... +def compressed(x): ... +def concatenate(arrays, axis=...): ... +def diag(v, k=...): ... +def left_shift(a, n): ... +def right_shift(a, n): ... +def put(a, indices, values, mode=...): ... +def putmask(a, mask, values): ... +def transpose(a, axes=...): ... +def reshape(a, new_shape, order=...): ... +def resize(x, new_shape): ... +def ndim(obj): ... +def shape(obj): ... +def size(obj, axis=...): ... +def where(condition, x=..., y=...): ... +def choose(indices, choices, out=..., mode=...): ... +def round_(a, decimals=..., out=...): ... +round = round_ + +def inner(a, b): ... +innerproduct = inner + +def outer(a, b): ... +outerproduct = outer + +def correlate(a, v, mode=..., propagate_mask=...): ... +def convolve(a, v, mode=..., propagate_mask=...): ... +def allequal(a, b, fill_value=...): ... +def allclose(a, b, masked_equal=..., rtol=..., atol=...): ... +def asarray(a, dtype=..., order=...): ... +def asanyarray(a, dtype=...): ... +def fromflex(fxarray): ... + +class _convert2ma: + __doc__: Any + def __init__(self, funcname, params=...): ... + def getdoc(self): ... + def __call__(self, *args, **params): ... + +arange: _convert2ma +empty: _convert2ma +empty_like: _convert2ma +frombuffer: _convert2ma +fromfunction: _convert2ma +identity: _convert2ma +ones: _convert2ma +zeros: _convert2ma + +def append(a, b, axis=...): ... +def dot(a, b, strict=..., out=...): ... +def mask_rowcols(a, axis=...): ... diff --git a/numpy/ma/extras.py b/numpy/ma/extras.py index 1bf03e966326..bd5fc2ca3bb2 100644 --- a/numpy/ma/extras.py +++ b/numpy/ma/extras.py @@ -614,7 +614,7 @@ def average(a, axis=None, weights=None, returned=False): "Length of weights not compatible with specified axis.") # setup wgt to broadcast along axis - wgt = np.broadcast_to(wgt, (a.ndim-1)*(1,) + wgt.shape) + wgt = np.broadcast_to(wgt, (a.ndim-1)*(1,) + wgt.shape, subok=True) wgt = wgt.swapaxes(-1, axis) if m is not nomask: @@ -1217,7 +1217,7 @@ def union1d(ar1, ar2): The output is always a masked array. See `numpy.union1d` for more details. - See also + See Also -------- numpy.union1d : Equivalent function for ndarrays. @@ -1322,7 +1322,7 @@ def cov(x, y=None, rowvar=True, bias=False, allow_masked=True, ddof=None): observation of all those variables. Also see `rowvar` below. y : array_like, optional An additional set of variables and observations. `y` has the same - form as `x`. + shape as `x`. rowvar : bool, optional If `rowvar` is True (default), then each row represents a variable, with observations in the columns. Otherwise, the relationship @@ -1483,7 +1483,7 @@ def makemat(cls, arr): # deprecate that class. In preparation, we use the unmasked version # to construct the matrix (with copy=False for backwards compatibility # with the .view) - data = super(MAxisConcatenator, cls).makemat(arr.data, copy=False) + data = super().makemat(arr.data, copy=False) return array(data, mask=arr.mask) def __getitem__(self, key): @@ -1491,7 +1491,7 @@ def __getitem__(self, key): if isinstance(key, str): raise MAError("Unavailable for masked array.") - return super(MAxisConcatenator, self).__getitem__(key) + return super().__getitem__(key) class mr_class(MAxisConcatenator): diff --git a/numpy/ma/extras.pyi b/numpy/ma/extras.pyi new file mode 100644 index 000000000000..e58e43badf23 --- /dev/null +++ b/numpy/ma/extras.pyi @@ -0,0 +1,84 @@ +from typing import Any, List +from numpy.lib.index_tricks import AxisConcatenator + +from numpy.ma.core import ( + dot as dot, + mask_rowcols as mask_rowcols, +) + +__all__: List[str] + +def count_masked(arr, axis=...): ... +def masked_all(shape, dtype = ...): ... +def masked_all_like(arr): ... + +class _fromnxfunction: + __name__: Any + __doc__: Any + def __init__(self, funcname): ... + def getdoc(self): ... + def __call__(self, *args, **params): ... + +class _fromnxfunction_single(_fromnxfunction): + def __call__(self, x, *args, **params): ... + +class _fromnxfunction_seq(_fromnxfunction): + def __call__(self, x, *args, **params): ... + +class _fromnxfunction_allargs(_fromnxfunction): + def __call__(self, *args, **params): ... + +atleast_1d: _fromnxfunction_allargs +atleast_2d: _fromnxfunction_allargs +atleast_3d: _fromnxfunction_allargs + +vstack: _fromnxfunction_seq +row_stack: _fromnxfunction_seq +hstack: _fromnxfunction_seq +column_stack: _fromnxfunction_seq +dstack: _fromnxfunction_seq +stack: _fromnxfunction_seq + +hsplit: _fromnxfunction_single +diagflat: _fromnxfunction_single + +def apply_along_axis(func1d, axis, arr, *args, **kwargs): ... +def apply_over_axes(func, a, axes): ... +def average(a, axis=..., weights=..., returned=...): ... +def median(a, axis=..., out=..., overwrite_input=..., keepdims=...): ... +def compress_nd(x, axis=...): ... +def compress_rowcols(x, axis=...): ... +def compress_rows(a): ... +def compress_cols(a): ... +def mask_rows(a, axis = ...): ... +def mask_cols(a, axis = ...): ... +def ediff1d(arr, to_end=..., to_begin=...): ... +def unique(ar1, return_index=..., return_inverse=...): ... +def intersect1d(ar1, ar2, assume_unique=...): ... +def setxor1d(ar1, ar2, assume_unique=...): ... +def in1d(ar1, ar2, assume_unique=..., invert=...): ... +def isin(element, test_elements, assume_unique=..., invert=...): ... +def union1d(ar1, ar2): ... +def setdiff1d(ar1, ar2, assume_unique=...): ... +def cov(x, y=..., rowvar=..., bias=..., allow_masked=..., ddof=...): ... +def corrcoef(x, y=..., rowvar=..., bias = ..., allow_masked=..., ddof = ...): ... + +class MAxisConcatenator(AxisConcatenator): + concatenate: Any + @classmethod + def makemat(cls, arr): ... + def __getitem__(self, key): ... + +class mr_class(MAxisConcatenator): + def __init__(self): ... + +mr_: mr_class + +def flatnotmasked_edges(a): ... +def notmasked_edges(a, axis=...): ... +def flatnotmasked_contiguous(a): ... +def notmasked_contiguous(a, axis=...): ... +def clump_unmasked(a): ... +def clump_masked(a): ... +def vander(x, n=...): ... +def polyfit(x, y, deg, rcond=..., full=..., w=..., cov=...): ... diff --git a/numpy/ma/mrecords.py b/numpy/ma/mrecords.py index 70087632e9f6..9ea4e4e3627d 100644 --- a/numpy/ma/mrecords.py +++ b/numpy/ma/mrecords.py @@ -13,23 +13,23 @@ # first place, and then rename the invalid fields with a trailing # underscore. Maybe we could just overload the parser function ? +from numpy.ma import ( + MAError, MaskedArray, masked, nomask, masked_array, getdata, + getmaskarray, filled +) +import numpy.ma as ma import warnings import numpy as np from numpy import ( - bool_, dtype, ndarray, recarray, array as narray - ) + bool_, dtype, ndarray, recarray, array as narray +) from numpy.core.records import ( - fromarrays as recfromarrays, fromrecords as recfromrecords - ) + fromarrays as recfromarrays, fromrecords as recfromrecords +) _byteorderconv = np.core.records._byteorderconv -import numpy.ma as ma -from numpy.ma import ( - MAError, MaskedArray, masked, nomask, masked_array, getdata, - getmaskarray, filled - ) _check_fill_value = ma.core._check_fill_value @@ -37,7 +37,7 @@ __all__ = [ 'MaskedRecords', 'mrecarray', 'fromarrays', 'fromrecords', 'fromtextfile', 'addfield', - ] +] reserved_fields = ['_data', '_mask', '_fieldmask', 'dtype'] @@ -199,7 +199,8 @@ def __getattribute__(self, attr): try: res = fielddict[attr][:2] except (TypeError, KeyError) as e: - raise AttributeError(f'record array has no attribute {attr}') from e + raise AttributeError( + f'record array has no attribute {attr}') from e # So far, so good _localdict = ndarray.__getattribute__(self, '__dict__') _data = ndarray.view(self, _localdict['_baseclass']) @@ -273,8 +274,9 @@ def __setattr__(self, attr, val): # Let's try to set the field try: res = fielddict[attr][:2] - except (TypeError, KeyError): - raise AttributeError(f'record array has no attribute {attr}') + except (TypeError, KeyError) as e: + raise AttributeError( + f'record array has no attribute {attr}') from e if val is masked: _fill_value = _localdict['_fill_value'] @@ -355,7 +357,7 @@ def __repr__(self): reprstr = [fmt % (f, getattr(self, f)) for f in self.dtype.names] reprstr.insert(0, 'masked_records(') reprstr.extend([fmt % (' fill_value', self.fill_value), - ' )']) + ' )']) return str("\n".join(reprstr)) def view(self, dtype=None, type=None): @@ -483,6 +485,7 @@ def __reduce__(self): (self.__class__, self._baseclass, (0,), 'b',), self.__getstate__()) + def _mrreconstruct(subtype, baseclass, baseshape, basetype,): """ Build a new MaskedArray from the information stored in a pickle. @@ -492,6 +495,7 @@ def _mrreconstruct(subtype, baseclass, baseshape, basetype,): _mask = ndarray.__new__(ndarray, baseshape, 'b1') return subtype.__new__(subtype, _data, mask=_mask, dtype=basetype,) + mrecarray = MaskedRecords @@ -656,8 +660,8 @@ def openfile(fname): # Try to open the file and guess its type try: f = open(fname) - except IOError: - raise IOError(f"No such file: '{fname}'") + except IOError as e: + raise IOError(f"No such file: '{fname}'") from e if f.readline()[:2] != "\\x": f.seek(0, 0) return f @@ -752,7 +756,7 @@ def addfield(mrecord, newfield, newfieldname=None): newdata = recarray(_data.shape, newdtype) # Add the existing field [newdata.setfield(_data.getfield(*f), *f) - for f in _data.dtype.fields.values()] + for f in _data.dtype.fields.values()] # Add the new field newdata.setfield(newfield._data, *newdata.dtype.fields[newfieldname]) newdata = newdata.view(MaskedRecords) @@ -762,7 +766,7 @@ def addfield(mrecord, newfield, newfieldname=None): newmask = recarray(_data.shape, newmdtype) # Add the old masks [newmask.setfield(_mask.getfield(*f), *f) - for f in _mask.dtype.fields.values()] + for f in _mask.dtype.fields.values()] # Add the mask of the new field newmask.setfield(getmaskarray(newfield), *newmask.dtype.fields[newfieldname]) diff --git a/numpy/ma/mrecords.pyi b/numpy/ma/mrecords.pyi new file mode 100644 index 000000000000..92d5afb897e2 --- /dev/null +++ b/numpy/ma/mrecords.pyi @@ -0,0 +1,88 @@ +from typing import List, Any, TypeVar + +from numpy import dtype +from numpy.ma import MaskedArray + +__all__: List[str] + +# TODO: Set the `bound` to something more suitable once we +# have proper shape support +_ShapeType = TypeVar("_ShapeType", bound=Any) +_DType_co = TypeVar("_DType_co", bound=dtype[Any], covariant=True) + +class MaskedRecords(MaskedArray[_ShapeType, _DType_co]): + def __new__( + cls, + shape, + dtype=..., + buf=..., + offset=..., + strides=..., + formats=..., + names=..., + titles=..., + byteorder=..., + aligned=..., + mask=..., + hard_mask=..., + fill_value=..., + keep_mask=..., + copy=..., + **options, + ): ... + _mask: Any + _fill_value: Any + @property + def _data(self): ... + @property + def _fieldmask(self): ... + def __array_finalize__(self, obj): ... + def __len__(self): ... + def __getattribute__(self, attr): ... + def __setattr__(self, attr, val): ... + def __getitem__(self, indx): ... + def __setitem__(self, indx, value): ... + def view(self, dtype=..., type=...): ... + def harden_mask(self): ... + def soften_mask(self): ... + def copy(self): ... + def tolist(self, fill_value=...): ... + def __reduce__(self): ... + +mrecarray = MaskedRecords + +def fromarrays( + arraylist, + dtype=..., + shape=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., + fill_value=..., +): ... + +def fromrecords( + reclist, + dtype=..., + shape=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., + fill_value=..., + mask=..., +): ... + +def fromtextfile( + fname, + delimitor=..., + commentchar=..., + missingchar=..., + varnames=..., + vartypes=..., +): ... + +def addfield(mrecord, newfield, newfieldname=...): ... diff --git a/numpy/ma/tests/test_core.py b/numpy/ma/tests/test_core.py index 0b2e7303cb72..b71fa9069f60 100644 --- a/numpy/ma/tests/test_core.py +++ b/numpy/ma/tests/test_core.py @@ -238,6 +238,26 @@ def test_creation_with_list_of_maskedarrays(self): assert_equal(data, [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0]]) assert_(data.mask is nomask) + def test_creation_with_list_of_maskedarrays_no_bool_cast(self): + # Tests the regression in gh-18551 + masked_str = np.ma.masked_array(['a', 'b'], mask=[True, False]) + normal_int = np.arange(2) + res = np.ma.asarray([masked_str, normal_int], dtype="U21") + assert_array_equal(res.mask, [[True, False], [False, False]]) + + # The above only failed due a long chain of oddity, try also with + # an object array that cannot be converted to bool always: + class NotBool(): + def __bool__(self): + raise ValueError("not a bool!") + masked_obj = np.ma.masked_array([NotBool(), 'b'], mask=[True, False]) + # Check that the NotBool actually fails like we would expect: + with pytest.raises(ValueError, match="not a bool!"): + np.asarray([masked_obj], dtype=bool) + + res = np.ma.asarray([masked_obj, normal_int]) + assert_array_equal(res.mask, [[True, False], [False, False]]) + def test_creation_from_ndarray_with_padding(self): x = np.array([('A', 0)], dtype={'names':['f0','f1'], 'formats':['S4','i8'], @@ -3037,6 +3057,13 @@ def test_allclose(self): a = masked_array([np.iinfo(np.int_).min], dtype=np.int_) assert_(allclose(a, a)) + def test_allclose_timedelta(self): + # Allclose currently works for timedelta64 as long as `atol` is + # an integer or also a timedelta64 + a = np.array([[1, 2, 3, 4]], dtype="m8[ns]") + assert allclose(a, a, atol=0) + assert allclose(a, a, atol=np.timedelta64(1, "ns")) + def test_allany(self): # Checks the any/all methods/functions. x = np.array([[0.13, 0.26, 0.90], @@ -5148,6 +5175,16 @@ def test_masked_array(): a = np.ma.array([0, 1, 2, 3], mask=[0, 0, 1, 0]) assert_equal(np.argwhere(a), [[1], [3]]) +def test_masked_array_no_copy(): + # check nomask array is updated in place + a = np.ma.array([1, 2, 3, 4]) + _ = np.ma.masked_where(a == 3, a, copy=False) + assert_array_equal(a.mask, [False, False, True, False]) + # check masked array is updated in place + a = np.ma.array([1, 2, 3, 4], mask=[1, 0, 0, 0]) + _ = np.ma.masked_where(a == 3, a, copy=False) + assert_array_equal(a.mask, [True, False, True, False]) + def test_append_masked_array(): a = np.ma.masked_equal([1,2,3], value=2) b = np.ma.masked_equal([4,3,2], value=2) @@ -5186,7 +5223,6 @@ def test_append_masked_array_along_axis(): assert_array_equal(result.data, expected.data) assert_array_equal(result.mask, expected.mask) - def test_default_fill_value_complex(): # regression test for Python 3, where 'unicode' was not defined assert_(default_fill_value(1 + 1j) == 1.e20 + 0.0j) diff --git a/numpy/ma/tests/test_extras.py b/numpy/ma/tests/test_extras.py index d237829cb71f..e735b9bc77fa 100644 --- a/numpy/ma/tests/test_extras.py +++ b/numpy/ma/tests/test_extras.py @@ -292,6 +292,23 @@ def test_complex(self): assert_almost_equal(wav1.real, expected1.real) assert_almost_equal(wav1.imag, expected1.imag) + def test_masked_weights(self): + # Test with masked weights. + # (Regression test for https://github.com/numpy/numpy/issues/10438) + a = np.ma.array(np.arange(9).reshape(3, 3), + mask=[[1, 0, 0], [1, 0, 0], [0, 0, 0]]) + weights_unmasked = masked_array([5, 28, 31], mask=False) + weights_masked = masked_array([5, 28, 31], mask=[1, 0, 0]) + + avg_unmasked = average(a, axis=0, + weights=weights_unmasked, returned=False) + expected_unmasked = np.array([6.0, 5.21875, 6.21875]) + assert_almost_equal(avg_unmasked, expected_unmasked) + + avg_masked = average(a, axis=0, weights=weights_masked, returned=False) + expected_masked = np.array([6.0, 5.576271186440678, 6.576271186440678]) + assert_almost_equal(avg_masked, expected_masked) + class TestConcatenator: # Tests for mr_, the equivalent of r_ for masked arrays. diff --git a/numpy/ma/tests/test_mrecords.py b/numpy/ma/tests/test_mrecords.py index c2f859273c38..27df519d266a 100644 --- a/numpy/ma/tests/test_mrecords.py +++ b/numpy/ma/tests/test_mrecords.py @@ -405,7 +405,7 @@ def test_fromarrays(self): for (f, l) in zip(('a', 'b', 'c'), (_a, _b, _c)): assert_equal(getattr(mrec, f)._mask, l._mask) # One record only - _x = ma.array([1, 1.1, 'one'], mask=[1, 0, 0],) + _x = ma.array([1, 1.1, 'one'], mask=[1, 0, 0], dtype=object) assert_equal_records(fromarrays(_x, dtype=mrec.dtype), mrec[0]) def test_fromrecords(self): diff --git a/numpy/ma/tests/test_subclassing.py b/numpy/ma/tests/test_subclassing.py index f2623406d183..1af5396252e9 100644 --- a/numpy/ma/tests/test_subclassing.py +++ b/numpy/ma/tests/test_subclassing.py @@ -28,19 +28,18 @@ def __new__(cls,arr,info={}): return x def __array_finalize__(self, obj): - if callable(getattr(super(SubArray, self), - '__array_finalize__', None)): - super(SubArray, self).__array_finalize__(obj) + if callable(getattr(super(), '__array_finalize__', None)): + super().__array_finalize__(obj) self.info = getattr(obj, 'info', {}).copy() return def __add__(self, other): - result = super(SubArray, self).__add__(other) + result = super().__add__(other) result.info['added'] = result.info.get('added', 0) + 1 return result def __iadd__(self, other): - result = super(SubArray, self).__iadd__(other) + result = super().__iadd__(other) result.info['iadded'] = result.info.get('iadded', 0) + 1 return result @@ -51,7 +50,7 @@ def __iadd__(self, other): class SubMaskedArray(MaskedArray): """Pure subclass of MaskedArray, keeping some info on subclass.""" def __new__(cls, info=None, **kwargs): - obj = super(SubMaskedArray, cls).__new__(cls, **kwargs) + obj = super().__new__(cls, **kwargs) obj._optinfo['info'] = info return obj @@ -123,12 +122,11 @@ def _validate_input(self, value): def __setitem__(self, item, value): # validation ensures direct assignment with ndarray or # masked_print_option will fail - super(ComplicatedSubArray, self).__setitem__( - item, self._validate_input(value)) + super().__setitem__(item, self._validate_input(value)) def __getitem__(self, item): # ensure getter returns our own class also for scalars - value = super(ComplicatedSubArray, self).__getitem__(item) + value = super().__getitem__(item) if not isinstance(value, np.ndarray): # scalar value = value.__array__().view(ComplicatedSubArray) return value @@ -143,7 +141,7 @@ def flat(self, value): y[:] = value def __array_wrap__(self, obj, context=None): - obj = super(ComplicatedSubArray, self).__array_wrap__(obj, context) + obj = super().__array_wrap__(obj, context) if context is not None and context[0] is np.multiply: obj.info['multiplied'] = obj.info.get('multiplied', 0) + 1 diff --git a/numpy/ma/testutils.py b/numpy/ma/testutils.py index 8d55e1763da1..2dd479abe480 100644 --- a/numpy/ma/testutils.py +++ b/numpy/ma/testutils.py @@ -134,8 +134,8 @@ def assert_equal(actual, desired, err_msg=''): msg = build_err_msg([actual, desired], err_msg, header='', names=('x', 'y')) raise ValueError(msg) - actual = np.array(actual, copy=False, subok=True) - desired = np.array(desired, copy=False, subok=True) + actual = np.asanyarray(actual) + desired = np.asanyarray(desired) (actual_dtype, desired_dtype) = (actual.dtype, desired.dtype) if actual_dtype.char == "S" and desired_dtype.char == "S": return _assert_equal_on_sequences(actual.tolist(), diff --git a/numpy/ma/timer_comparison.py b/numpy/ma/timer_comparison.py index aff72f79b8d2..9eb1a23cd693 100644 --- a/numpy/ma/timer_comparison.py +++ b/numpy/ma/timer_comparison.py @@ -7,12 +7,9 @@ from numpy.testing import build_err_msg -# Fixme: this does not look right. -np.seterr(all='ignore') pi = np.pi - class ModuleTester: def __init__(self, module): self.module = module @@ -111,6 +108,7 @@ def assert_array_equal(self, x, y, err_msg=''): self.assert_array_compare(self.equal, x, y, err_msg=err_msg, header='Arrays are not equal') + @np.errstate(all='ignore') def test_0(self): """ Tests creation @@ -121,6 +119,7 @@ def test_0(self): xm = self.masked_array(x, mask=m) xm[0] + @np.errstate(all='ignore') def test_1(self): """ Tests creation @@ -148,6 +147,7 @@ def test_1(self): xf.shape = s assert(self.count(xm) == len(m1) - reduce(lambda x, y:x+y, m1)) + @np.errstate(all='ignore') def test_2(self): """ Tests conversions and indexing. @@ -190,6 +190,7 @@ def test_2(self): m3 = self.make_mask(m, copy=1) assert(m is not m3) + @np.errstate(all='ignore') def test_3(self): """ Tests resize/repeat @@ -209,6 +210,7 @@ def test_3(self): y8 = x4.repeat(2, 0) assert self.allequal(y5, y8) + @np.errstate(all='ignore') def test_4(self): """ Test of take, transpose, inner, outer products. @@ -232,6 +234,7 @@ def test_4(self): assert t[1] == 2 assert t[2] == 3 + @np.errstate(all='ignore') def test_5(self): """ Tests inplace w/ scalar @@ -284,6 +287,7 @@ def test_5(self): x += 1. assert self.allequal(x, y + 1.) + @np.errstate(all='ignore') def test_6(self): """ Tests inplace w/ array @@ -335,6 +339,7 @@ def test_6(self): x /= a xm /= a + @np.errstate(all='ignore') def test_7(self): "Tests ufunc" d = (self.array([1.0, 0, -1, pi/2]*2, mask=[0, 1]+[0]*6), @@ -369,6 +374,7 @@ def test_7(self): self.assert_array_equal(ur.filled(0), mr.filled(0), f) self.assert_array_equal(ur._mask, mr._mask) + @np.errstate(all='ignore') def test_99(self): # test average ott = self.array([0., 1., 2., 3.], mask=[1, 0, 0, 0]) @@ -414,6 +420,7 @@ def test_99(self): self.assert_array_equal(self.average(z, axis=1), [2.5, 5.0]) self.assert_array_equal(self.average(z, axis=0, weights=w2), [0., 1., 99., 99., 4.0, 10.0]) + @np.errstate(all='ignore') def test_A(self): x = self.arange(24) x[5:6] = self.masked diff --git a/numpy/matrixlib/__init__.pyi b/numpy/matrixlib/__init__.pyi index b240bb327a13..e4b5c19a2bd0 100644 --- a/numpy/matrixlib/__init__.pyi +++ b/numpy/matrixlib/__init__.pyi @@ -1,6 +1,11 @@ -from typing import Any +from typing import Any, List -matrix: Any -bmat: Any -mat: Any -asmatrix: Any +from numpy import ( + matrix as matrix, +) + +__all__: List[str] + +def bmat(obj, ldict=..., gdict=...): ... +def asmatrix(data, dtype=...): ... +mat = asmatrix diff --git a/numpy/polynomial/__init__.py b/numpy/polynomial/__init__.py index c832094e277d..4b4361163b2e 100644 --- a/numpy/polynomial/__init__.py +++ b/numpy/polynomial/__init__.py @@ -15,37 +15,36 @@ This package provides *convenience classes* for each of six different kinds of polynomials: - ============ ================ - **Name** **Provides** - ============ ================ - Polynomial Power series - Chebyshev Chebyshev series - Legendre Legendre series - Laguerre Laguerre series - Hermite Hermite series - HermiteE HermiteE series - ============ ================ + ======================== ================ + **Name** **Provides** + ======================== ================ + `~polynomial.Polynomial` Power series + `~chebyshev.Chebyshev` Chebyshev series + `~legendre.Legendre` Legendre series + `~laguerre.Laguerre` Laguerre series + `~hermite.Hermite` Hermite series + `~hermite_e.HermiteE` HermiteE series + ======================== ================ These *convenience classes* provide a consistent interface for creating, manipulating, and fitting data with polynomials of different bases. The convenience classes are the preferred interface for the `~numpy.polynomial` -package, and are available from the `numpy.polynomial` namespace. -This eliminates the need to -navigate to the corresponding submodules, e.g. ``np.polynomial.Polynomial`` -or ``np.polynomial.Chebyshev`` instead of -``np.polynomial.polynomial.Polynomial`` or +package, and are available from the ``numpy.polynomial`` namespace. +This eliminates the need to navigate to the corresponding submodules, e.g. +``np.polynomial.Polynomial`` or ``np.polynomial.Chebyshev`` instead of +``np.polynomial.polynomial.Polynomial`` or ``np.polynomial.chebyshev.Chebyshev``, respectively. The classes provide a more consistent and concise interface than the type-specific functions defined in the submodules for each type of polynomial. For example, to fit a Chebyshev polynomial with degree ``1`` to data given -by arrays ``xdata`` and ``ydata``, the +by arrays ``xdata`` and ``ydata``, the `~chebyshev.Chebyshev.fit` class method:: >>> from numpy.polynomial import Chebyshev >>> c = Chebyshev.fit(xdata, ydata, deg=1) -is preferred over the `chebyshev.chebfit` function from the -`numpy.polynomial.chebyshev` module:: +is preferred over the `chebyshev.chebfit` function from the +``np.polynomial.chebyshev`` module:: >>> from numpy.polynomial.chebyshev import chebfit >>> c = chebfit(xdata, ydata, deg=1) @@ -58,8 +57,8 @@ The following lists the various constants and methods common to all of the classes representing the various kinds of polynomials. In the following, the term ``Poly`` represents any one of the convenience classes (e.g. -``Polynomial``, ``Chebyshev``, ``Hermite``, etc.) while the lowercase ``p`` -represents an **instance** of a polynomial class. +`~polynomial.Polynomial`, `~chebyshev.Chebyshev`, `~hermite.Hermite`, etc.) +while the lowercase ``p`` represents an **instance** of a polynomial class. Constants --------- @@ -77,7 +76,7 @@ - ``Poly.basis(degree)`` -- Basis polynomial of given degree - ``Poly.identity()`` -- ``p`` where ``p(x) = x`` for all ``x`` -- ``Poly.fit(x, y, deg)`` -- ``p`` of degree ``deg`` with coefficients +- ``Poly.fit(x, y, deg)`` -- ``p`` of degree ``deg`` with coefficients determined by the least-squares fit to the data ``x``, ``y`` - ``Poly.fromroots(roots)`` -- ``p`` with specified roots - ``p.copy()`` -- Create a copy of ``p`` @@ -121,6 +120,16 @@ from .hermite_e import HermiteE from .laguerre import Laguerre +__all__ = [ + "set_default_printstyle", + "polynomial", "Polynomial", + "chebyshev", "Chebyshev", + "legendre", "Legendre", + "hermite", "Hermite", + "hermite_e", "HermiteE", + "laguerre", "Laguerre", +] + def set_default_printstyle(style): """ diff --git a/numpy/polynomial/__init__.pyi b/numpy/polynomial/__init__.pyi index 817ba22ac83f..bebedb3a6990 100644 --- a/numpy/polynomial/__init__.pyi +++ b/numpy/polynomial/__init__.pyi @@ -1,9 +1,20 @@ -from typing import Any +from typing import List -Polynomial: Any -Chebyshev: Any -Legendre: Any -Hermite: Any -HermiteE: Any -Laguerre: Any -set_default_printstyle: Any +from numpy.polynomial import ( + chebyshev as chebyshev, + hermite as hermite, + hermite_e as hermite_e, + laguerre as laguerre, + legendre as legendre, + polynomial as polynomial, +) +from numpy.polynomial.chebyshev import Chebyshev as Chebyshev +from numpy.polynomial.hermite import Hermite as Hermite +from numpy.polynomial.hermite_e import HermiteE as HermiteE +from numpy.polynomial.laguerre import Laguerre as Laguerre +from numpy.polynomial.legendre import Legendre as Legendre +from numpy.polynomial.polynomial import Polynomial as Polynomial + +__all__: List[str] + +def set_default_printstyle(style): ... diff --git a/numpy/polynomial/_polybase.py b/numpy/polynomial/_polybase.py index 59c380f10e3a..b04b8e66b09c 100644 --- a/numpy/polynomial/_polybase.py +++ b/numpy/polynomial/_polybase.py @@ -694,7 +694,7 @@ def trim(self, tol=0): Returns ------- new_series : series - Contains the new set of coefficients. + New instance of series with trimmed coefficients. """ coef = pu.trimcoef(self.coef, tol) @@ -757,9 +757,6 @@ def convert(self, domain=None, kind=None, window=None): Conversion between domains and class types can result in numerically ill defined series. - Examples - -------- - """ if kind is None: kind = self.__class__ @@ -940,7 +937,7 @@ class domain in NumPy 1.4 and ``None`` in later versions. also returned. w : array_like, shape (M,), optional Weights. If not None the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. diff --git a/numpy/polynomial/_polybase.pyi b/numpy/polynomial/_polybase.pyi new file mode 100644 index 000000000000..c4160146947f --- /dev/null +++ b/numpy/polynomial/_polybase.pyi @@ -0,0 +1,69 @@ +import abc +from typing import Any, List, ClassVar + +__all__: List[str] + +class ABCPolyBase(abc.ABC): + __hash__: ClassVar[None] # type: ignore[assignment] + __array_ufunc__: ClassVar[None] + maxpower: ClassVar[int] + coef: Any + @property + @abc.abstractmethod + def domain(self): ... + @property + @abc.abstractmethod + def window(self): ... + @property + @abc.abstractmethod + def basis_name(self): ... + def has_samecoef(self, other): ... + def has_samedomain(self, other): ... + def has_samewindow(self, other): ... + def has_sametype(self, other): ... + def __init__(self, coef, domain=..., window=...): ... + def __format__(self, fmt_str): ... + def __call__(self, arg): ... + def __iter__(self): ... + def __len__(self): ... + def __neg__(self): ... + def __pos__(self): ... + def __add__(self, other): ... + def __sub__(self, other): ... + def __mul__(self, other): ... + def __truediv__(self, other): ... + def __floordiv__(self, other): ... + def __mod__(self, other): ... + def __divmod__(self, other): ... + def __pow__(self, other): ... + def __radd__(self, other): ... + def __rsub__(self, other): ... + def __rmul__(self, other): ... + def __rdiv__(self, other): ... + def __rtruediv__(self, other): ... + def __rfloordiv__(self, other): ... + def __rmod__(self, other): ... + def __rdivmod__(self, other): ... + def __eq__(self, other): ... + def __ne__(self, other): ... + def copy(self): ... + def degree(self): ... + def cutdeg(self, deg): ... + def trim(self, tol=...): ... + def truncate(self, size): ... + def convert(self, domain=..., kind=..., window=...): ... + def mapparms(self): ... + def integ(self, m=..., k = ..., lbnd=...): ... + def deriv(self, m=...): ... + def roots(self): ... + def linspace(self, n=..., domain=...): ... + @classmethod + def fit(cls, x, y, deg, domain=..., rcond=..., full=..., w=..., window=...): ... + @classmethod + def fromroots(cls, roots, domain = ..., window=...): ... + @classmethod + def identity(cls, domain=..., window=...): ... + @classmethod + def basis(cls, deg, domain=..., window=...): ... + @classmethod + def cast(cls, series, domain=..., window=...): ... diff --git a/numpy/polynomial/chebyshev.py b/numpy/polynomial/chebyshev.py index 6745c9371b57..d24fc738fcf4 100644 --- a/numpy/polynomial/chebyshev.py +++ b/numpy/polynomial/chebyshev.py @@ -1149,9 +1149,6 @@ def chebval(x, c, tensor=True): ----- The evaluation uses Clenshaw recursion, aka synthetic division. - Examples - -------- - """ c = np.array(c, ndmin=1, copy=True) if c.dtype.char in '?bBhHiIlLqQpP': @@ -1586,7 +1583,7 @@ def chebfit(x, y, deg, rcond=None, full=False, w=None): information from the singular value decomposition is also returned. w : array_like, shape (`M`,), optional Weights. If not None, the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. @@ -1952,8 +1949,8 @@ def chebpts1(npts): if _npts < 1: raise ValueError("npts must be >= 1") - x = np.linspace(-np.pi, 0, _npts, endpoint=False) + np.pi/(2*_npts) - return np.cos(x) + x = 0.5 * np.pi / _npts * np.arange(-_npts+1, _npts+1, 2) + return np.sin(x) def chebpts2(npts): diff --git a/numpy/polynomial/chebyshev.pyi b/numpy/polynomial/chebyshev.pyi new file mode 100644 index 000000000000..841c0859b1b0 --- /dev/null +++ b/numpy/polynomial/chebyshev.pyi @@ -0,0 +1,51 @@ +from typing import Any, List + +from numpy import ndarray, dtype, int_ +from numpy.polynomial._polybase import ABCPolyBase +from numpy.polynomial.polyutils import trimcoef + +__all__: List[str] + +chebtrim = trimcoef + +def poly2cheb(pol): ... +def cheb2poly(c): ... + +chebdomain: ndarray[Any, dtype[int_]] +chebzero: ndarray[Any, dtype[int_]] +chebone: ndarray[Any, dtype[int_]] +chebx: ndarray[Any, dtype[int_]] + +def chebline(off, scl): ... +def chebfromroots(roots): ... +def chebadd(c1, c2): ... +def chebsub(c1, c2): ... +def chebmulx(c): ... +def chebmul(c1, c2): ... +def chebdiv(c1, c2): ... +def chebpow(c, pow, maxpower=...): ... +def chebder(c, m=..., scl=..., axis=...): ... +def chebint(c, m=..., k = ..., lbnd=..., scl=..., axis=...): ... +def chebval(x, c, tensor=...): ... +def chebval2d(x, y, c): ... +def chebgrid2d(x, y, c): ... +def chebval3d(x, y, z, c): ... +def chebgrid3d(x, y, z, c): ... +def chebvander(x, deg): ... +def chebvander2d(x, y, deg): ... +def chebvander3d(x, y, z, deg): ... +def chebfit(x, y, deg, rcond=..., full=..., w=...): ... +def chebcompanion(c): ... +def chebroots(c): ... +def chebinterpolate(func, deg, args = ...): ... +def chebgauss(deg): ... +def chebweight(x): ... +def chebpts1(npts): ... +def chebpts2(npts): ... + +class Chebyshev(ABCPolyBase): + @classmethod + def interpolate(cls, func, deg, domain=..., args = ...): ... + domain: Any + window: Any + basis_name: Any diff --git a/numpy/polynomial/hermite.py b/numpy/polynomial/hermite.py index c679c529880f..eef5c25b225e 100644 --- a/numpy/polynomial/hermite.py +++ b/numpy/polynomial/hermite.py @@ -1311,7 +1311,7 @@ def hermfit(x, y, deg, rcond=None, full=False, w=None): information from the singular value decomposition is also returned. w : array_like, shape (`M`,), optional Weights. If not None, the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. diff --git a/numpy/polynomial/hermite.pyi b/numpy/polynomial/hermite.pyi new file mode 100644 index 000000000000..8364a5b0fcbc --- /dev/null +++ b/numpy/polynomial/hermite.pyi @@ -0,0 +1,46 @@ +from typing import Any, List + +from numpy import ndarray, dtype, int_, float_ +from numpy.polynomial._polybase import ABCPolyBase +from numpy.polynomial.polyutils import trimcoef + +__all__: list[str] + +hermtrim = trimcoef + +def poly2herm(pol): ... +def herm2poly(c): ... + +hermdomain: ndarray[Any, dtype[int_]] +hermzero: ndarray[Any, dtype[int_]] +hermone: ndarray[Any, dtype[int_]] +hermx: ndarray[Any, dtype[float_]] + +def hermline(off, scl): ... +def hermfromroots(roots): ... +def hermadd(c1, c2): ... +def hermsub(c1, c2): ... +def hermmulx(c): ... +def hermmul(c1, c2): ... +def hermdiv(c1, c2): ... +def hermpow(c, pow, maxpower=...): ... +def hermder(c, m=..., scl=..., axis=...): ... +def hermint(c, m=..., k = ..., lbnd=..., scl=..., axis=...): ... +def hermval(x, c, tensor=...): ... +def hermval2d(x, y, c): ... +def hermgrid2d(x, y, c): ... +def hermval3d(x, y, z, c): ... +def hermgrid3d(x, y, z, c): ... +def hermvander(x, deg): ... +def hermvander2d(x, y, deg): ... +def hermvander3d(x, y, z, deg): ... +def hermfit(x, y, deg, rcond=..., full=..., w=...): ... +def hermcompanion(c): ... +def hermroots(c): ... +def hermgauss(deg): ... +def hermweight(x): ... + +class Hermite(ABCPolyBase): + domain: Any + window: Any + basis_name: Any diff --git a/numpy/polynomial/hermite_e.py b/numpy/polynomial/hermite_e.py index 1ce8ebe04ca2..05d1337b0b33 100644 --- a/numpy/polynomial/hermite_e.py +++ b/numpy/polynomial/hermite_e.py @@ -1302,7 +1302,7 @@ def hermefit(x, y, deg, rcond=None, full=False, w=None): information from the singular value decomposition is also returned. w : array_like, shape (`M`,), optional Weights. If not None, the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. diff --git a/numpy/polynomial/hermite_e.pyi b/numpy/polynomial/hermite_e.pyi new file mode 100644 index 000000000000..c029bfda7788 --- /dev/null +++ b/numpy/polynomial/hermite_e.pyi @@ -0,0 +1,46 @@ +from typing import Any, List + +from numpy import ndarray, dtype, int_ +from numpy.polynomial._polybase import ABCPolyBase +from numpy.polynomial.polyutils import trimcoef + +__all__: list[str] + +hermetrim = trimcoef + +def poly2herme(pol): ... +def herme2poly(c): ... + +hermedomain: ndarray[Any, dtype[int_]] +hermezero: ndarray[Any, dtype[int_]] +hermeone: ndarray[Any, dtype[int_]] +hermex: ndarray[Any, dtype[int_]] + +def hermeline(off, scl): ... +def hermefromroots(roots): ... +def hermeadd(c1, c2): ... +def hermesub(c1, c2): ... +def hermemulx(c): ... +def hermemul(c1, c2): ... +def hermediv(c1, c2): ... +def hermepow(c, pow, maxpower=...): ... +def hermeder(c, m=..., scl=..., axis=...): ... +def hermeint(c, m=..., k = ..., lbnd=..., scl=..., axis=...): ... +def hermeval(x, c, tensor=...): ... +def hermeval2d(x, y, c): ... +def hermegrid2d(x, y, c): ... +def hermeval3d(x, y, z, c): ... +def hermegrid3d(x, y, z, c): ... +def hermevander(x, deg): ... +def hermevander2d(x, y, deg): ... +def hermevander3d(x, y, z, deg): ... +def hermefit(x, y, deg, rcond=..., full=..., w=...): ... +def hermecompanion(c): ... +def hermeroots(c): ... +def hermegauss(deg): ... +def hermeweight(x): ... + +class HermiteE(ABCPolyBase): + domain: Any + window: Any + basis_name: Any diff --git a/numpy/polynomial/laguerre.py b/numpy/polynomial/laguerre.py index 9cff0b71ca68..69d55751087d 100644 --- a/numpy/polynomial/laguerre.py +++ b/numpy/polynomial/laguerre.py @@ -1308,7 +1308,7 @@ def lagfit(x, y, deg, rcond=None, full=False, w=None): information from the singular value decomposition is also returned. w : array_like, shape (`M`,), optional Weights. If not None, the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. diff --git a/numpy/polynomial/laguerre.pyi b/numpy/polynomial/laguerre.pyi new file mode 100644 index 000000000000..2b9ab34e0afa --- /dev/null +++ b/numpy/polynomial/laguerre.pyi @@ -0,0 +1,46 @@ +from typing import Any, List + +from numpy import ndarray, dtype, int_ +from numpy.polynomial._polybase import ABCPolyBase +from numpy.polynomial.polyutils import trimcoef + +__all__: list[str] + +lagtrim = trimcoef + +def poly2lag(pol): ... +def lag2poly(c): ... + +lagdomain: ndarray[Any, dtype[int_]] +lagzero: ndarray[Any, dtype[int_]] +lagone: ndarray[Any, dtype[int_]] +lagx: ndarray[Any, dtype[int_]] + +def lagline(off, scl): ... +def lagfromroots(roots): ... +def lagadd(c1, c2): ... +def lagsub(c1, c2): ... +def lagmulx(c): ... +def lagmul(c1, c2): ... +def lagdiv(c1, c2): ... +def lagpow(c, pow, maxpower=...): ... +def lagder(c, m=..., scl=..., axis=...): ... +def lagint(c, m=..., k = ..., lbnd=..., scl=..., axis=...): ... +def lagval(x, c, tensor=...): ... +def lagval2d(x, y, c): ... +def laggrid2d(x, y, c): ... +def lagval3d(x, y, z, c): ... +def laggrid3d(x, y, z, c): ... +def lagvander(x, deg): ... +def lagvander2d(x, y, deg): ... +def lagvander3d(x, y, z, deg): ... +def lagfit(x, y, deg, rcond=..., full=..., w=...): ... +def lagcompanion(c): ... +def lagroots(c): ... +def laggauss(deg): ... +def lagweight(x): ... + +class Laguerre(ABCPolyBase): + domain: Any + window: Any + basis_name: Any diff --git a/numpy/polynomial/legendre.py b/numpy/polynomial/legendre.py index 427f9f82f381..cd4da2a79e75 100644 --- a/numpy/polynomial/legendre.py +++ b/numpy/polynomial/legendre.py @@ -605,9 +605,6 @@ def legpow(c, pow, maxpower=16): -------- legadd, legsub, legmulx, legmul, legdiv - Examples - -------- - """ return pu._pow(legmul, c, pow, maxpower) @@ -890,9 +887,6 @@ def legval(x, c, tensor=True): ----- The evaluation uses Clenshaw recursion, aka synthetic division. - Examples - -------- - """ c = np.array(c, ndmin=1, copy=False) if c.dtype.char in '?bBhHiIlLqQpP': @@ -1328,7 +1322,7 @@ def legfit(x, y, deg, rcond=None, full=False, w=None): information from the singular value decomposition is also returned. w : array_like, shape (`M`,), optional Weights. If not None, the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. diff --git a/numpy/polynomial/legendre.pyi b/numpy/polynomial/legendre.pyi new file mode 100644 index 000000000000..86aef179304e --- /dev/null +++ b/numpy/polynomial/legendre.pyi @@ -0,0 +1,46 @@ +from typing import Any, List + +from numpy import ndarray, dtype, int_ +from numpy.polynomial._polybase import ABCPolyBase +from numpy.polynomial.polyutils import trimcoef + +__all__: list[str] + +legtrim = trimcoef + +def poly2leg(pol): ... +def leg2poly(c): ... + +legdomain: ndarray[Any, dtype[int_]] +legzero: ndarray[Any, dtype[int_]] +legone: ndarray[Any, dtype[int_]] +legx: ndarray[Any, dtype[int_]] + +def legline(off, scl): ... +def legfromroots(roots): ... +def legadd(c1, c2): ... +def legsub(c1, c2): ... +def legmulx(c): ... +def legmul(c1, c2): ... +def legdiv(c1, c2): ... +def legpow(c, pow, maxpower=...): ... +def legder(c, m=..., scl=..., axis=...): ... +def legint(c, m=..., k = ..., lbnd=..., scl=..., axis=...): ... +def legval(x, c, tensor=...): ... +def legval2d(x, y, c): ... +def leggrid2d(x, y, c): ... +def legval3d(x, y, z, c): ... +def leggrid3d(x, y, z, c): ... +def legvander(x, deg): ... +def legvander2d(x, y, deg): ... +def legvander3d(x, y, z, deg): ... +def legfit(x, y, deg, rcond=..., full=..., w=...): ... +def legcompanion(c): ... +def legroots(c): ... +def leggauss(deg): ... +def legweight(x): ... + +class Legendre(ABCPolyBase): + domain: Any + window: Any + basis_name: Any diff --git a/numpy/polynomial/polynomial.py b/numpy/polynomial/polynomial.py index 1baa7d870881..940eed5e38d5 100644 --- a/numpy/polynomial/polynomial.py +++ b/numpy/polynomial/polynomial.py @@ -156,7 +156,7 @@ def polyfromroots(roots): .. math:: p(x) = (x - r_0) * (x - r_1) * ... * (x - r_n), - where the `r_n` are the roots specified in `roots`. If a zero has + where the ``r_n`` are the roots specified in `roots`. If a zero has multiplicity n, then it must appear in `roots` n times. For instance, if 2 is a root of multiplicity three and 3 is a root of multiplicity 2, then `roots` looks something like [2, 2, 2, 3, 3]. The roots can appear @@ -192,11 +192,11 @@ def polyfromroots(roots): Notes ----- The coefficients are determined by multiplying together linear factors - of the form `(x - r_i)`, i.e. + of the form ``(x - r_i)``, i.e. .. math:: p(x) = (x - r_0) (x - r_1) ... (x - r_n) - where ``n == len(roots) - 1``; note that this implies that `1` is always + where ``n == len(roots) - 1``; note that this implies that ``1`` is always returned for :math:`a_n`. Examples @@ -1253,7 +1253,7 @@ def polyfit(x, y, deg, rcond=None, full=False, w=None): to solve the fit's matrix equation) is also returned. w : array_like, shape (`M`,), optional Weights. If not None, the contribution of each point - ``(x[i],y[i])`` to the fit is weighted by `w[i]`. Ideally the + ``(x[i],y[i])`` to the fit is weighted by ``w[i]``. Ideally the weights are chosen so that the errors of the products ``w[i]*y[i]`` all have the same variance. The default value is None. diff --git a/numpy/polynomial/polynomial.pyi b/numpy/polynomial/polynomial.pyi new file mode 100644 index 000000000000..f779300a9c5a --- /dev/null +++ b/numpy/polynomial/polynomial.pyi @@ -0,0 +1,41 @@ +from typing import Any, List + +from numpy import ndarray, dtype, int_ +from numpy.polynomial._polybase import ABCPolyBase +from numpy.polynomial.polyutils import trimcoef + +__all__: list[str] + +polytrim = trimcoef + +polydomain: ndarray[Any, dtype[int_]] +polyzero: ndarray[Any, dtype[int_]] +polyone: ndarray[Any, dtype[int_]] +polyx: ndarray[Any, dtype[int_]] + +def polyline(off, scl): ... +def polyfromroots(roots): ... +def polyadd(c1, c2): ... +def polysub(c1, c2): ... +def polymulx(c): ... +def polymul(c1, c2): ... +def polydiv(c1, c2): ... +def polypow(c, pow, maxpower=...): ... +def polyder(c, m=..., scl=..., axis=...): ... +def polyint(c, m=..., k=..., lbnd=..., scl=..., axis=...): ... +def polyval(x, c, tensor=...): ... +def polyvalfromroots(x, r, tensor=...): ... +def polyval2d(x, y, c): ... +def polygrid2d(x, y, c): ... +def polyval3d(x, y, z, c): ... +def polygrid3d(x, y, z, c): ... +def polyvander(x, deg): ... +def polyvander2d(x, y, deg): ... +def polyvander3d(x, y, z, deg): ... +def polyfit(x, y, deg, rcond=..., full=..., w=...): ... +def polyroots(c): ... + +class Polynomial(ABCPolyBase): + domain: Any + window: Any + basis_name: Any diff --git a/numpy/polynomial/polyutils.py b/numpy/polynomial/polyutils.py index d81ee975413c..3b0f0a9e57ee 100644 --- a/numpy/polynomial/polyutils.py +++ b/numpy/polynomial/polyutils.py @@ -4,15 +4,6 @@ This module provides: error and warning objects; a polynomial base class; and some routines used in both the `polynomial` and `chebyshev` modules. -Error objects -------------- - -.. autosummary:: - :toctree: generated/ - - PolyError base class for this sub-package's errors. - PolyDomainError raised when domains are mismatched. - Warning objects --------------- @@ -21,14 +12,6 @@ RankWarning raised in least-squares fit for rank-deficient matrix. -Base class ----------- - -.. autosummary:: - :toctree: generated/ - - PolyBase Obsolete base class for the polynomial classes. Do not use. - Functions --------- @@ -50,8 +33,8 @@ import numpy as np __all__ = [ - 'RankWarning', 'PolyError', 'PolyDomainError', 'as_series', 'trimseq', - 'trimcoef', 'getdomain', 'mapdomain', 'mapparms', 'PolyBase'] + 'RankWarning', 'as_series', 'trimseq', + 'trimcoef', 'getdomain', 'mapdomain', 'mapparms'] # # Warnings and Exceptions @@ -61,35 +44,6 @@ class RankWarning(UserWarning): """Issued by chebfit when the design matrix is rank deficient.""" pass -class PolyError(Exception): - """Base class for errors in this module.""" - pass - -class PolyDomainError(PolyError): - """Issued by the generic Poly class when two domains don't match. - - This is raised when an binary operation is passed Poly objects with - different domains. - - """ - pass - -# -# Base class for all polynomial types -# - -class PolyBase: - """ - Base class for all polynomial types. - - Deprecated in numpy 1.9.0, use the abstract - ABCPolyBase class instead. Note that the latter - requires a number of virtual functions to be - implemented. - - """ - pass - # # Helper functions to convert inputs to 1-D arrays # @@ -509,7 +463,7 @@ def _fromroots(line_f, mul_f, roots): The ``line`` function, such as ``polyline`` mul_f : function(array_like, array_like) -> ndarray The ``mul`` function, such as ``polymul`` - roots : + roots See the ``fromroots`` functions for more detail """ if len(roots) == 0: @@ -537,7 +491,7 @@ def _valnd(val_f, c, *args): ---------- val_f : function(array_like, array_like, tensor: bool) -> array_like The ``val`` function, such as ``polyval`` - c, args : + c, args See the ``vald`` functions for more detail """ args = [np.asanyarray(a) for a in args] @@ -567,7 +521,7 @@ def _gridnd(val_f, c, *args): ---------- val_f : function(array_like, array_like, tensor: bool) -> array_like The ``val`` function, such as ``polyval`` - c, args : + c, args See the ``gridd`` functions for more detail """ for xi in args: @@ -586,7 +540,7 @@ def _div(mul_f, c1, c2): ---------- mul_f : function(array_like, array_like) -> array_like The ``mul`` function, such as ``polymul`` - c1, c2 : + c1, c2 See the ``div`` functions for more detail """ # c1, c2 are trimmed copies @@ -646,7 +600,7 @@ def _fit(vander_f, x, y, deg, rcond=None, full=False, w=None): ---------- vander_f : function(array_like, int) -> ndarray The 1d vander function, such as ``polyvander`` - c1, c2 : + c1, c2 See the ``fit`` functions for more detail """ x = np.asarray(x) + 0.0 @@ -732,12 +686,12 @@ def _pow(mul_f, c, pow, maxpower): Parameters ---------- - vander_f : function(array_like, int) -> ndarray - The 1d vander function, such as ``polyvander`` - pow, maxpower : - See the ``pow`` functions for more detail mul_f : function(array_like, array_like) -> ndarray The ``mul`` function, such as ``polymul`` + c : array_like + 1-D array of array of series coefficients + pow, maxpower + See the ``pow`` functions for more detail """ # c is a trimmed copy [c] = as_series([c]) diff --git a/numpy/polynomial/polyutils.pyi b/numpy/polynomial/polyutils.pyi new file mode 100644 index 000000000000..52c9cfc4a607 --- /dev/null +++ b/numpy/polynomial/polyutils.pyi @@ -0,0 +1,12 @@ +from typing import List + +__all__: List[str] + +class RankWarning(UserWarning): ... + +def trimseq(seq): ... +def as_series(alist, trim=...): ... +def trimcoef(c, tol=...): ... +def getdomain(x): ... +def mapparms(old, new): ... +def mapdomain(x, old, new): ... diff --git a/numpy/polynomial/tests/test_legendre.py b/numpy/polynomial/tests/test_legendre.py index a2a212c242e3..92399c160ecb 100644 --- a/numpy/polynomial/tests/test_legendre.py +++ b/numpy/polynomial/tests/test_legendre.py @@ -305,6 +305,9 @@ def test_legint_axis(self): res = leg.legint(c2d, k=3, axis=1) assert_almost_equal(res, tgt) + def test_legint_zerointord(self): + assert_equal(leg.legint((1, 2, 3), 0), (1, 2, 3)) + class TestDerivative: @@ -345,6 +348,9 @@ def test_legder_axis(self): res = leg.legder(c2d, axis=1) assert_almost_equal(res, tgt) + def test_legder_orderhigherthancoeff(self): + c = (1, 2, 3, 4) + assert_equal(leg.legder(c, 4), [0]) class TestVander: # some random values in [-1, 1) @@ -393,6 +399,9 @@ def test_legvander3d(self): van = leg.legvander3d([x1], [x2], [x3], [1, 2, 3]) assert_(van.shape == (1, 5, 24)) + def test_legvander_negdeg(self): + assert_raises(ValueError, leg.legvander, (1, 2, 3), -1) + class TestFitting: @@ -541,6 +550,9 @@ def test_legtrim(self): def test_legline(self): assert_equal(leg.legline(3, 4), [3, 4]) + def test_legline_zeroscl(self): + assert_equal(leg.legline(3, 0), [3]) + def test_leg2poly(self): for i in range(10): assert_almost_equal(leg.leg2poly([0]*i + [1]), Llist[i]) diff --git a/numpy/polynomial/tests/test_polynomial.py b/numpy/polynomial/tests/test_polynomial.py index 5fd1a82a2faf..a0a09fcf4a93 100644 --- a/numpy/polynomial/tests/test_polynomial.py +++ b/numpy/polynomial/tests/test_polynomial.py @@ -473,6 +473,10 @@ def test_polyvander3d(self): van = poly.polyvander3d([x1], [x2], [x3], [1, 2, 3]) assert_(van.shape == (1, 5, 24)) + def test_polyvandernegdeg(self): + x = np.arange(3) + assert_raises(ValueError, poly.polyvander, x, -1) + class TestCompanion: @@ -591,3 +595,6 @@ def test_polytrim(self): def test_polyline(self): assert_equal(poly.polyline(3, 4), [3, 4]) + + def test_polyline_zero(self): + assert_equal(poly.polyline(3, 0), [3]) diff --git a/numpy/polynomial/tests/test_polyutils.py b/numpy/polynomial/tests/test_polyutils.py index 1b27f53b5988..cc630790da1c 100644 --- a/numpy/polynomial/tests/test_polyutils.py +++ b/numpy/polynomial/tests/test_polyutils.py @@ -40,6 +40,21 @@ def test_trimcoef(self): assert_equal(pu.trimcoef(coef, 1), coef[:-3]) assert_equal(pu.trimcoef(coef, 2), [0]) + def test_vander_nd_exception(self): + # n_dims != len(points) + assert_raises(ValueError, pu._vander_nd, (), (1, 2, 3), [90]) + # n_dims != len(degrees) + assert_raises(ValueError, pu._vander_nd, (), (), [90.65]) + # n_dims == 0 + assert_raises(ValueError, pu._vander_nd, (), (), []) + + def test_div_zerodiv(self): + # c2[-1] == 0 + assert_raises(ZeroDivisionError, pu._div, pu._div, (1, 2, 3), [0]) + + def test_pow_too_large(self): + # power > maxpower + assert_raises(ValueError, pu._pow, (), [1, 2, 3], 5, 4) class TestDomain: diff --git a/numpy/polynomial/tests/test_printing.py b/numpy/polynomial/tests/test_printing.py index 27a8ab9f20b2..4e9902a69588 100644 --- a/numpy/polynomial/tests/test_printing.py +++ b/numpy/polynomial/tests/test_printing.py @@ -85,7 +85,7 @@ def test_laguerre_str(self, inp, tgt): class TestStrAscii: @pytest.fixture(scope='class', autouse=True) - def use_unicode(self): + def use_ascii(self): poly.set_default_printstyle('ascii') @pytest.mark.parametrize(('inp', 'tgt'), ( @@ -161,7 +161,10 @@ def test_laguerre_str(self, inp, tgt): class TestLinebreaking: - poly.set_default_printstyle('ascii') + + @pytest.fixture(scope='class', autouse=True) + def use_ascii(self): + poly.set_default_printstyle('ascii') def test_single_line_one_less(self): # With 'ascii' style, len(str(p)) is default linewidth - 1 (i.e. 74) @@ -283,19 +286,19 @@ def test_nonnumeric_object_coefficients(coefs, tgt): class TestFormat: def test_format_unicode(self): - poly.Polynomial._use_unicode = False + poly.set_default_printstyle('ascii') p = poly.Polynomial([1, 2, 0, -1]) assert_equal(format(p, 'unicode'), "1.0 + 2.0·x¹ + 0.0·x² - 1.0·x³") def test_format_ascii(self): - poly.Polynomial._use_unicode = True + poly.set_default_printstyle('unicode') p = poly.Polynomial([1, 2, 0, -1]) assert_equal( format(p, 'ascii'), "1.0 + 2.0 x**1 + 0.0 x**2 - 1.0 x**3" ) def test_empty_formatstr(self): - poly.Polynomial._use_unicode = False + poly.set_default_printstyle('ascii') p = poly.Polynomial([1, 2, 3]) assert_equal(format(p), "1.0 + 2.0 x**1 + 3.0 x**2") assert_equal(f"{p}", "1.0 + 2.0 x**1 + 3.0 x**2") diff --git a/numpy/random/__init__.py b/numpy/random/__init__.py index 7efa5c07fcfe..2e8f99fe3045 100644 --- a/numpy/random/__init__.py +++ b/numpy/random/__init__.py @@ -17,6 +17,7 @@ --------------------------------------------- --- MT19937 PCG64 +PCG64DXSM Philox SFC64 ============================================= === @@ -183,13 +184,14 @@ from ._generator import Generator, default_rng from .bit_generator import SeedSequence, BitGenerator from ._mt19937 import MT19937 -from ._pcg64 import PCG64 +from ._pcg64 import PCG64, PCG64DXSM from ._philox import Philox from ._sfc64 import SFC64 from .mtrand import * __all__ += ['Generator', 'RandomState', 'SeedSequence', 'MT19937', - 'Philox', 'PCG64', 'SFC64', 'default_rng', 'BitGenerator'] + 'Philox', 'PCG64', 'PCG64DXSM', 'SFC64', 'default_rng', + 'BitGenerator'] def __RandomState_ctor(): diff --git a/numpy/random/__init__.pyi b/numpy/random/__init__.pyi index f7c3cfafe2cf..48b620c48b92 100644 --- a/numpy/random/__init__.pyi +++ b/numpy/random/__init__.pyi @@ -1,61 +1,68 @@ -from typing import Any +from typing import List -beta: Any -binomial: Any -bytes: Any -chisquare: Any -choice: Any -dirichlet: Any -exponential: Any -f: Any -gamma: Any -geometric: Any -get_state: Any -gumbel: Any -hypergeometric: Any -laplace: Any -logistic: Any -lognormal: Any -logseries: Any -multinomial: Any -multivariate_normal: Any -negative_binomial: Any -noncentral_chisquare: Any -noncentral_f: Any -normal: Any -pareto: Any -permutation: Any -poisson: Any -power: Any -rand: Any -randint: Any -randn: Any -random: Any -random_integers: Any -random_sample: Any -ranf: Any -rayleigh: Any -sample: Any -seed: Any -set_state: Any -shuffle: Any -standard_cauchy: Any -standard_exponential: Any -standard_gamma: Any -standard_normal: Any -standard_t: Any -triangular: Any -uniform: Any -vonmises: Any -wald: Any -weibull: Any -zipf: Any -Generator: Any -RandomState: Any -SeedSequence: Any -MT19937: Any -Philox: Any -PCG64: Any -SFC64: Any -default_rng: Any -BitGenerator: Any +from numpy.random._generator import Generator as Generator +from numpy.random._generator import default_rng as default_rng +from numpy.random._mt19937 import MT19937 as MT19937 +from numpy.random._pcg64 import ( + PCG64 as PCG64, + PCG64DXSM as PCG64DXSM, +) +from numpy.random._philox import Philox as Philox +from numpy.random._sfc64 import SFC64 as SFC64 +from numpy.random.bit_generator import BitGenerator as BitGenerator +from numpy.random.bit_generator import SeedSequence as SeedSequence +from numpy.random.mtrand import ( + RandomState as RandomState, + beta as beta, + binomial as binomial, + bytes as bytes, + chisquare as chisquare, + choice as choice, + dirichlet as dirichlet, + exponential as exponential, + f as f, + gamma as gamma, + geometric as geometric, + get_state as get_state, + gumbel as gumbel, + hypergeometric as hypergeometric, + laplace as laplace, + logistic as logistic, + lognormal as lognormal, + logseries as logseries, + multinomial as multinomial, + multivariate_normal as multivariate_normal, + negative_binomial as negative_binomial, + noncentral_chisquare as noncentral_chisquare, + noncentral_f as noncentral_f, + normal as normal, + pareto as pareto, + permutation as permutation, + poisson as poisson, + power as power, + rand as rand, + randint as randint, + randn as randn, + random as random, + random_integers as random_integers, + random_sample as random_sample, + ranf as ranf, + rayleigh as rayleigh, + sample as sample, + seed as seed, + set_state as set_state, + shuffle as shuffle, + standard_cauchy as standard_cauchy, + standard_exponential as standard_exponential, + standard_gamma as standard_gamma, + standard_normal as standard_normal, + standard_t as standard_t, + triangular as triangular, + uniform as uniform, + vonmises as vonmises, + wald as wald, + weibull as weibull, + zipf as zipf, +) + +__all__: List[str] diff --git a/numpy/random/_bounded_integers.pyx.in b/numpy/random/_bounded_integers.pyx.in index 9f46685d3258..7eb6aff1e9f1 100644 --- a/numpy/random/_bounded_integers.pyx.in +++ b/numpy/random/_bounded_integers.pyx.in @@ -8,6 +8,7 @@ __all__ = [] np.import_array() + cdef extern from "numpy/random/distributions.h": # Generate random numbers in closed interval [off, off + rng]. uint64_t random_bounded_uint64(bitgen_t *bitgen_state, @@ -51,6 +52,17 @@ cdef extern from "numpy/random/distributions.h": np.npy_bool *out) nogil +cdef object format_bounds_error(bint closed, object low): + # Special case low == 0 to provide a better exception for users + # since low = 0 is the default single-argument case. + if not np.any(low): + comp = '<' if closed else '<=' + return f'high {comp} 0' + else: + comp = '>' if closed else '>=' + return f'low {comp} high' + + {{ py: type_info = (('uint32', 'uint32', 'uint64', 'NPY_UINT64', 0, 0, 0, '0X100000000ULL'), @@ -99,8 +111,7 @@ cdef object _rand_{{nptype}}_broadcast(np.ndarray low, np.ndarray high, object s if np.any(high_comp(high_arr, {{ub}})): raise ValueError('high is out of bounds for {{nptype}}') if np.any(low_high_comp(low_arr, high_arr)): - comp = '>' if closed else '>=' - raise ValueError('low {comp} high'.format(comp=comp)) + raise ValueError(format_bounds_error(closed, low_arr)) low_arr = np.PyArray_FROM_OTF(low, np.{{npctype}}, np.NPY_ALIGNED | np.NPY_FORCECAST) high_arr = np.PyArray_FROM_OTF(high, np.{{npctype}}, np.NPY_ALIGNED | np.NPY_FORCECAST) @@ -173,8 +184,7 @@ cdef object _rand_{{nptype}}_broadcast(object low, object high, object size, # Avoid object dtype path if already an integer high_lower_comp = np.less if closed else np.less_equal if np.any(high_lower_comp(high_arr, {{lb}})): - comp = '>' if closed else '>=' - raise ValueError('low {comp} high'.format(comp=comp)) + raise ValueError(format_bounds_error(closed, low_arr)) high_m1 = high_arr if closed else high_arr - dt.type(1) if np.any(np.greater(high_m1, {{ub}})): raise ValueError('high is out of bounds for {{nptype}}') @@ -191,13 +201,11 @@ cdef object _rand_{{nptype}}_broadcast(object low, object high, object size, if closed_upper > {{ub}}: raise ValueError('high is out of bounds for {{nptype}}') if closed_upper < {{lb}}: - comp = '>' if closed else '>=' - raise ValueError('low {comp} high'.format(comp=comp)) + raise ValueError(format_bounds_error(closed, low_arr)) highm1_data[i] = <{{nptype}}_t>closed_upper if np.any(np.greater(low_arr, highm1_arr)): - comp = '>' if closed else '>=' - raise ValueError('low {comp} high'.format(comp=comp)) + raise ValueError(format_bounds_error(closed, low_arr)) high_arr = highm1_arr low_arr = np.PyArray_FROM_OTF(low, np.{{npctype}}, np.NPY_ALIGNED | np.NPY_FORCECAST) @@ -316,8 +324,7 @@ cdef object _rand_{{nptype}}(object low, object high, object size, if high > {{ub}}: raise ValueError("high is out of bounds for {{nptype}}") if low > high: # -1 already subtracted, closed interval - comp = '>' if closed else '>=' - raise ValueError('low {comp} high'.format(comp=comp)) + raise ValueError(format_bounds_error(closed, low)) rng = <{{utype}}_t>(high - low) off = <{{utype}}_t>(<{{nptype}}_t>low) diff --git a/numpy/random/_common.pyx b/numpy/random/_common.pyx index 6d77aed03daa..ad43f281226a 100644 --- a/numpy/random/_common.pyx +++ b/numpy/random/_common.pyx @@ -25,11 +25,11 @@ cdef uint64_t MAXSIZE = sys.maxsize cdef object benchmark(bitgen_t *bitgen, object lock, Py_ssize_t cnt, object method): """Benchmark command used by BitGenerator""" cdef Py_ssize_t i - if method==u'uint64': + if method=='uint64': with lock, nogil: for i in range(cnt): bitgen.next_uint64(bitgen.state) - elif method==u'double': + elif method=='double': with lock, nogil: for i in range(cnt): bitgen.next_double(bitgen.state) @@ -121,8 +121,8 @@ cdef object prepare_cffi(bitgen_t *bitgen): """ try: import cffi - except ImportError: - raise ImportError('cffi cannot be imported.') + except ImportError as e: + raise ImportError('cffi cannot be imported.') from e ffi = cffi.FFI() _cffi = interface(bitgen.state, @@ -219,8 +219,8 @@ cdef np.ndarray int_to_array(object value, object name, object bits, object uint cdef validate_output_shape(iter_shape, np.ndarray output): - cdef np.npy_intp *shape - cdef ndim, i + cdef np.npy_intp *dims + cdef np.npy_intp ndim, i cdef bint error dims = np.PyArray_DIMS(output) ndim = np.PyArray_NDIM(output) @@ -232,13 +232,34 @@ cdef validate_output_shape(iter_shape, np.ndarray output): ) -cdef check_output(object out, object dtype, object size): +cdef check_output(object out, object dtype, object size, bint require_c_array): + """ + Check user-supplied output array properties and shape + + Parameters + ---------- + out : {ndarray, None} + The array to check. If None, returns immediately. + dtype : dtype + The required dtype of out. + size : {None, int, tuple[int]} + The size passed. If out is an ndarray, verifies that the shape of out + matches size. + require_c_array : bool + Whether out must be a C-array. If False, out can be either C- or F- + ordered. If True, must be C-ordered. In either case, must be + contiguous, writable, aligned and in native byte-order. + """ if out is None: return cdef np.ndarray out_array = out - if not (np.PyArray_CHKFLAGS(out_array, np.NPY_CARRAY) or - np.PyArray_CHKFLAGS(out_array, np.NPY_FARRAY)): - raise ValueError('Supplied output array is not contiguous, writable or aligned.') + if not (np.PyArray_ISCARRAY(out_array) or + (np.PyArray_ISFARRAY(out_array) and not require_c_array)): + req = "C-" if require_c_array else "" + raise ValueError( + f'Supplied output array must be {req}contiguous, writable, ' + f'aligned, and in machine byte-order.' + ) if out_array.dtype != dtype: raise TypeError('Supplied output array has the wrong type. ' 'Expected {0}, got {1}'.format(np.dtype(dtype), out_array.dtype)) @@ -264,7 +285,7 @@ cdef object double_fill(void *func, bitgen_t *state, object size, object lock, o return out_val if out is not None: - check_output(out, np.float64, size) + check_output(out, np.float64, size, False) out_array = out else: out_array = np.empty(size, np.double) @@ -288,7 +309,7 @@ cdef object float_fill(void *func, bitgen_t *state, object size, object lock, ob return out_val if out is not None: - check_output(out, np.float32, size) + check_output(out, np.float32, size, False) out_array = out else: out_array = np.empty(size, np.float32) @@ -310,7 +331,7 @@ cdef object float_fill_from_double(void *func, bitgen_t *state, object size, obj return random_func(state) if out is not None: - check_output(out, np.float32, size) + check_output(out, np.float32, size, False) out_array = out else: out_array = np.empty(size, np.float32) @@ -521,7 +542,7 @@ cdef object cont(void *func, void *state, object size, object lock, int narg, cdef np.ndarray a_arr, b_arr, c_arr cdef double _a = 0.0, _b = 0.0, _c = 0.0 cdef bint is_scalar = True - check_output(out, np.float64, size) + check_output(out, np.float64, size, narg > 0) if narg > 0: a_arr = np.PyArray_FROM_OTF(a, np.NPY_DOUBLE, np.NPY_ALIGNED) is_scalar = is_scalar and np.PyArray_NDIM(a_arr) == 0 @@ -971,7 +992,7 @@ cdef object cont_f(void *func, bitgen_t *state, object size, object lock, cdef float _a cdef bint is_scalar = True cdef int requirements = np.NPY_ALIGNED | np.NPY_FORCECAST - check_output(out, np.float32, size) + check_output(out, np.float32, size, True) a_arr = np.PyArray_FROMANY(a, np.NPY_FLOAT32, 0, 0, requirements) is_scalar = np.PyArray_NDIM(a_arr) == 0 diff --git a/numpy/random/_examples/cffi/parse.py b/numpy/random/_examples/cffi/parse.py index 73d8646c7c4a..daff6bdece01 100644 --- a/numpy/random/_examples/cffi/parse.py +++ b/numpy/random/_examples/cffi/parse.py @@ -17,11 +17,20 @@ def parse_distributions_h(ffi, inc_dir): continue s.append(line) ffi.cdef('\n'.join(s)) - + with open(os.path.join(inc_dir, 'random', 'distributions.h')) as fid: s = [] in_skip = 0 + ignoring = False for line in fid: + # check for and remove extern "C" guards + if ignoring: + if line.strip().startswith('#endif'): + ignoring = False + continue + if line.strip().startswith('#ifdef __cplusplus'): + ignoring = True + # massage the include file if line.strip().startswith('#'): continue diff --git a/numpy/random/_examples/cython/setup.py b/numpy/random/_examples/cython/setup.py index 83f06fde8581..7e0dd3e057a6 100644 --- a/numpy/random/_examples/cython/setup.py +++ b/numpy/random/_examples/cython/setup.py @@ -4,19 +4,20 @@ Usage: python setup.py build_ext -i """ +from distutils.core import setup +from os.path import dirname, join, abspath import numpy as np -from distutils.core import setup from Cython.Build import cythonize +from numpy.distutils.misc_util import get_info from setuptools.extension import Extension -from os.path import join, dirname path = dirname(__file__) src_dir = join(dirname(path), '..', 'src') defs = [('NPY_NO_DEPRECATED_API', 0)] inc_path = np.get_include() -# not so nice. We need the random/lib library from numpy -lib_path = join(np.get_include(), '..', '..', 'random', 'lib') +lib_path = [abspath(join(np.get_include(), '..', '..', 'random', 'lib'))] +lib_path += get_info('npymath')['library_dirs'] extending = Extension("extending", sources=[join('.', 'extending.pyx')], @@ -29,10 +30,10 @@ distributions = Extension("extending_distributions", sources=[join('.', 'extending_distributions.pyx')], include_dirs=[inc_path], - library_dirs=[lib_path], - libraries=['npyrandom'], + library_dirs=lib_path, + libraries=['npyrandom', 'npymath'], define_macros=defs, - ) + ) extensions = [extending, distributions] diff --git a/numpy/random/_examples/numba/extending.py b/numpy/random/_examples/numba/extending.py index da4d9394461a..f387db69502a 100644 --- a/numpy/random/_examples/numba/extending.py +++ b/numpy/random/_examples/numba/extending.py @@ -44,7 +44,7 @@ def numpycall(): assert r1.shape == r2.shape t1 = timeit(numbacall, number=1000) -+print(f'{t1:.2f} secs for {n} PCG64 (Numba/PCG64) gaussian randoms') +print(f'{t1:.2f} secs for {n} PCG64 (Numba/PCG64) gaussian randoms') t2 = timeit(numpycall, number=1000) print(f'{t2:.2f} secs for {n} PCG64 (NumPy/PCG64) gaussian randoms') diff --git a/numpy/random/_generator.pyi b/numpy/random/_generator.pyi new file mode 100644 index 000000000000..14dc5513174d --- /dev/null +++ b/numpy/random/_generator.pyi @@ -0,0 +1,651 @@ +import sys +from typing import Any, Callable, Dict, Optional, Tuple, Type, Union, overload, TypeVar + +from numpy import ( + bool_, + dtype, + float32, + float64, + int8, + int16, + int32, + int64, + int_, + ndarray, + uint, + uint8, + uint16, + uint32, + uint64, +) +from numpy.random import BitGenerator, SeedSequence +from numpy.typing import ( + ArrayLike, + _ArrayLikeFloat_co, + _ArrayLikeInt_co, + _DoubleCodes, + _DTypeLikeBool, + _DTypeLikeInt, + _DTypeLikeUInt, + _Float32Codes, + _Float64Codes, + _Int8Codes, + _Int16Codes, + _Int32Codes, + _Int64Codes, + _IntCodes, + _ShapeLike, + _SingleCodes, + _SupportsDType, + _UInt8Codes, + _UInt16Codes, + _UInt32Codes, + _UInt64Codes, + _UIntCodes, +) + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_ArrayType = TypeVar("_ArrayType", bound=ndarray[Any, Any]) + +_DTypeLikeFloat32 = Union[ + dtype[float32], + _SupportsDType[dtype[float32]], + Type[float32], + _Float32Codes, + _SingleCodes, +] + +_DTypeLikeFloat64 = Union[ + dtype[float64], + _SupportsDType[dtype[float64]], + Type[float], + Type[float64], + _Float64Codes, + _DoubleCodes, +] + +class Generator: + def __init__(self, bit_generator: BitGenerator) -> None: ... + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + def __getstate__(self) -> Dict[str, Any]: ... + def __setstate__(self, state: Dict[str, Any]) -> None: ... + def __reduce__(self) -> Tuple[Callable[[str], Generator], Tuple[str], Dict[str, Any]]: ... + @property + def bit_generator(self) -> BitGenerator: ... + def bytes(self, length: int) -> bytes: ... + @overload + def standard_normal( # type: ignore[misc] + self, + size: None = ..., + dtype: Union[_DTypeLikeFloat32, _DTypeLikeFloat64] = ..., + out: None = ..., + ) -> float: ... + @overload + def standard_normal( # type: ignore[misc] + self, + size: _ShapeLike = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_normal( # type: ignore[misc] + self, + *, + out: ndarray[Any, dtype[float64]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_normal( # type: ignore[misc] + self, + size: _ShapeLike = ..., + dtype: _DTypeLikeFloat32 = ..., + out: Optional[ndarray[Any, dtype[float32]]] = ..., + ) -> ndarray[Any, dtype[float32]]: ... + @overload + def standard_normal( # type: ignore[misc] + self, + size: _ShapeLike = ..., + dtype: _DTypeLikeFloat64 = ..., + out: Optional[ndarray[Any, dtype[float64]]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def permutation(self, x: int, axis: int = ...) -> ndarray[Any, dtype[int64]]: ... + @overload + def permutation(self, x: ArrayLike, axis: int = ...) -> ndarray[Any, Any]: ... + @overload + def standard_exponential( # type: ignore[misc] + self, + size: None = ..., + dtype: Union[_DTypeLikeFloat32, _DTypeLikeFloat64] = ..., + method: Literal["zig", "inv"] = ..., + out: None = ..., + ) -> float: ... + @overload + def standard_exponential( + self, + size: _ShapeLike = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_exponential( + self, + *, + out: ndarray[Any, dtype[float64]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_exponential( + self, + size: _ShapeLike = ..., + *, + method: Literal["zig", "inv"] = ..., + out: Optional[ndarray[Any, dtype[float64]]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_exponential( + self, + size: _ShapeLike = ..., + dtype: _DTypeLikeFloat32 = ..., + method: Literal["zig", "inv"] = ..., + out: Optional[ndarray[Any, dtype[float32]]] = ..., + ) -> ndarray[Any, dtype[float32]]: ... + @overload + def standard_exponential( + self, + size: _ShapeLike = ..., + dtype: _DTypeLikeFloat64 = ..., + method: Literal["zig", "inv"] = ..., + out: Optional[ndarray[Any, dtype[float64]]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def random( # type: ignore[misc] + self, + size: None = ..., + dtype: Union[_DTypeLikeFloat32, _DTypeLikeFloat64] = ..., + out: None = ..., + ) -> float: ... + @overload + def random( + self, + *, + out: ndarray[Any, dtype[float64]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def random( + self, + size: _ShapeLike = ..., + *, + out: Optional[ndarray[Any, dtype[float64]]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def random( + self, + size: _ShapeLike = ..., + dtype: _DTypeLikeFloat32 = ..., + out: Optional[ndarray[Any, dtype[float32]]] = ..., + ) -> ndarray[Any, dtype[float32]]: ... + @overload + def random( + self, + size: _ShapeLike = ..., + dtype: _DTypeLikeFloat64 = ..., + out: Optional[ndarray[Any, dtype[float64]]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def beta(self, a: float, b: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def beta( + self, a: _ArrayLikeFloat_co, b: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def exponential(self, scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def exponential( + self, scale: _ArrayLikeFloat_co = ..., size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: int, + high: Optional[int] = ..., + ) -> int: ... + @overload + def integers( # type: ignore[misc] + self, + low: int, + high: Optional[int] = ..., + size: None = ..., + dtype: _DTypeLikeBool = ..., + endpoint: bool = ..., + ) -> bool: ... + @overload + def integers( # type: ignore[misc] + self, + low: int, + high: Optional[int] = ..., + size: None = ..., + dtype: Union[_DTypeLikeInt, _DTypeLikeUInt] = ..., + endpoint: bool = ..., + ) -> int: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: _DTypeLikeBool = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[bool_]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[int8], Type[int8], _Int8Codes, _SupportsDType[dtype[int8]]] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[int8]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[int16], Type[int16], _Int16Codes, _SupportsDType[dtype[int16]]] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[int16]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[int32], Type[int32], _Int32Codes, _SupportsDType[dtype[int32]]] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[Union[int32]]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Optional[ + Union[dtype[int64], Type[int64], _Int64Codes, _SupportsDType[dtype[int64]]] + ] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[uint8], Type[uint8], _UInt8Codes, _SupportsDType[dtype[uint8]]] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[uint8]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[uint16], Type[uint16], _UInt16Codes, _SupportsDType[dtype[uint16]] + ] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[Union[uint16]]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[uint32], Type[uint32], _UInt32Codes, _SupportsDType[dtype[uint32]] + ] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[uint32]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[uint64], Type[uint64], _UInt64Codes, _SupportsDType[dtype[uint64]] + ] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[uint64]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[int_], Type[int], Type[int_], _IntCodes, _SupportsDType[dtype[int_]] + ] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def integers( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[uint], Type[uint], _UIntCodes, _SupportsDType[dtype[uint]]] = ..., + endpoint: bool = ..., + ) -> ndarray[Any, dtype[uint]]: ... + # TODO: Use a TypeVar _T here to get away from Any output? Should be int->ndarray[Any,dtype[int64]], ArrayLike[_T] -> Union[_T, ndarray[Any,Any]] + @overload + def choice( + self, + a: int, + size: None = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + axis: int = ..., + shuffle: bool = ..., + ) -> int: ... + @overload + def choice( + self, + a: int, + size: _ShapeLike = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + axis: int = ..., + shuffle: bool = ..., + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def choice( + self, + a: ArrayLike, + size: None = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + axis: int = ..., + shuffle: bool = ..., + ) -> Any: ... + @overload + def choice( + self, + a: ArrayLike, + size: _ShapeLike = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + axis: int = ..., + shuffle: bool = ..., + ) -> ndarray[Any, Any]: ... + @overload + def uniform(self, low: float = ..., high: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def uniform( + self, + low: _ArrayLikeFloat_co = ..., + high: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def normal(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def normal( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_gamma( # type: ignore[misc] + self, + shape: float, + size: None = ..., + dtype: Union[_DTypeLikeFloat32, _DTypeLikeFloat64] = ..., + out: None = ..., + ) -> float: ... + @overload + def standard_gamma( + self, + shape: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_gamma( + self, + shape: _ArrayLikeFloat_co, + *, + out: ndarray[Any, dtype[float64]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_gamma( + self, + shape: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + dtype: _DTypeLikeFloat32 = ..., + out: Optional[ndarray[Any, dtype[float32]]] = ..., + ) -> ndarray[Any, dtype[float32]]: ... + @overload + def standard_gamma( + self, + shape: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + dtype: _DTypeLikeFloat64 = ..., + out: Optional[ndarray[Any, dtype[float64]]] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def gamma(self, shape: float, scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def gamma( + self, + shape: _ArrayLikeFloat_co, + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def f(self, dfnum: float, dfden: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def f( + self, dfnum: _ArrayLikeFloat_co, dfden: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def noncentral_f(self, dfnum: float, dfden: float, nonc: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def noncentral_f( + self, + dfnum: _ArrayLikeFloat_co, + dfden: _ArrayLikeFloat_co, + nonc: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def chisquare(self, df: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def chisquare( + self, df: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def noncentral_chisquare(self, df: float, nonc: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def noncentral_chisquare( + self, df: _ArrayLikeFloat_co, nonc: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_t(self, df: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def standard_t( + self, df: _ArrayLikeFloat_co, size: None = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_t( + self, df: _ArrayLikeFloat_co, size: _ShapeLike = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def vonmises(self, mu: float, kappa: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def vonmises( + self, mu: _ArrayLikeFloat_co, kappa: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def pareto(self, a: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def pareto( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def weibull(self, a: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def weibull( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def power(self, a: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def power( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_cauchy(self, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def standard_cauchy(self, size: _ShapeLike = ...) -> ndarray[Any, dtype[float64]]: ... + @overload + def laplace(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def laplace( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def gumbel(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def gumbel( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def logistic(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def logistic( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def lognormal(self, mean: float = ..., sigma: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def lognormal( + self, + mean: _ArrayLikeFloat_co = ..., + sigma: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def rayleigh(self, scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def rayleigh( + self, scale: _ArrayLikeFloat_co = ..., size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def wald(self, mean: float, scale: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def wald( + self, mean: _ArrayLikeFloat_co, scale: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def triangular(self, left: float, mode: float, right: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def triangular( + self, + left: _ArrayLikeFloat_co, + mode: _ArrayLikeFloat_co, + right: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def binomial(self, n: int, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def binomial( + self, n: _ArrayLikeInt_co, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def negative_binomial(self, n: float, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def negative_binomial( + self, n: _ArrayLikeFloat_co, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def poisson(self, lam: float = ..., size: None = ...) -> int: ... # type: ignore[misc] + @overload + def poisson( + self, lam: _ArrayLikeFloat_co = ..., size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def zipf(self, a: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def zipf( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def geometric(self, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def geometric( + self, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def hypergeometric(self, ngood: int, nbad: int, nsample: int, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def hypergeometric( + self, + ngood: _ArrayLikeInt_co, + nbad: _ArrayLikeInt_co, + nsample: _ArrayLikeInt_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def logseries(self, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def logseries( + self, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + def multivariate_normal( + self, + mean: _ArrayLikeFloat_co, + cov: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + check_valid: Literal["warn", "raise", "ignore"] = ..., + tol: float = ..., + *, + method: Literal["svd", "eigh", "cholesky"] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + def multinomial( + self, n: _ArrayLikeInt_co, pvals: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int64]]: ... + def multivariate_hypergeometric( + self, + colors: _ArrayLikeInt_co, + nsample: int, + size: Optional[_ShapeLike] = ..., + method: Literal["marginals", "count"] = ..., + ) -> ndarray[Any, dtype[int64]]: ... + def dirichlet( + self, alpha: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + def permuted( + self, x: ArrayLike, *, axis: Optional[int] = ..., out: Optional[ndarray[Any, Any]] = ... + ) -> ndarray[Any, Any]: ... + def shuffle(self, x: ArrayLike, axis: int = ...) -> None: ... + +def default_rng( + seed: Union[None, _ArrayLikeInt_co, SeedSequence, BitGenerator, Generator] = ... +) -> Generator: ... diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index bff023d7ffb6..cd0b248723d2 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -2,6 +2,7 @@ #cython: wraparound=False, nonecheck=False, boundscheck=False, cdivision=True, language_level=3 import operator import warnings +from collections.abc import Sequence from cpython.pycapsule cimport PyCapsule_IsValid, PyCapsule_GetPointer from cpython cimport (Py_INCREF, PyFloat_AsDouble) @@ -175,8 +176,8 @@ cdef class Generator: Examples -------- >>> from numpy.random import Generator, PCG64 - >>> rg = Generator(PCG64()) - >>> rg.standard_normal() + >>> rng = Generator(PCG64()) + >>> rng.standard_normal() -0.203 # random See Also @@ -385,7 +386,7 @@ cdef class Generator: 0.0, '', CONS_NONE, None) - def standard_exponential(self, size=None, dtype=np.float64, method=u'zig', out=None): + def standard_exponential(self, size=None, dtype=np.float64, method='zig', out=None): """ standard_exponential(size=None, dtype=np.float64, method='zig', out=None) @@ -425,12 +426,12 @@ cdef class Generator: """ _dtype = np.dtype(dtype) if _dtype == np.float64: - if method == u'zig': + if method == 'zig': return double_fill(&random_standard_exponential_fill, &self._bitgen, size, self.lock, out) else: return double_fill(&random_standard_exponential_inv_fill, &self._bitgen, size, self.lock, out) elif _dtype == np.float32: - if method == u'zig': + if method == 'zig': return float_fill(&random_standard_exponential_fill_f, &self._bitgen, size, self.lock, out) else: return float_fill(&random_standard_exponential_inv_fill_f, &self._bitgen, size, self.lock, out) @@ -578,13 +579,13 @@ cdef class Generator: Returns ------- - out : str + out : bytes String of length `length`. Examples -------- >>> np.random.default_rng().bytes(10) - ' eh\\x85\\x022SZ\\xbf\\xa4' #random + b'\xfeC\x9b\x86\x17\xf2\xa1\xafcp' # random """ cdef Py_ssize_t n_uint32 = ((length - 1) // 4 + 1) @@ -598,7 +599,7 @@ cdef class Generator: """ choice(a, size=None, replace=True, p=None, axis=0, shuffle=True) - Generates a random sample from a given 1-D array + Generates a random sample from a given array Parameters ---------- @@ -613,11 +614,12 @@ cdef class Generator: len(size)``. Default is None, in which case a single value is returned. replace : bool, optional - Whether the sample is with or without replacement + Whether the sample is with or without replacement. Default is True, + meaning that a value of ``a`` can be selected multiple times. p : 1-D array_like, optional The probabilities associated with each entry in a. - If not given the sample assumes a uniform distribution over all - entries in a. + If not given, the sample assumes a uniform distribution over all + entries in ``a``. axis : int, optional The axis along which the selection is performed. The default, 0, selects by row. @@ -643,6 +645,12 @@ cdef class Generator: -------- integers, shuffle, permutation + Notes + ----- + Setting user-specified probabilities through ``p`` uses a more general but less + efficient sampler than the default. The general sampler produces a different sample + than the optimized sampler even if each element of ``p`` is 1 / len(a). + Examples -------- Generate a uniform random sample from np.arange(5) of size 3: @@ -664,6 +672,13 @@ cdef class Generator: array([3,1,0]) # random >>> #This is equivalent to rng.permutation(np.arange(5))[:3] + Generate a uniform random sample from a 2-D array along the first + axis (the default), without replacement: + + >>> rng.choice([[0, 1, 2], [3, 4, 5], [6, 7, 8]], 2, replace=False) + array([[3, 4, 5], # random + [0, 1, 2]]) + Generate a non-uniform random sample from np.arange(5) of size 3 without replacement: @@ -686,20 +701,22 @@ cdef class Generator: cdef uint64_t set_size, mask cdef uint64_t[::1] hash_set # Format and Verify input + a_original = a a = np.array(a, copy=False) if a.ndim == 0: try: # __index__ must return an integer by python rules. pop_size = operator.index(a.item()) - except TypeError: - raise ValueError("a must an array or an integer") + except TypeError as exc: + raise ValueError("a must be a sequence or an integer, " + f"not {type(a_original)}") from exc if pop_size <= 0 and np.prod(size) != 0: - raise ValueError("a must be a positive integer unless no" + raise ValueError("a must be a positive integer unless no " "samples are taken") else: pop_size = a.shape[axis] if pop_size == 0 and np.prod(size) != 0: - raise ValueError("a cannot be empty unless no samples are" + raise ValueError("a cannot be empty unless no samples are " "taken") if p is not None: @@ -859,7 +876,8 @@ cdef class Generator: greater than or equal to low. The default value is 0. high : float or array_like of floats Upper boundary of the output interval. All values generated will be - less than high. The default value is 1.0. + less than high. high - low must be non-negative. The default value + is 1.0. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -885,10 +903,6 @@ cdef class Generator: anywhere within the interval ``[a, b)``, and zero elsewhere. When ``high`` == ``low``, values of ``low`` will be returned. - If ``high`` < ``low``, the results are officially undefined - and may eventually raise an error, i.e. do not rely on this - function to behave when passed arguments satisfying that - inequality condition. Examples -------- @@ -914,7 +928,7 @@ cdef class Generator: """ cdef bint is_scalar = True cdef np.ndarray alow, ahigh, arange - cdef double _low, _high, range + cdef double _low, _high, rng cdef object temp alow = np.PyArray_FROM_OTF(low, np.NPY_DOUBLE, np.NPY_ALIGNED) @@ -923,13 +937,13 @@ cdef class Generator: if np.PyArray_NDIM(alow) == np.PyArray_NDIM(ahigh) == 0: _low = PyFloat_AsDouble(low) _high = PyFloat_AsDouble(high) - range = _high - _low - if not np.isfinite(range): - raise OverflowError('Range exceeds valid bounds') + rng = _high - _low + if not np.isfinite(rng): + raise OverflowError('high - low range exceeds valid bounds') return cont(&random_uniform, &self._bitgen, size, self.lock, 2, _low, '', CONS_NONE, - range, '', CONS_NONE, + rng, 'high - low', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, None) @@ -943,7 +957,7 @@ cdef class Generator: raise OverflowError('Range exceeds valid bounds') return cont(&random_uniform, &self._bitgen, size, self.lock, 2, alow, '', CONS_NONE, - arange, '', CONS_NONE, + arange, 'high - low', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, None) @@ -984,14 +998,14 @@ cdef class Generator: ----- For random samples from :math:`N(\\mu, \\sigma^2)`, use one of:: - mu + sigma * gen.standard_normal(size=...) - gen.normal(mu, sigma, size=...) + mu + sigma * rng.standard_normal(size=...) + rng.normal(mu, sigma, size=...) Examples -------- >>> rng = np.random.default_rng() >>> rng.standard_normal() - 2.1923875335537315 #random + 2.1923875335537315 # random >>> s = rng.standard_normal(8000) >>> s @@ -1095,7 +1109,7 @@ cdef class Generator: 0.0 # may vary >>> abs(sigma - np.std(s, ddof=1)) - 0.1 # may vary + 0.0 # may vary Display the histogram of the samples, along with the probability density function: @@ -1727,33 +1741,45 @@ cdef class Generator: ... 7515, 8230, 8770]) Does their energy intake deviate systematically from the recommended - value of 7725 kJ? - - We have 10 degrees of freedom, so is the sample mean within 95% of the - recommended value? + value of 7725 kJ? Our null hypothesis will be the absence of deviation, + and the alternate hypothesis will be the presence of an effect that could be + either positive or negative, hence making our test 2-tailed. + + Because we are estimating the mean and we have N=11 values in our sample, + we have N-1=10 degrees of freedom. We set our significance level to 95% and + compute the t statistic using the empirical mean and empirical standard + deviation of our intake. We use a ddof of 1 to base the computation of our + empirical standard deviation on an unbiased estimate of the variance (note: + the final estimate is not unbiased due to the concave nature of the square + root). - >>> s = np.random.default_rng().standard_t(10, size=100000) >>> np.mean(intake) 6753.636363636364 >>> intake.std(ddof=1) 1142.1232221373727 + >>> t = (np.mean(intake)-7725)/(intake.std(ddof=1)/np.sqrt(len(intake))) + >>> t + -2.8207540608310198 - Calculate the t statistic, setting the ddof parameter to the unbiased - value so the divisor in the standard deviation will be degrees of - freedom, N-1. + We draw 1000000 samples from Student's t distribution with the adequate + degrees of freedom. - >>> t = (np.mean(intake)-7725)/(intake.std(ddof=1)/np.sqrt(len(intake))) >>> import matplotlib.pyplot as plt + >>> s = np.random.default_rng().standard_t(10, size=1000000) >>> h = plt.hist(s, bins=100, density=True) - For a one-sided t-test, how far out in the distribution does the t - statistic appear? + Does our t statistic land in one of the two critical regions found at + both tails of the distribution? + + >>> np.sum(np.abs(t) < np.abs(s)) / float(len(s)) + 0.018318 #random < 0.05, statistic is in critical region - >>> np.sum(s= 0. A sequence of expectation - intervals must be broadcastable over the requested size. + Expected number of events occurring in a fixed-time interval, + must be >= 0. A sequence must be broadcastable over the requested + size. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -3191,7 +3218,7 @@ cdef class Generator: How many trials succeeded after a single run? >>> (z == 1).sum() / 10000. - 0.34889999999999999 #random + 0.34889999999999999 # random """ return disc(&random_geometric, &self._bitgen, size, self.lock, 1, 0, @@ -3732,7 +3759,20 @@ cdef class Generator: pix = np.PyArray_DATA(parr) check_array_constraint(parr, 'pvals', CONS_BOUNDED_0_1) if kahan_sum(pix, d-1) > (1.0 + 1e-12): - raise ValueError("sum(pvals[:-1]) > 1.0") + # When floating, but not float dtype, and close, improve the error + # 1.0001 works for float16 and float32 + if (isinstance(pvals, np.ndarray) + and np.issubdtype(pvals.dtype, np.floating) + and pvals.dtype != float + and pvals.sum() < 1.0001): + msg = ("sum(pvals[:-1].astype(np.float64)) > 1.0. The pvals " + "array is cast to 64-bit floating point prior to " + "checking the sum. Precision changes when casting may " + "cause problems even if the sum of the original pvals " + "is valid.") + else: + msg = "sum(pvals[:-1]) > 1.0" + raise ValueError(msg) if np.PyArray_NDIM(on) != 0: # vector check_array_constraint(on, 'n', CONS_NON_NEGATIVE) @@ -4023,7 +4063,7 @@ cdef class Generator: The drawn samples, of shape ``(size, k)``. Raises - ------- + ------ ValueError If any value in ``alpha`` is less than or equal to zero @@ -4350,14 +4390,14 @@ cdef class Generator: """ shuffle(x, axis=0) - Modify a sequence in-place by shuffling its contents. + Modify an array or sequence in-place by shuffling its contents. The order of sub-arrays is changed but their contents remains the same. Parameters ---------- - x : array_like - The array or list to be shuffled. + x : ndarray or MutableSequence + The array, list or mutable sequence to be shuffled. axis : int, optional The axis which `x` is shuffled along. Default is 0. It is only supported on `ndarray` objects. @@ -4393,7 +4433,9 @@ cdef class Generator: char* x_ptr char* buf_ptr - axis = normalize_axis_index(axis, np.ndim(x)) + if isinstance(x, np.ndarray): + # Only call ndim on ndarrays, see GH 18142 + axis = normalize_axis_index(axis, np.ndim(x)) if type(x) is np.ndarray and x.ndim == 1 and x.size: # Fast, statically typed path: shuffle the underlying buffer. @@ -4417,7 +4459,11 @@ cdef class Generator: with self.lock, nogil: _shuffle_raw_wrap(&self._bitgen, n, 1, itemsize, stride, x_ptr, buf_ptr) - elif isinstance(x, np.ndarray) and x.ndim and x.size: + elif isinstance(x, np.ndarray): + if x.size == 0: + # shuffling is a no-op + return + x = np.swapaxes(x, 0, axis) buf = np.empty_like(x[0, ...]) with self.lock: @@ -4426,11 +4472,21 @@ cdef class Generator: if i == j: # i == j is not needed and memcpy is undefined. continue - buf[...] = x[j] - x[j] = x[i] - x[i] = buf + buf[...] = x[j, ...] + x[j, ...] = x[i, ...] + x[i, ...] = buf else: # Untyped path. + if not isinstance(x, Sequence): + # See gh-18206. We may decide to deprecate here in the future. + warnings.warn( + f"you are shuffling a '{type(x).__name__}' object " + "which is not a subclass of 'Sequence'; " + "`shuffle` is not guaranteed to behave correctly. " + "E.g., non-numpy array/tensor objects with view semantics " + "may contain duplicates after shuffling.", + UserWarning, stacklevel=1) # Cython does not add a level + if axis != 0: raise NotImplementedError("Axis argument is only supported " "on ndarray objects") @@ -4537,7 +4593,7 @@ def default_rng(seed=None): Examples -------- - ``default_rng`` is the reccomended constructor for the random number class + ``default_rng`` is the recommended constructor for the random number class ``Generator``. Here are several ways we can construct a random number generator using ``default_rng`` and the ``Generator`` class. diff --git a/numpy/random/_mt19937.pyi b/numpy/random/_mt19937.pyi new file mode 100644 index 000000000000..1b8bacdae4eb --- /dev/null +++ b/numpy/random/_mt19937.pyi @@ -0,0 +1,28 @@ +import sys +from typing import Any, Union + +from numpy import dtype, ndarray, uint32 +from numpy.random.bit_generator import BitGenerator, SeedSequence +from numpy.typing import _ArrayLikeInt_co + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +class _MT19937Internal(TypedDict): + key: ndarray[Any, dtype[uint32]] + pos: int + +class _MT19937State(TypedDict): + bit_generator: str + state: _MT19937Internal + +class MT19937(BitGenerator): + def __init__(self, seed: Union[None, _ArrayLikeInt_co, SeedSequence] = ...) -> None: ... + def _legacy_seeding(self, seed: _ArrayLikeInt_co) -> None: ... + def jumped(self, jumps: int = ...) -> MT19937: ... + @property + def state(self) -> _MT19937State: ... + @state.setter + def state(self, value: _MT19937State) -> None: ... diff --git a/numpy/random/_pcg64.pyi b/numpy/random/_pcg64.pyi new file mode 100644 index 000000000000..25e2fdde602d --- /dev/null +++ b/numpy/random/_pcg64.pyi @@ -0,0 +1,48 @@ +import sys +from typing import Union + +from numpy.random.bit_generator import BitGenerator, SeedSequence +from numpy.typing import _ArrayLikeInt_co + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +class _PCG64Internal(TypedDict): + state: int + inc: int + +class _PCG64State(TypedDict): + bit_generator: str + state: _PCG64Internal + has_uint32: int + uinteger: int + +class PCG64(BitGenerator): + def __init__(self, seed: Union[None, _ArrayLikeInt_co, SeedSequence] = ...) -> None: ... + def jumped(self, jumps: int = ...) -> PCG64: ... + @property + def state( + self, + ) -> _PCG64State: ... + @state.setter + def state( + self, + value: _PCG64State, + ) -> None: ... + def advance(self, delta: int) -> PCG64: ... + +class PCG64DXSM(BitGenerator): + def __init__(self, seed: Union[None, _ArrayLikeInt_co, SeedSequence] = ...) -> None: ... + def jumped(self, jumps: int = ...) -> PCG64DXSM: ... + @property + def state( + self, + ) -> _PCG64State: ... + @state.setter + def state( + self, + value: _PCG64State, + ) -> None: ... + def advance(self, delta: int) -> PCG64DXSM: ... diff --git a/numpy/random/_pcg64.pyx b/numpy/random/_pcg64.pyx index 605aae4bcb21..8a00dc265f23 100644 --- a/numpy/random/_pcg64.pyx +++ b/numpy/random/_pcg64.pyx @@ -26,6 +26,10 @@ cdef extern from "src/pcg64/pcg64.h": void pcg64_get_state(pcg64_state *state, uint64_t *state_arr, int *has_uint32, uint32_t *uinteger) void pcg64_set_state(pcg64_state *state, uint64_t *state_arr, int has_uint32, uint32_t uinteger) + uint64_t pcg64_cm_next64(pcg64_state *state) nogil + uint32_t pcg64_cm_next32(pcg64_state *state) nogil + void pcg64_cm_advance(pcg64_state *state, uint64_t *step) + cdef uint64_t pcg64_uint64(void* st) nogil: return pcg64_next64(st) @@ -35,6 +39,14 @@ cdef uint32_t pcg64_uint32(void *st) nogil: cdef double pcg64_double(void* st) nogil: return uint64_to_double(pcg64_next64(st)) +cdef uint64_t pcg64_cm_uint64(void* st) nogil: + return pcg64_cm_next64(st) + +cdef uint32_t pcg64_cm_uint32(void *st) nogil: + return pcg64_cm_next32( st) + +cdef double pcg64_cm_double(void* st) nogil: + return uint64_to_double(pcg64_cm_next64(st)) cdef class PCG64(BitGenerator): """ @@ -268,3 +280,239 @@ cdef class PCG64(BitGenerator): pcg64_advance(&self.rng_state, np.PyArray_DATA(d)) self._reset_state_variables() return self + + +cdef class PCG64DXSM(BitGenerator): + """ + PCG64DXSM(seed=None) + + BitGenerator for the PCG-64 DXSM pseudo-random number generator. + + Parameters + ---------- + seed : {None, int, array_like[ints], SeedSequence}, optional + A seed to initialize the `BitGenerator`. If None, then fresh, + unpredictable entropy will be pulled from the OS. If an ``int`` or + ``array_like[ints]`` is passed, then it will be passed to + `SeedSequence` to derive the initial `BitGenerator` state. One may also + pass in a `SeedSequence` instance. + + Notes + ----- + PCG-64 DXSM is a 128-bit implementation of O'Neill's permutation congruential + generator ([1]_, [2]_). PCG-64 DXSM has a period of :math:`2^{128}` and supports + advancing an arbitrary number of steps as well as :math:`2^{127}` streams. + The specific member of the PCG family that we use is PCG CM DXSM 128/64. It + differs from ``PCG64`` in that it uses the stronger DXSM output function, + a 64-bit "cheap multiplier" in the LCG, and outputs from the state before + advancing it rather than advance-then-output. + + ``PCG64DXSM`` provides a capsule containing function pointers that produce + doubles, and unsigned 32 and 64- bit integers. These are not + directly consumable in Python and must be consumed by a ``Generator`` + or similar object that supports low-level access. + + Supports the method :meth:`advance` to advance the RNG an arbitrary number of + steps. The state of the PCG-64 DXSM RNG is represented by 2 128-bit unsigned + integers. + + **State and Seeding** + + The ``PCG64DXSM`` state vector consists of 2 unsigned 128-bit values, + which are represented externally as Python ints. One is the state of the + PRNG, which is advanced by a linear congruential generator (LCG). The + second is a fixed odd increment used in the LCG. + + The input seed is processed by `SeedSequence` to generate both values. The + increment is not independently settable. + + **Parallel Features** + + The preferred way to use a BitGenerator in parallel applications is to use + the `SeedSequence.spawn` method to obtain entropy values, and to use these + to generate new BitGenerators: + + >>> from numpy.random import Generator, PCG64DXSM, SeedSequence + >>> sg = SeedSequence(1234) + >>> rg = [Generator(PCG64DXSM(s)) for s in sg.spawn(10)] + + **Compatibility Guarantee** + + ``PCG64DXSM`` makes a guarantee that a fixed seed and will always produce + the same random integer stream. + + References + ---------- + .. [1] `"PCG, A Family of Better Random Number Generators" + `_ + .. [2] O'Neill, Melissa E. `"PCG: A Family of Simple Fast Space-Efficient + Statistically Good Algorithms for Random Number Generation" + `_ + """ + cdef pcg64_state rng_state + cdef pcg64_random_t pcg64_random_state + + def __init__(self, seed=None): + BitGenerator.__init__(self, seed) + self.rng_state.pcg_state = &self.pcg64_random_state + + self._bitgen.state = &self.rng_state + self._bitgen.next_uint64 = &pcg64_cm_uint64 + self._bitgen.next_uint32 = &pcg64_cm_uint32 + self._bitgen.next_double = &pcg64_cm_double + self._bitgen.next_raw = &pcg64_cm_uint64 + # Seed the _bitgen + val = self._seed_seq.generate_state(4, np.uint64) + pcg64_set_seed(&self.rng_state, + np.PyArray_DATA(val), + (np.PyArray_DATA(val) + 2)) + self._reset_state_variables() + + cdef _reset_state_variables(self): + self.rng_state.has_uint32 = 0 + self.rng_state.uinteger = 0 + + cdef jump_inplace(self, jumps): + """ + Jump state in-place + Not part of public API + + Parameters + ---------- + jumps : integer, positive + Number of times to jump the state of the rng. + + Notes + ----- + The step size is phi-1 when multiplied by 2**128 where phi is the + golden ratio. + """ + step = 0x9e3779b97f4a7c15f39cc0605cedc835 + self.advance(step * int(jumps)) + + def jumped(self, jumps=1): + """ + jumped(jumps=1) + + Returns a new bit generator with the state jumped. + + Jumps the state as-if jumps * 210306068529402873165736369884012333109 + random numbers have been generated. + + Parameters + ---------- + jumps : integer, positive + Number of times to jump the state of the bit generator returned + + Returns + ------- + bit_generator : PCG64DXSM + New instance of generator jumped iter times + + Notes + ----- + The step size is phi-1 when multiplied by 2**128 where phi is the + golden ratio. + """ + cdef PCG64DXSM bit_generator + + bit_generator = self.__class__() + bit_generator.state = self.state + bit_generator.jump_inplace(jumps) + + return bit_generator + + @property + def state(self): + """ + Get or set the PRNG state + + Returns + ------- + state : dict + Dictionary containing the information required to describe the + state of the PRNG + """ + cdef np.ndarray state_vec + cdef int has_uint32 + cdef uint32_t uinteger + + # state_vec is state.high, state.low, inc.high, inc.low + state_vec = np.empty(4, dtype=np.uint64) + pcg64_get_state(&self.rng_state, + np.PyArray_DATA(state_vec), + &has_uint32, &uinteger) + state = int(state_vec[0]) * 2**64 + int(state_vec[1]) + inc = int(state_vec[2]) * 2**64 + int(state_vec[3]) + return {'bit_generator': self.__class__.__name__, + 'state': {'state': state, 'inc': inc}, + 'has_uint32': has_uint32, + 'uinteger': uinteger} + + @state.setter + def state(self, value): + cdef np.ndarray state_vec + cdef int has_uint32 + cdef uint32_t uinteger + if not isinstance(value, dict): + raise TypeError('state must be a dict') + bitgen = value.get('bit_generator', '') + if bitgen != self.__class__.__name__: + raise ValueError('state must be for a {0} ' + 'RNG'.format(self.__class__.__name__)) + state_vec = np.empty(4, dtype=np.uint64) + state_vec[0] = value['state']['state'] // 2 ** 64 + state_vec[1] = value['state']['state'] % 2 ** 64 + state_vec[2] = value['state']['inc'] // 2 ** 64 + state_vec[3] = value['state']['inc'] % 2 ** 64 + has_uint32 = value['has_uint32'] + uinteger = value['uinteger'] + pcg64_set_state(&self.rng_state, + np.PyArray_DATA(state_vec), + has_uint32, uinteger) + + def advance(self, delta): + """ + advance(delta) + + Advance the underlying RNG as-if delta draws have occurred. + + Parameters + ---------- + delta : integer, positive + Number of draws to advance the RNG. Must be less than the + size state variable in the underlying RNG. + + Returns + ------- + self : PCG64 + RNG advanced delta steps + + Notes + ----- + Advancing a RNG updates the underlying RNG state as-if a given + number of calls to the underlying RNG have been made. In general + there is not a one-to-one relationship between the number output + random values from a particular distribution and the number of + draws from the core RNG. This occurs for two reasons: + + * The random values are simulated using a rejection-based method + and so, on average, more than one value from the underlying + RNG is required to generate an single draw. + * The number of bits required to generate a simulated value + differs from the number of bits generated by the underlying + RNG. For example, two 16-bit integer values can be simulated + from a single draw of a 32-bit RNG. + + Advancing the RNG state resets any pre-computed random numbers. + This is required to ensure exact reproducibility. + """ + delta = wrap_int(delta, 128) + + cdef np.ndarray d = np.empty(2, dtype=np.uint64) + d[0] = delta // 2**64 + d[1] = delta % 2**64 + pcg64_cm_advance(&self.rng_state, np.PyArray_DATA(d)) + self._reset_state_variables() + return self + diff --git a/numpy/random/_philox.pyi b/numpy/random/_philox.pyi new file mode 100644 index 000000000000..f6a5b9b9b3c8 --- /dev/null +++ b/numpy/random/_philox.pyi @@ -0,0 +1,42 @@ +import sys +from typing import Any, Union + +from numpy import dtype, ndarray, uint64 +from numpy.random.bit_generator import BitGenerator, SeedSequence +from numpy.typing import _ArrayLikeInt_co + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +class _PhiloxInternal(TypedDict): + counter: ndarray[Any, dtype[uint64]] + key: ndarray[Any, dtype[uint64]] + +class _PhiloxState(TypedDict): + bit_generator: str + state: _PhiloxInternal + buffer: ndarray[Any, dtype[uint64]] + buffer_pos: int + has_uint32: int + uinteger: int + +class Philox(BitGenerator): + def __init__( + self, + seed: Union[None, _ArrayLikeInt_co, SeedSequence] = ..., + counter: Union[None, _ArrayLikeInt_co] = ..., + key: Union[None, _ArrayLikeInt_co] = ..., + ) -> None: ... + @property + def state( + self, + ) -> _PhiloxState: ... + @state.setter + def state( + self, + value: _PhiloxState, + ) -> None: ... + def jumped(self, jumps: int = ...) -> Philox: ... + def advance(self, delta: int) -> Philox: ... diff --git a/numpy/random/_philox.pyx b/numpy/random/_philox.pyx index 7e88804904a6..0fe8ebd7cd5f 100644 --- a/numpy/random/_philox.pyx +++ b/numpy/random/_philox.pyx @@ -1,10 +1,5 @@ from cpython.pycapsule cimport PyCapsule_New -try: - from threading import Lock -except ImportError: - from dummy_threading import Lock - import numpy as np cimport numpy as np diff --git a/numpy/random/_pickle.py b/numpy/random/_pickle.py index 29ff696448fe..a32f64f4a3d3 100644 --- a/numpy/random/_pickle.py +++ b/numpy/random/_pickle.py @@ -1,6 +1,6 @@ from .mtrand import RandomState from ._philox import Philox -from ._pcg64 import PCG64 +from ._pcg64 import PCG64, PCG64DXSM from ._sfc64 import SFC64 from ._generator import Generator @@ -8,6 +8,7 @@ BitGenerators = {'MT19937': MT19937, 'PCG64': PCG64, + 'PCG64DXSM': PCG64DXSM, 'Philox': Philox, 'SFC64': SFC64, } @@ -19,7 +20,7 @@ def __generator_ctor(bit_generator_name='MT19937'): Parameters ---------- - bit_generator_name: str + bit_generator_name : str String containing the core BitGenerator Returns @@ -42,7 +43,7 @@ def __bit_generator_ctor(bit_generator_name='MT19937'): Parameters ---------- - bit_generator_name: str + bit_generator_name : str String containing the name of the BitGenerator Returns @@ -65,7 +66,7 @@ def __randomstate_ctor(bit_generator_name='MT19937'): Parameters ---------- - bit_generator_name: str + bit_generator_name : str String containing the core BitGenerator Returns diff --git a/numpy/random/_sfc64.pyi b/numpy/random/_sfc64.pyi new file mode 100644 index 000000000000..72a271c9243e --- /dev/null +++ b/numpy/random/_sfc64.pyi @@ -0,0 +1,34 @@ +import sys +from typing import Any, Union + +from numpy import dtype as dtype +from numpy import ndarray as ndarray +from numpy import uint64 +from numpy.random.bit_generator import BitGenerator, SeedSequence +from numpy.typing import _ArrayLikeInt_co + +if sys.version_info >= (3, 8): + from typing import TypedDict +else: + from typing_extensions import TypedDict + +class _SFC64Internal(TypedDict): + state: ndarray[Any, dtype[uint64]] + +class _SFC64State(TypedDict): + bit_generator: str + state: _SFC64Internal + has_uint32: int + uinteger: int + +class SFC64(BitGenerator): + def __init__(self, seed: Union[None, _ArrayLikeInt_co, SeedSequence] = ...) -> None: ... + @property + def state( + self, + ) -> _SFC64State: ... + @state.setter + def state( + self, + value: _SFC64State, + ) -> None: ... diff --git a/numpy/random/bit_generator.pyi b/numpy/random/bit_generator.pyi new file mode 100644 index 000000000000..5b68dde6ccbf --- /dev/null +++ b/numpy/random/bit_generator.pyi @@ -0,0 +1,121 @@ +import abc +import sys +from threading import Lock +from typing import ( + Any, + Callable, + Dict, + List, + Mapping, + NamedTuple, + Optional, + Sequence, + Tuple, + Type, + TypedDict, + TypeVar, + Union, + overload, +) + +from numpy import dtype, ndarray, uint32, uint64 +from numpy.typing import _ArrayLikeInt_co, _ShapeLike, _SupportsDType, _UInt32Codes, _UInt64Codes + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_T = TypeVar("_T") + +_DTypeLikeUint32 = Union[ + dtype[uint32], + _SupportsDType[dtype[uint32]], + Type[uint32], + _UInt32Codes, +] +_DTypeLikeUint64 = Union[ + dtype[uint64], + _SupportsDType[dtype[uint64]], + Type[uint64], + _UInt64Codes, +] + +class _SeedSeqState(TypedDict): + entropy: Union[None, int, Sequence[int]] + spawn_key: Tuple[int, ...] + pool_size: int + n_children_spawned: int + +class _Interface(NamedTuple): + state_address: Any + state: Any + next_uint64: Any + next_uint32: Any + next_double: Any + bit_generator: Any + +class ISeedSequence(abc.ABC): + @abc.abstractmethod + def generate_state( + self, n_words: int, dtype: Union[_DTypeLikeUint32, _DTypeLikeUint64] = ... + ) -> ndarray[Any, dtype[Union[uint32, uint64]]]: ... + +class ISpawnableSeedSequence(ISeedSequence): + @abc.abstractmethod + def spawn(self: _T, n_children: int) -> List[_T]: ... + +class SeedlessSeedSequence(ISpawnableSeedSequence): + def generate_state( + self, n_words: int, dtype: Union[_DTypeLikeUint32, _DTypeLikeUint64] = ... + ) -> ndarray[Any, dtype[Union[uint32, uint64]]]: ... + def spawn(self: _T, n_children: int) -> List[_T]: ... + +class SeedSequence(ISpawnableSeedSequence): + entropy: Union[None, int, Sequence[int]] + spawn_key: Tuple[int, ...] + pool_size: int + n_children_spawned: int + pool: ndarray[Any, dtype[uint32]] + def __init__( + self, + entropy: Union[None, int, Sequence[int], _ArrayLikeInt_co] = ..., + *, + spawn_key: Sequence[int] = ..., + pool_size: int = ..., + n_children_spawned: int = ..., + ) -> None: ... + def __repr__(self) -> str: ... + @property + def state( + self, + ) -> _SeedSeqState: ... + def generate_state( + self, n_words: int, dtype: Union[_DTypeLikeUint32, _DTypeLikeUint64] = ... + ) -> ndarray[Any, dtype[Union[uint32, uint64]]]: ... + def spawn(self, n_children: int) -> List[SeedSequence]: ... + +class BitGenerator(abc.ABC): + lock: Lock + def __init__(self, seed: Union[None, _ArrayLikeInt_co, SeedSequence] = ...) -> None: ... + def __getstate__(self) -> Dict[str, Any]: ... + def __setstate__(self, state: Dict[str, Any]) -> None: ... + def __reduce__( + self, + ) -> Tuple[Callable[[str], BitGenerator], Tuple[str], Tuple[Dict[str, Any]]]: ... + @abc.abstractmethod + @property + def state(self) -> Mapping[str, Any]: ... + @state.setter + def state(self, value: Mapping[str, Any]) -> None: ... + @overload + def random_raw(self, size: None = ..., output: Literal[True] = ...) -> int: ... # type: ignore[misc] + @overload + def random_raw(self, size: _ShapeLike = ..., output: Literal[True] = ...) -> ndarray[Any, dtype[uint64]]: ... # type: ignore[misc] + @overload + def random_raw(self, size: Optional[_ShapeLike] = ..., output: Literal[False] = ...) -> None: ... # type: ignore[misc] + def _benchmark(self, cnt: int, method: str = ...) -> None: ... + @property + def ctypes(self) -> _Interface: ... + @property + def cffi(self) -> _Interface: ... diff --git a/numpy/random/bit_generator.pyx b/numpy/random/bit_generator.pyx index 9b0c363ef04b..123d77b40e2e 100644 --- a/numpy/random/bit_generator.pyx +++ b/numpy/random/bit_generator.pyx @@ -43,10 +43,7 @@ except ImportError: from random import SystemRandom randbits = SystemRandom().getrandbits -try: - from threading import Lock -except ImportError: - from dummy_threading import Lock +from threading import Lock from cpython.pycapsule cimport PyCapsule_New @@ -587,8 +584,8 @@ cdef class BitGenerator(): """ return random_raw(&self._bitgen, self.lock, size, output) - def _benchmark(self, Py_ssize_t cnt, method=u'uint64'): - '''Used in tests''' + def _benchmark(self, Py_ssize_t cnt, method='uint64'): + """Used in tests""" return benchmark(&self._bitgen, self.lock, cnt, method) @property diff --git a/numpy/random/include/legacy-distributions.h b/numpy/random/include/legacy-distributions.h index b8ba0841c127..f6c5cf0532d1 100644 --- a/numpy/random/include/legacy-distributions.h +++ b/numpy/random/include/legacy-distributions.h @@ -17,6 +17,7 @@ extern double legacy_weibull(aug_bitgen_t *aug_state, double a); extern double legacy_power(aug_bitgen_t *aug_state, double a); extern double legacy_gamma(aug_bitgen_t *aug_state, double shape, double scale); extern double legacy_chisquare(aug_bitgen_t *aug_state, double df); +extern double legacy_rayleigh(bitgen_t *bitgen_state, double mode); extern double legacy_noncentral_chisquare(aug_bitgen_t *aug_state, double df, double nonc); extern double legacy_noncentral_f(aug_bitgen_t *aug_state, double dfnum, @@ -31,6 +32,7 @@ extern double legacy_f(aug_bitgen_t *aug_state, double dfnum, double dfden); extern double legacy_normal(aug_bitgen_t *aug_state, double loc, double scale); extern double legacy_standard_gamma(aug_bitgen_t *aug_state, double shape); extern double legacy_exponential(aug_bitgen_t *aug_state, double scale); +extern double legacy_vonmises(bitgen_t *bitgen_state, double mu, double kappa); extern int64_t legacy_random_binomial(bitgen_t *bitgen_state, double p, int64_t n, binomial_t *binomial); extern int64_t legacy_negative_binomial(aug_bitgen_t *aug_state, double n, @@ -38,7 +40,7 @@ extern int64_t legacy_negative_binomial(aug_bitgen_t *aug_state, double n, extern int64_t legacy_random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, int64_t sample); -extern int64_t legacy_random_logseries(bitgen_t *bitgen_state, double p); +extern int64_t legacy_logseries(bitgen_t *bitgen_state, double p); extern int64_t legacy_random_poisson(bitgen_t *bitgen_state, double lam); extern int64_t legacy_random_zipf(bitgen_t *bitgen_state, double a); extern int64_t legacy_random_geometric(bitgen_t *bitgen_state, double p); diff --git a/numpy/random/mtrand.pyi b/numpy/random/mtrand.pyi new file mode 100644 index 000000000000..3137b0a955ad --- /dev/null +++ b/numpy/random/mtrand.pyi @@ -0,0 +1,579 @@ +import sys +from typing import Any, Callable, Dict, Optional, Tuple, Type, Union, overload + +from numpy import ( + bool_, + dtype, + float32, + float64, + int8, + int16, + int32, + int64, + int_, + ndarray, + uint, + uint8, + uint16, + uint32, + uint64, +) +from numpy.random.bit_generator import BitGenerator +from numpy.typing import ( + ArrayLike, + _ArrayLikeFloat_co, + _ArrayLikeInt_co, + _DoubleCodes, + _DTypeLikeBool, + _DTypeLikeInt, + _DTypeLikeUInt, + _Float32Codes, + _Float64Codes, + _Int8Codes, + _Int16Codes, + _Int32Codes, + _Int64Codes, + _IntCodes, + _ShapeLike, + _SingleCodes, + _SupportsDType, + _UInt8Codes, + _UInt16Codes, + _UInt32Codes, + _UInt64Codes, + _UIntCodes, +) + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal + +_DTypeLikeFloat32 = Union[ + dtype[float32], + _SupportsDType[dtype[float32]], + Type[float32], + _Float32Codes, + _SingleCodes, +] + +_DTypeLikeFloat64 = Union[ + dtype[float64], + _SupportsDType[dtype[float64]], + Type[float], + Type[float64], + _Float64Codes, + _DoubleCodes, +] + +class RandomState: + _bit_generator: BitGenerator + def __init__(self, seed: Union[None, _ArrayLikeInt_co, BitGenerator] = ...) -> None: ... + def __repr__(self) -> str: ... + def __str__(self) -> str: ... + def __getstate__(self) -> Dict[str, Any]: ... + def __setstate__(self, state: Dict[str, Any]) -> None: ... + def __reduce__(self) -> Tuple[Callable[[str], RandomState], Tuple[str], Dict[str, Any]]: ... + def seed(self, seed: Optional[_ArrayLikeFloat_co] = ...) -> None: ... + @overload + def get_state(self, legacy: Literal[False] = ...) -> Dict[str, Any]: ... + @overload + def get_state( + self, legacy: Literal[True] = ... + ) -> Union[Dict[str, Any], Tuple[str, ndarray[Any, dtype[uint32]], int, int, float]]: ... + def set_state( + self, state: Union[Dict[str, Any], Tuple[str, ndarray[Any, dtype[uint32]], int, int, float]] + ) -> None: ... + @overload + def random_sample(self, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def random_sample(self, size: _ShapeLike = ...) -> ndarray[Any, dtype[float64]]: ... + @overload + def random(self, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def random(self, size: _ShapeLike = ...) -> ndarray[Any, dtype[float64]]: ... + @overload + def beta(self, a: float, b: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def beta( + self, a: _ArrayLikeFloat_co, b: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def exponential(self, scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def exponential( + self, scale: _ArrayLikeFloat_co = ..., size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_exponential(self, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def standard_exponential(self, size: _ShapeLike = ...) -> ndarray[Any, dtype[float64]]: ... + @overload + def tomaxint(self, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def tomaxint(self, size: _ShapeLike = ...) -> ndarray[Any, dtype[int_]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: int, + high: Optional[int] = ..., + ) -> int: ... + @overload + def randint( # type: ignore[misc] + self, + low: int, + high: Optional[int] = ..., + size: None = ..., + dtype: _DTypeLikeBool = ..., + ) -> bool: ... + @overload + def randint( # type: ignore[misc] + self, + low: int, + high: Optional[int] = ..., + size: None = ..., + dtype: Union[_DTypeLikeInt, _DTypeLikeUInt] = ..., + ) -> int: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: _DTypeLikeBool = ..., + ) -> ndarray[Any, dtype[bool_]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[int8], Type[int8], _Int8Codes, _SupportsDType[dtype[int8]]] = ..., + ) -> ndarray[Any, dtype[int8]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[int16], Type[int16], _Int16Codes, _SupportsDType[dtype[int16]]] = ..., + ) -> ndarray[Any, dtype[int16]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[int32], Type[int32], _Int32Codes, _SupportsDType[dtype[int32]]] = ..., + ) -> ndarray[Any, dtype[Union[int32]]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Optional[ + Union[dtype[int64], Type[int64], _Int64Codes, _SupportsDType[dtype[int64]]] + ] = ..., + ) -> ndarray[Any, dtype[int64]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[uint8], Type[uint8], _UInt8Codes, _SupportsDType[dtype[uint8]]] = ..., + ) -> ndarray[Any, dtype[uint8]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[uint16], Type[uint16], _UInt16Codes, _SupportsDType[dtype[uint16]] + ] = ..., + ) -> ndarray[Any, dtype[Union[uint16]]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[uint32], Type[uint32], _UInt32Codes, _SupportsDType[dtype[uint32]] + ] = ..., + ) -> ndarray[Any, dtype[uint32]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[uint64], Type[uint64], _UInt64Codes, _SupportsDType[dtype[uint64]] + ] = ..., + ) -> ndarray[Any, dtype[uint64]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[ + dtype[int_], Type[int], Type[int_], _IntCodes, _SupportsDType[dtype[int_]] + ] = ..., + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def randint( # type: ignore[misc] + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + dtype: Union[dtype[uint], Type[uint], _UIntCodes, _SupportsDType[dtype[uint]]] = ..., + ) -> ndarray[Any, dtype[uint]]: ... + def bytes(self, length: int) -> bytes: ... + @overload + def choice( + self, + a: int, + size: None = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + ) -> int: ... + @overload + def choice( + self, + a: int, + size: _ShapeLike = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def choice( + self, + a: ArrayLike, + size: None = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + ) -> Any: ... + @overload + def choice( + self, + a: ArrayLike, + size: _ShapeLike = ..., + replace: bool = ..., + p: Optional[_ArrayLikeFloat_co] = ..., + ) -> ndarray[Any, Any]: ... + @overload + def uniform(self, low: float = ..., high: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def uniform( + self, + low: _ArrayLikeFloat_co = ..., + high: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def rand(self) -> float: ... + @overload + def rand(self, *args: int) -> ndarray[Any, dtype[float64]]: ... + @overload + def randn(self) -> float: ... + @overload + def randn(self, *args: int) -> ndarray[Any, dtype[float64]]: ... + @overload + def random_integers(self, low: int, high: Optional[int] = ..., size: None = ...) -> int: ... # type: ignore[misc] + @overload + def random_integers( + self, + low: _ArrayLikeInt_co, + high: Optional[_ArrayLikeInt_co] = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def standard_normal(self, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def standard_normal( # type: ignore[misc] + self, size: _ShapeLike = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def normal(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def normal( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_gamma( # type: ignore[misc] + self, + shape: float, + size: None = ..., + ) -> float: ... + @overload + def standard_gamma( + self, + shape: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def gamma(self, shape: float, scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def gamma( + self, + shape: _ArrayLikeFloat_co, + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def f(self, dfnum: float, dfden: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def f( + self, dfnum: _ArrayLikeFloat_co, dfden: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def noncentral_f(self, dfnum: float, dfden: float, nonc: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def noncentral_f( + self, + dfnum: _ArrayLikeFloat_co, + dfden: _ArrayLikeFloat_co, + nonc: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def chisquare(self, df: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def chisquare( + self, df: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def noncentral_chisquare(self, df: float, nonc: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def noncentral_chisquare( + self, df: _ArrayLikeFloat_co, nonc: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_t(self, df: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def standard_t( + self, df: _ArrayLikeFloat_co, size: None = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_t( + self, df: _ArrayLikeFloat_co, size: _ShapeLike = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def vonmises(self, mu: float, kappa: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def vonmises( + self, mu: _ArrayLikeFloat_co, kappa: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def pareto(self, a: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def pareto( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def weibull(self, a: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def weibull( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def power(self, a: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def power( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def standard_cauchy(self, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def standard_cauchy(self, size: _ShapeLike = ...) -> ndarray[Any, dtype[float64]]: ... + @overload + def laplace(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def laplace( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def gumbel(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def gumbel( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def logistic(self, loc: float = ..., scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def logistic( + self, + loc: _ArrayLikeFloat_co = ..., + scale: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def lognormal(self, mean: float = ..., sigma: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def lognormal( + self, + mean: _ArrayLikeFloat_co = ..., + sigma: _ArrayLikeFloat_co = ..., + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def rayleigh(self, scale: float = ..., size: None = ...) -> float: ... # type: ignore[misc] + @overload + def rayleigh( + self, scale: _ArrayLikeFloat_co = ..., size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def wald(self, mean: float, scale: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def wald( + self, mean: _ArrayLikeFloat_co, scale: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def triangular(self, left: float, mode: float, right: float, size: None = ...) -> float: ... # type: ignore[misc] + @overload + def triangular( + self, + left: _ArrayLikeFloat_co, + mode: _ArrayLikeFloat_co, + right: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[float64]]: ... + @overload + def binomial(self, n: int, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def binomial( + self, n: _ArrayLikeInt_co, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def negative_binomial(self, n: float, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def negative_binomial( + self, n: _ArrayLikeFloat_co, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def poisson(self, lam: float = ..., size: None = ...) -> int: ... # type: ignore[misc] + @overload + def poisson( + self, lam: _ArrayLikeFloat_co = ..., size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def zipf(self, a: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def zipf( + self, a: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def geometric(self, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def geometric( + self, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def hypergeometric(self, ngood: int, nbad: int, nsample: int, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def hypergeometric( + self, + ngood: _ArrayLikeInt_co, + nbad: _ArrayLikeInt_co, + nsample: _ArrayLikeInt_co, + size: Optional[_ShapeLike] = ..., + ) -> ndarray[Any, dtype[int_]]: ... + @overload + def logseries(self, p: float, size: None = ...) -> int: ... # type: ignore[misc] + @overload + def logseries( + self, p: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + def multivariate_normal( + self, + mean: _ArrayLikeFloat_co, + cov: _ArrayLikeFloat_co, + size: Optional[_ShapeLike] = ..., + check_valid: Literal["warn", "raise", "ignore"] = ..., + tol: float = ..., + ) -> ndarray[Any, dtype[float64]]: ... + def multinomial( + self, n: _ArrayLikeInt_co, pvals: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[int_]]: ... + def dirichlet( + self, alpha: _ArrayLikeFloat_co, size: Optional[_ShapeLike] = ... + ) -> ndarray[Any, dtype[float64]]: ... + def shuffle(self, x: ArrayLike) -> None: ... + @overload + def permutation(self, x: int) -> ndarray[Any, dtype[int_]]: ... + @overload + def permutation(self, x: ArrayLike) -> ndarray[Any, Any]: ... + +_rand: RandomState + +beta = _rand.beta +binomial = _rand.binomial +bytes = _rand.bytes +chisquare = _rand.chisquare +choice = _rand.choice +dirichlet = _rand.dirichlet +exponential = _rand.exponential +f = _rand.f +gamma = _rand.gamma +get_state = _rand.get_state +geometric = _rand.geometric +gumbel = _rand.gumbel +hypergeometric = _rand.hypergeometric +laplace = _rand.laplace +logistic = _rand.logistic +lognormal = _rand.lognormal +logseries = _rand.logseries +multinomial = _rand.multinomial +multivariate_normal = _rand.multivariate_normal +negative_binomial = _rand.negative_binomial +noncentral_chisquare = _rand.noncentral_chisquare +noncentral_f = _rand.noncentral_f +normal = _rand.normal +pareto = _rand.pareto +permutation = _rand.permutation +poisson = _rand.poisson +power = _rand.power +rand = _rand.rand +randint = _rand.randint +randn = _rand.randn +random = _rand.random +random_integers = _rand.random_integers +random_sample = _rand.random_sample +rayleigh = _rand.rayleigh +seed = _rand.seed +set_state = _rand.set_state +shuffle = _rand.shuffle +standard_cauchy = _rand.standard_cauchy +standard_exponential = _rand.standard_exponential +standard_gamma = _rand.standard_gamma +standard_normal = _rand.standard_normal +standard_t = _rand.standard_t +triangular = _rand.triangular +uniform = _rand.uniform +vonmises = _rand.vonmises +wald = _rand.wald +weibull = _rand.weibull +zipf = _rand.zipf +# Two legacy that are trivial wrappers around random_sample +sample = _rand.random_sample +ranf = _rand.random_sample diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index d43e7f5aa6d9..863879a0465f 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -2,6 +2,7 @@ #cython: wraparound=False, nonecheck=False, boundscheck=False, cdivision=True, language_level=3 import operator import warnings +from collections.abc import Sequence import numpy as np @@ -50,7 +51,6 @@ cdef extern from "numpy/random/distributions.h": void random_standard_uniform_fill(bitgen_t* bitgen_state, np.npy_intp cnt, double *out) nogil int64_t random_positive_int(bitgen_t *bitgen_state) nogil double random_uniform(bitgen_t *bitgen_state, double lower, double range) nogil - double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil double random_laplace(bitgen_t *bitgen_state, double loc, double scale) nogil double random_gumbel(bitgen_t *bitgen_state, double loc, double scale) nogil double random_logistic(bitgen_t *bitgen_state, double loc, double scale) nogil @@ -79,6 +79,7 @@ cdef extern from "include/legacy-distributions.h": double legacy_gamma(aug_bitgen_t *aug_state, double shape, double scale) nogil double legacy_power(aug_bitgen_t *aug_state, double a) nogil double legacy_chisquare(aug_bitgen_t *aug_state, double df) nogil + double legacy_rayleigh(aug_bitgen_t *aug_state, double mode) nogil double legacy_noncentral_chisquare(aug_bitgen_t *aug_state, double df, double nonc) nogil double legacy_noncentral_f(aug_bitgen_t *aug_state, double dfnum, double dfden, @@ -89,7 +90,7 @@ cdef extern from "include/legacy-distributions.h": int64_t n, binomial_t *binomial) nogil int64_t legacy_negative_binomial(aug_bitgen_t *aug_state, double n, double p) nogil int64_t legacy_random_hypergeometric(bitgen_t *bitgen_state, int64_t good, int64_t bad, int64_t sample) nogil - int64_t legacy_random_logseries(bitgen_t *bitgen_state, double p) nogil + int64_t legacy_logseries(bitgen_t *bitgen_state, double p) nogil int64_t legacy_random_poisson(bitgen_t *bitgen_state, double lam) nogil int64_t legacy_random_zipf(bitgen_t *bitgen_state, double a) nogil int64_t legacy_random_geometric(bitgen_t *bitgen_state, double p) nogil @@ -99,6 +100,7 @@ cdef extern from "include/legacy-distributions.h": double legacy_f(aug_bitgen_t *aug_state, double dfnum, double dfden) nogil double legacy_exponential(aug_bitgen_t *aug_state, double scale) nogil double legacy_power(aug_bitgen_t *state, double a) nogil + double legacy_vonmises(bitgen_t *bitgen_state, double mu, double kappa) nogil np.import_array() @@ -783,7 +785,7 @@ cdef class RandomState: Returns ------- - out : str + out : bytes String of length `length`. See Also @@ -818,17 +820,18 @@ cdef class RandomState: ---------- a : 1-D array-like or int If an ndarray, a random sample is generated from its elements. - If an int, the random sample is generated as if a were np.arange(a) + If an int, the random sample is generated as if it were ``np.arange(a)`` size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. Default is None, in which case a single value is returned. replace : boolean, optional - Whether the sample is with or without replacement + Whether the sample is with or without replacement. Default is True, + meaning that a value of ``a`` can be selected multiple times. p : 1-D array-like, optional The probabilities associated with each entry in a. - If not given the sample assumes a uniform distribution over all - entries in a. + If not given, the sample assumes a uniform distribution over all + entries in ``a``. Returns ------- @@ -851,6 +854,10 @@ cdef class RandomState: Notes ----- + Setting user-specified probabilities through ``p`` uses a more general but less + efficient sampler than the default. The general sampler produces a different sample + than the optimized sampler even if each element of ``p`` is 1 / len(a). + Sampling random rows from a 2-D array is not possible with this function, but is possible with `Generator.choice` through its ``axis`` keyword. @@ -2146,33 +2153,45 @@ cdef class RandomState: ... 7515, 8230, 8770]) Does their energy intake deviate systematically from the recommended - value of 7725 kJ? - - We have 10 degrees of freedom, so is the sample mean within 95% of the - recommended value? + value of 7725 kJ? Our null hypothesis will be the absence of deviation, + and the alternate hypothesis will be the presence of an effect that could be + either positive or negative, hence making our test 2-tailed. + + Because we are estimating the mean and we have N=11 values in our sample, + we have N-1=10 degrees of freedom. We set our significance level to 95% and + compute the t statistic using the empirical mean and empirical standard + deviation of our intake. We use a ddof of 1 to base the computation of our + empirical standard deviation on an unbiased estimate of the variance (note: + the final estimate is not unbiased due to the concave nature of the square + root). - >>> s = np.random.standard_t(10, size=100000) >>> np.mean(intake) 6753.636363636364 >>> intake.std(ddof=1) 1142.1232221373727 + >>> t = (np.mean(intake)-7725)/(intake.std(ddof=1)/np.sqrt(len(intake))) + >>> t + -2.8207540608310198 - Calculate the t statistic, setting the ddof parameter to the unbiased - value so the divisor in the standard deviation will be degrees of - freedom, N-1. + We draw 1000000 samples from Student's t distribution with the adequate + degrees of freedom. - >>> t = (np.mean(intake)-7725)/(intake.std(ddof=1)/np.sqrt(len(intake))) >>> import matplotlib.pyplot as plt + >>> s = np.random.standard_t(10, size=1000000) >>> h = plt.hist(s, bins=100, density=True) - For a one-sided t-test, how far out in the distribution does the t - statistic appear? + Does our t statistic land in one of the two critical regions found at + both tails of the distribution? - >>> np.sum(s>> np.sum(np.abs(t) < np.abs(s)) / float(len(s)) + 0.018318 #random < 0.05, statistic is in critical region - So the p-value is about 0.009, which says the null hypothesis has a - probability of about 99% of being true. + The probability value for this 2-tailed test is about 1.83%, which is + lower than the 5% pre-determined significance threshold. + + Therefore, the probability of observing values as extreme as our intake + conditionally on the null hypothesis being true is too low, and we reject + the null hypothesis of no deviation. """ return cont(&legacy_standard_t, &self._aug_state, size, self.lock, 1, @@ -2264,7 +2283,7 @@ cdef class RandomState: >>> plt.show() """ - return cont(&random_vonmises, &self._bitgen, size, self.lock, 2, + return cont(&legacy_vonmises, &self._bitgen, size, self.lock, 2, mu, 'mu', CONS_NONE, kappa, 'kappa', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, None) @@ -3069,7 +3088,7 @@ cdef class RandomState: 0.087300000000000003 # random """ - return cont(&random_rayleigh, &self._bitgen, size, self.lock, 1, + return cont(&legacy_rayleigh, &self._bitgen, size, self.lock, 1, scale, 'scale', CONS_NON_NEGATIVE, 0.0, '', CONS_NONE, 0.0, '', CONS_NONE, None) @@ -3511,8 +3530,9 @@ cdef class RandomState: Parameters ---------- lam : float or array_like of floats - Expectation of interval, must be >= 0. A sequence of expectation - intervals must be broadcastable over the requested size. + Expected number of events occurring in a fixed-time interval, + must be >= 0. A sequence must be broadcastable over the requested + size. size : int or tuple of ints, optional Output shape. If the given shape is, e.g., ``(m, n, k)``, then ``m * n * k`` samples are drawn. If size is ``None`` (default), @@ -3936,7 +3956,7 @@ cdef class RandomState: >>> plt.show() """ - out = disc(&legacy_random_logseries, &self._bitgen, size, self.lock, 1, 0, + out = disc(&legacy_logseries, &self._bitgen, size, self.lock, 1, 0, p, 'p', CONS_BOUNDED_0_1, 0.0, '', CONS_NONE, 0.0, '', CONS_NONE) @@ -4214,7 +4234,20 @@ cdef class RandomState: pix = np.PyArray_DATA(parr) check_array_constraint(parr, 'pvals', CONS_BOUNDED_0_1) if kahan_sum(pix, d-1) > (1.0 + 1e-12): - raise ValueError("sum(pvals[:-1]) > 1.0") + # When floating, but not float dtype, and close, improve the error + # 1.0001 works for float16 and float32 + if (isinstance(pvals, np.ndarray) + and np.issubdtype(pvals.dtype, np.floating) + and pvals.dtype != float + and pvals.sum() < 1.0001): + msg = ("sum(pvals[:-1].astype(np.float64)) > 1.0. The pvals " + "array is cast to 64-bit floating point prior to " + "checking the sum. Precision changes when casting may " + "cause problems even if the sum of the original pvals " + "is valid.") + else: + msg = "sum(pvals[:-1]) > 1.0" + raise ValueError(msg) if size is None: shape = (d,) @@ -4402,8 +4435,8 @@ cdef class RandomState: Parameters ---------- - x : array_like - The array or list to be shuffled. + x : ndarray or MutableSequence + The array, list or mutable sequence to be shuffled. Returns ------- @@ -4456,7 +4489,22 @@ cdef class RandomState: self._shuffle_raw(n, sizeof(np.npy_intp), stride, x_ptr, buf_ptr) else: self._shuffle_raw(n, itemsize, stride, x_ptr, buf_ptr) - elif isinstance(x, np.ndarray) and x.ndim and x.size: + elif isinstance(x, np.ndarray): + if x.size == 0: + # shuffling is a no-op + return + + if x.ndim == 1 and x.dtype.type is np.object_: + warnings.warn( + "Shuffling a one dimensional array subclass containing " + "objects gives incorrect results for most array " + "subclasses. " + "Please us the new random number API instead: " + "https://numpy.org/doc/stable/reference/random/index.html\n" + "The new API fixes this issue. This version will not " + "be fixed due to stability guarantees of the API.", + UserWarning, stacklevel=1) # Cython adds no stacklevel + buf = np.empty_like(x[0, ...]) with self.lock: for i in reversed(range(1, n)): @@ -4468,6 +4516,16 @@ cdef class RandomState: x[i] = buf else: # Untyped path. + if not isinstance(x, Sequence): + # See gh-18206. We may decide to deprecate here in the future. + warnings.warn( + f"you are shuffling a '{type(x).__name__}' object " + "which is not a subclass of 'Sequence'; " + "`shuffle` is not guaranteed to behave correctly. " + "E.g., non-numpy array/tensor objects with view semantics " + "may contain duplicates after shuffling.", + UserWarning, stacklevel=1) # Cython does not add a level + with self.lock: for i in reversed(range(1, n)): j = random_interval(&self._bitgen, i) diff --git a/numpy/random/setup.py b/numpy/random/setup.py index bfd08e4691aa..dce9a101ebce 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -23,7 +23,7 @@ def generate_libraries(ext, build_dir): # enable unix large file support on 32 bit systems # (64 bit off_t, lseek -> lseek64 etc.) - if sys.platform[:3] == "aix": + if sys.platform[:3] == 'aix': defs = [('_LARGE_FILES', None)] else: defs = [('_FILE_OFFSET_BITS', '64'), @@ -116,7 +116,7 @@ def generate_libraries(ext, build_dir): # gen.pyx, src/distributions/distributions.c config.add_extension(gen, sources=[f'{gen}.c'], - libraries=EXTRA_LIBRARIES, + libraries=EXTRA_LIBRARIES + ['npymath'], extra_compile_args=EXTRA_COMPILE_ARGS, include_dirs=['.', 'src'], extra_link_args=EXTRA_LINK_ARGS, @@ -124,13 +124,14 @@ def generate_libraries(ext, build_dir): define_macros=defs, ) config.add_data_files('_bounded_integers.pxd') + mtrand_libs = ['m', 'npymath'] if os.name != 'nt' else ['npymath'] config.add_extension('mtrand', sources=['mtrand.c', 'src/legacy/legacy-distributions.c', 'src/distributions/distributions.c', ], include_dirs=['.', 'src', 'src/legacy'], - libraries=['m'] if os.name != 'nt' else [], + libraries=mtrand_libs, extra_compile_args=EXTRA_COMPILE_ARGS, extra_link_args=EXTRA_LINK_ARGS, depends=depends + ['mtrand.pyx'], diff --git a/numpy/random/src/distributions/distributions.c b/numpy/random/src/distributions/distributions.c index 93e0bdc5f6d2..9bdfa9bead82 100644 --- a/numpy/random/src/distributions/distributions.c +++ b/numpy/random/src/distributions/distributions.c @@ -47,7 +47,7 @@ static double standard_exponential_unlikely(bitgen_t *bitgen_state, uint8_t idx, double x) { if (idx == 0) { /* Switch to 1.0 - U to avoid log(0.0), see GH 13361 */ - return ziggurat_exp_r - log(1.0 - next_double(bitgen_state)); + return ziggurat_exp_r - npy_log1p(-next_double(bitgen_state)); } else if ((fe_double[idx - 1] - fe_double[idx]) * next_double(bitgen_state) + fe_double[idx] < exp(-x)) { @@ -84,7 +84,7 @@ static float standard_exponential_unlikely_f(bitgen_t *bitgen_state, uint8_t idx, float x) { if (idx == 0) { /* Switch to 1.0 - U to avoid log(0.0), see GH 13361 */ - return ziggurat_exp_r_f - logf(1.0f - next_float(bitgen_state)); + return ziggurat_exp_r_f - npy_log1pf(-next_float(bitgen_state)); } else if ((fe_float[idx - 1] - fe_float[idx]) * next_float(bitgen_state) + fe_float[idx] < expf(-x)) { @@ -121,7 +121,7 @@ void random_standard_exponential_inv_fill(bitgen_t * bitgen_state, npy_intp cnt, { npy_intp i; for (i = 0; i < cnt; i++) { - out[i] = -log(1.0 - next_double(bitgen_state)); + out[i] = -npy_log1p(-next_double(bitgen_state)); } } @@ -129,7 +129,7 @@ void random_standard_exponential_inv_fill_f(bitgen_t * bitgen_state, npy_intp cn { npy_intp i; for (i = 0; i < cnt; i++) { - out[i] = -log(1.0 - next_float(bitgen_state)); + out[i] = -npy_log1p(-next_float(bitgen_state)); } } @@ -155,8 +155,8 @@ double random_standard_normal(bitgen_t *bitgen_state) { if (idx == 0) { for (;;) { /* Switch to 1.0 - U to avoid log(0.0), see GH 13361 */ - xx = -ziggurat_nor_inv_r * log(1.0 - next_double(bitgen_state)); - yy = -log(1.0 - next_double(bitgen_state)); + xx = -ziggurat_nor_inv_r * npy_log1p(-next_double(bitgen_state)); + yy = -npy_log1p(-next_double(bitgen_state)); if (yy + yy > xx * xx) return ((rabs >> 8) & 0x1) ? -(ziggurat_nor_r + xx) : ziggurat_nor_r + xx; @@ -196,8 +196,8 @@ float random_standard_normal_f(bitgen_t *bitgen_state) { if (idx == 0) { for (;;) { /* Switch to 1.0 - U to avoid log(0.0), see GH 13361 */ - xx = -ziggurat_nor_inv_r_f * logf(1.0f - next_float(bitgen_state)); - yy = -logf(1.0f - next_float(bitgen_state)); + xx = -ziggurat_nor_inv_r_f * npy_log1pf(-next_float(bitgen_state)); + yy = -npy_log1pf(-next_float(bitgen_state)); if (yy + yy > xx * xx) return ((rabs >> 8) & 0x1) ? -(ziggurat_nor_r_f + xx) : ziggurat_nor_r_f + xx; @@ -508,7 +508,7 @@ double random_lognormal(bitgen_t *bitgen_state, double mean, double sigma) { } double random_rayleigh(bitgen_t *bitgen_state, double mode) { - return mode * sqrt(-2.0 * log(1.0 - next_double(bitgen_state))); + return mode * sqrt(2.0 * random_standard_exponential(bitgen_state)); } double random_standard_t(bitgen_t *bitgen_state, double df) { @@ -843,6 +843,7 @@ double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) { return NPY_NAN; } if (kappa < 1e-8) { + /* Use a uniform for very small values of kappa */ return M_PI * (2 * next_double(bitgen_state) - 1); } else { /* with double precision rho is zero until 1.4e-8 */ @@ -853,9 +854,23 @@ double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) { */ s = (1. / kappa + kappa); } else { - double r = 1 + sqrt(1 + 4 * kappa * kappa); - double rho = (r - sqrt(2 * r)) / (2 * kappa); - s = (1 + rho * rho) / (2 * rho); + if (kappa <= 1e6) { + /* Path for 1e-5 <= kappa <= 1e6 */ + double r = 1 + sqrt(1 + 4 * kappa * kappa); + double rho = (r - sqrt(2 * r)) / (2 * kappa); + s = (1 + rho * rho) / (2 * rho); + } else { + /* Fallback to wrapped normal distribution for kappa > 1e6 */ + result = mu + sqrt(1. / kappa) * random_standard_normal(bitgen_state); + /* Ensure result is within bounds */ + if (result < -M_PI) { + result += 2*M_PI; + } + if (result > M_PI) { + result -= 2*M_PI; + } + return result; + } } while (1) { @@ -891,17 +906,11 @@ double random_vonmises(bitgen_t *bitgen_state, double mu, double kappa) { } } -/* - * RAND_INT_TYPE is used to share integer generators with RandomState which - * used long in place of int64_t. If changing a distribution that uses - * RAND_INT_TYPE, then the original unmodified copy must be retained for - * use in RandomState by copying to the legacy distributions source file. - */ -RAND_INT_TYPE random_logseries(bitgen_t *bitgen_state, double p) { +int64_t random_logseries(bitgen_t *bitgen_state, double p) { double q, r, U, V; - RAND_INT_TYPE result; + int64_t result; - r = log(1.0 - p); + r = npy_log1p(-p); while (1) { V = next_double(bitgen_state); @@ -911,7 +920,7 @@ RAND_INT_TYPE random_logseries(bitgen_t *bitgen_state, double p) { U = next_double(bitgen_state); q = 1.0 - exp(r * U); if (V <= q * q) { - result = (RAND_INT_TYPE)floor(1 + log(V) / log(q)); + result = (int64_t)floor(1 + log(V) / log(q)); if ((result < 1) || (V == 0.0)) { continue; } else { @@ -925,6 +934,14 @@ RAND_INT_TYPE random_logseries(bitgen_t *bitgen_state, double p) { } } +/* + * RAND_INT_TYPE is used to share integer generators with RandomState which + * used long in place of int64_t. If changing a distribution that uses + * RAND_INT_TYPE, then the original unmodified copy must be retained for + * use in RandomState by copying to the legacy distributions source file. + */ + +/* Still used but both generator and mtrand via legacy_random_geometric */ RAND_INT_TYPE random_geometric_search(bitgen_t *bitgen_state, double p) { double U; RAND_INT_TYPE X; @@ -942,11 +959,11 @@ RAND_INT_TYPE random_geometric_search(bitgen_t *bitgen_state, double p) { return X; } -RAND_INT_TYPE random_geometric_inversion(bitgen_t *bitgen_state, double p) { - return (RAND_INT_TYPE)ceil(log(1.0 - next_double(bitgen_state)) / log(1.0 - p)); +int64_t random_geometric_inversion(bitgen_t *bitgen_state, double p) { + return (int64_t)ceil(-random_standard_exponential(bitgen_state) / npy_log1p(-p)); } -RAND_INT_TYPE random_geometric(bitgen_t *bitgen_state, double p) { +int64_t random_geometric(bitgen_t *bitgen_state, double p) { if (p >= 0.333333333333333333333333) { return random_geometric_search(bitgen_state, p); } else { @@ -971,7 +988,7 @@ RAND_INT_TYPE random_zipf(bitgen_t *bitgen_state, double a) { * just reject this value. This function then models a Zipf * distribution truncated to sys.maxint. */ - if (X > RAND_INT_MAX || X < 1.0) { + if (X > (double)RAND_INT_MAX || X < 1.0) { continue; } diff --git a/numpy/random/src/legacy/legacy-distributions.c b/numpy/random/src/legacy/legacy-distributions.c index fd067fe8dc2b..443c1a4bf781 100644 --- a/numpy/random/src/legacy/legacy-distributions.c +++ b/numpy/random/src/legacy/legacy-distributions.c @@ -1,3 +1,13 @@ +/* + * This file contains generation code for distribution that have been modified + * since Generator was introduced. These are preserved using identical code + * to what was in NumPy 1.16 so that the stream of values generated by + * RandomState is not changed when there are changes that affect Generator. + * + * These functions should not be changed except if they contain code that + * cannot be compiled. They should not be changed for bug fixes, performance + * improvements that can change the values produced, or enhancements to precision. + */ #include "include/legacy-distributions.h" @@ -102,6 +112,10 @@ double legacy_chisquare(aug_bitgen_t *aug_state, double df) { return 2.0 * legacy_standard_gamma(aug_state, df / 2.0); } +double legacy_rayleigh(bitgen_t *bitgen_state, double mode) { + return mode * sqrt(-2.0 * npy_log1p(-next_double(bitgen_state))); +} + double legacy_noncentral_chisquare(aug_bitgen_t *aug_state, double df, double nonc) { double out; @@ -369,24 +383,115 @@ int64_t legacy_random_hypergeometric(bitgen_t *bitgen_state, int64_t good, } -int64_t legacy_random_logseries(bitgen_t *bitgen_state, double p) { - return (int64_t)random_logseries(bitgen_state, p); -} - - int64_t legacy_random_poisson(bitgen_t *bitgen_state, double lam) { +int64_t legacy_random_poisson(bitgen_t *bitgen_state, double lam) { return (int64_t)random_poisson(bitgen_state, lam); } - int64_t legacy_random_zipf(bitgen_t *bitgen_state, double a) { +int64_t legacy_random_zipf(bitgen_t *bitgen_state, double a) { return (int64_t)random_zipf(bitgen_state, a); } - int64_t legacy_random_geometric(bitgen_t *bitgen_state, double p) { - return (int64_t)random_geometric(bitgen_state, p); + +static long legacy_geometric_inversion(bitgen_t *bitgen_state, double p) { + return (long)ceil(npy_log1p(-next_double(bitgen_state)) / log(1 - p)); } - void legacy_random_multinomial(bitgen_t *bitgen_state, RAND_INT_TYPE n, +int64_t legacy_random_geometric(bitgen_t *bitgen_state, double p) { + if (p >= 0.333333333333333333333333) { + return (int64_t)random_geometric_search(bitgen_state, p); + } else { + return (int64_t)legacy_geometric_inversion(bitgen_state, p); + } +} + +void legacy_random_multinomial(bitgen_t *bitgen_state, RAND_INT_TYPE n, RAND_INT_TYPE *mnix, double *pix, npy_intp d, binomial_t *binomial) { return random_multinomial(bitgen_state, n, mnix, pix, d, binomial); } + +double legacy_vonmises(bitgen_t *bitgen_state, double mu, double kappa) { + double s; + double U, V, W, Y, Z; + double result, mod; + int neg; + if (npy_isnan(kappa)) { + return NPY_NAN; + } + if (kappa < 1e-8) { + return M_PI * (2 * next_double(bitgen_state) - 1); + } else { + /* with double precision rho is zero until 1.4e-8 */ + if (kappa < 1e-5) { + /* + * second order taylor expansion around kappa = 0 + * precise until relatively large kappas as second order is 0 + */ + s = (1. / kappa + kappa); + } else { + /* Path for 1e-5 <= kappa <= 1e6 */ + double r = 1 + sqrt(1 + 4 * kappa * kappa); + double rho = (r - sqrt(2 * r)) / (2 * kappa); + s = (1 + rho * rho) / (2 * rho); + } + + while (1) { + U = next_double(bitgen_state); + Z = cos(M_PI * U); + W = (1 + s * Z) / (s + Z); + Y = kappa * (s - W); + V = next_double(bitgen_state); + /* + * V==0.0 is ok here since Y >= 0 always leads + * to accept, while Y < 0 always rejects + */ + if ((Y * (2 - Y) - V >= 0) || (log(Y / V) + 1 - Y >= 0)) { + break; + } + } + + U = next_double(bitgen_state); + + result = acos(W); + if (U < 0.5) { + result = -result; + } + result += mu; + neg = (result < 0); + mod = fabs(result); + mod = (fmod(mod + M_PI, 2 * M_PI) - M_PI); + if (neg) { + mod *= -1; + } + + return mod; + } +} + +int64_t legacy_logseries(bitgen_t *bitgen_state, double p) { + double q, r, U, V; + long result; + + r = log(1.0 - p); + + while (1) { + V = next_double(bitgen_state); + if (V >= p) { + return 1; + } + U = next_double(bitgen_state); + q = 1.0 - exp(r * U); + if (V <= q * q) { + result = (long)floor(1 + log(V) / log(q)); + if ((result < 1) || (V == 0.0)) { + continue; + } else { + return (int64_t)result; + } + } + if (V >= q) { + return 1; + } + return 2; + } +} \ No newline at end of file diff --git a/numpy/random/src/pcg64/pcg64.c b/numpy/random/src/pcg64/pcg64.c index b15973aefafc..c623c809b02e 100644 --- a/numpy/random/src/pcg64/pcg64.c +++ b/numpy/random/src/pcg64/pcg64.c @@ -61,6 +61,10 @@ pcg_setseq_128_xsl_rr_64_boundedrand_r(pcg_state_setseq_128 *rng, uint64_t bound); extern inline void pcg_setseq_128_advance_r(pcg_state_setseq_128 *rng, pcg128_t delta); +extern inline uint64_t pcg_cm_random_r(pcg_state_setseq_128 *rng); +extern inline void pcg_cm_step_r(pcg_state_setseq_128 *rng); +extern inline uint64_t pcg_output_cm_128_64(pcg128_t state); +extern inline void pcg_cm_srandom_r(pcg_state_setseq_128 *rng, pcg128_t initstate, pcg128_t initseq); /* Multi-step advance functions (jump-ahead, jump-back) * @@ -117,6 +121,9 @@ pcg128_t pcg_advance_lcg_128(pcg128_t state, pcg128_t delta, pcg128_t cur_mult, extern inline uint64_t pcg64_next64(pcg64_state *state); extern inline uint32_t pcg64_next32(pcg64_state *state); +extern inline uint64_t pcg64_cm_next64(pcg64_state *state); +extern inline uint32_t pcg64_cm_next32(pcg64_state *state); + extern void pcg64_advance(pcg64_state *state, uint64_t *step) { pcg128_t delta; #ifndef PCG_EMULATED_128BIT_MATH @@ -128,6 +135,17 @@ extern void pcg64_advance(pcg64_state *state, uint64_t *step) { pcg64_advance_r(state->pcg_state, delta); } +extern void pcg64_cm_advance(pcg64_state *state, uint64_t *step) { + pcg128_t delta; +#ifndef PCG_EMULATED_128BIT_MATH + delta = (((pcg128_t)step[0]) << 64) | step[1]; +#else + delta.high = step[0]; + delta.low = step[1]; +#endif + pcg_cm_advance_r(state->pcg_state, delta); +} + extern void pcg64_set_seed(pcg64_state *state, uint64_t *seed, uint64_t *inc) { pcg128_t s, i; #ifndef PCG_EMULATED_128BIT_MATH diff --git a/numpy/random/src/pcg64/pcg64.h b/numpy/random/src/pcg64/pcg64.h index 2a7217dd9ba8..90a83fd5edf9 100644 --- a/numpy/random/src/pcg64/pcg64.h +++ b/numpy/random/src/pcg64/pcg64.h @@ -57,11 +57,11 @@ #define inline __forceinline #endif -#if __GNUC_GNU_INLINE__ && !defined(__cplusplus) +#if defined(__GNUC_GNU_INLINE__) && !defined(__cplusplus) #error Nonstandard GNU inlining semantics. Compile with -std=c99 or better. #endif -#if __cplusplus +#ifdef __cplusplus extern "C" { #endif @@ -104,6 +104,9 @@ typedef struct { , PCG_128BIT_CONSTANT(0x0000000000000001ULL, 0xda3e39cb94b95bdbULL) \ } +#define PCG_CHEAP_MULTIPLIER_128 (0xda942042e4dd58b5ULL) + + static inline uint64_t pcg_rotr_64(uint64_t value, unsigned int rot) { #ifdef _WIN32 return _rotr64(value, rot); @@ -198,6 +201,77 @@ pcg_setseq_128_xsl_rr_64_random_r(pcg_state_setseq_128 *rng) { #endif } +static inline pcg128_t pcg128_mult_64(pcg128_t a, uint64_t b) { + uint64_t h1; + pcg128_t result; + + h1 = a.high * b; + _pcg_mult64(a.low, b, &(result.high), &(result.low)); + result.high += h1; + return result; +} + +static inline void pcg_cm_step_r(pcg_state_setseq_128 *rng) { +#if defined _WIN32 && _MSC_VER >= 1900 && _M_AMD64 + uint64_t h1; + pcg128_t product; + + /* Manually inline the multiplication and addition using intrinsics */ + h1 = rng->state.high * PCG_CHEAP_MULTIPLIER_128; + product.low = + _umul128(rng->state.low, PCG_CHEAP_MULTIPLIER_128, &(product.high)); + product.high += h1; + _addcarry_u64(_addcarry_u64(0, product.low, rng->inc.low, &(rng->state.low)), + product.high, rng->inc.high, &(rng->state.high)); +#else + rng->state = pcg128_add(pcg128_mult_64(rng->state, PCG_CHEAP_MULTIPLIER_128), + rng->inc); +#endif +} + + +static inline void pcg_cm_srandom_r(pcg_state_setseq_128 *rng, pcg128_t initstate, pcg128_t initseq) { + rng->state = PCG_128BIT_CONSTANT(0ULL, 0ULL); + rng->inc.high = initseq.high << 1u; + rng->inc.high |= initseq.low >> 63u; + rng->inc.low = (initseq.low << 1u) | 1u; + pcg_cm_step_r(rng); + rng->state = pcg128_add(rng->state, initstate); + pcg_cm_step_r(rng); +} + +static inline uint64_t pcg_cm_random_r(pcg_state_setseq_128* rng) +{ + /* Lots of manual inlining to help out certain compilers to generate + * performant code. */ + uint64_t hi = rng->state.high; + uint64_t lo = rng->state.low; + + /* Run the DXSM output function on the pre-iterated state. */ + lo |= 1; + hi ^= hi >> 32; + hi *= 0xda942042e4dd58b5ULL; + hi ^= hi >> 48; + hi *= lo; + + /* Run the CM step. */ +#if defined _WIN32 && _MSC_VER >= 1900 && _M_AMD64 + uint64_t h1; + pcg128_t product; + + /* Manually inline the multiplication and addition using intrinsics */ + h1 = rng->state.high * PCG_CHEAP_MULTIPLIER_128; + product.low = + _umul128(rng->state.low, PCG_CHEAP_MULTIPLIER_128, &(product.high)); + product.high += h1; + _addcarry_u64(_addcarry_u64(0, product.low, rng->inc.low, &(rng->state.low)), + product.high, rng->inc.high, &(rng->state.high)); +#else + rng->state = pcg128_add(pcg128_mult_64(rng->state, PCG_CHEAP_MULTIPLIER_128), + rng->inc); +#endif + return hi; +} #else /* PCG_EMULATED_128BIT_MATH */ static inline void pcg_setseq_128_step_r(pcg_state_setseq_128 *rng) { @@ -209,6 +283,37 @@ static inline uint64_t pcg_output_xsl_rr_128_64(pcg128_t state) { state >> 122u); } +static inline void pcg_cm_step_r(pcg_state_setseq_128 *rng) { + rng-> state = rng->state * PCG_CHEAP_MULTIPLIER_128 + rng->inc; +} + +static inline uint64_t pcg_output_cm_128_64(pcg128_t state) { + uint64_t hi = state >> 64; + uint64_t lo = state; + + lo |= 1; + hi ^= hi >> 32; + hi *= 0xda942042e4dd58b5ULL; + hi ^= hi >> 48; + hi *= lo; + return hi; +} + +static inline void pcg_cm_srandom_r(pcg_state_setseq_128 *rng, pcg128_t initstate, pcg128_t initseq) { + rng->state = 0U; + rng->inc = (initseq << 1u) | 1u; + pcg_cm_step_r(rng); + rng->state += initstate; + pcg_cm_step_r(rng); +} + +static inline uint64_t pcg_cm_random_r(pcg_state_setseq_128* rng) +{ + uint64_t ret = pcg_output_cm_128_64(rng->state); + pcg_cm_step_r(rng); + return ret; +} + static inline uint64_t pcg_setseq_128_xsl_rr_64_random_r(pcg_state_setseq_128* rng) { @@ -248,6 +353,12 @@ static inline void pcg_setseq_128_advance_r(pcg_state_setseq_128 *rng, PCG_DEFAULT_MULTIPLIER_128, rng->inc); } +static inline void pcg_cm_advance_r(pcg_state_setseq_128 *rng, pcg128_t delta) { + rng->state = pcg_advance_lcg_128(rng->state, delta, + PCG_128BIT_CONSTANT(0, PCG_CHEAP_MULTIPLIER_128), + rng->inc); +} + typedef pcg_state_setseq_128 pcg64_random_t; #define pcg64_random_r pcg_setseq_128_xsl_rr_64_random_r #define pcg64_boundedrand_r pcg_setseq_128_xsl_rr_64_boundedrand_r @@ -255,7 +366,7 @@ typedef pcg_state_setseq_128 pcg64_random_t; #define pcg64_advance_r pcg_setseq_128_advance_r #define PCG64_INITIALIZER PCG_STATE_SETSEQ_128_INITIALIZER -#if __cplusplus +#ifdef __cplusplus } #endif @@ -281,7 +392,24 @@ static inline uint32_t pcg64_next32(pcg64_state *state) { return (uint32_t)(next & 0xffffffff); } +static inline uint64_t pcg64_cm_next64(pcg64_state *state) { + return pcg_cm_random_r(state->pcg_state); +} + +static inline uint32_t pcg64_cm_next32(pcg64_state *state) { + uint64_t next; + if (state->has_uint32) { + state->has_uint32 = 0; + return state->uinteger; + } + next = pcg_cm_random_r(state->pcg_state); + state->has_uint32 = 1; + state->uinteger = (uint32_t)(next >> 32); + return (uint32_t)(next & 0xffffffff); +} + void pcg64_advance(pcg64_state *state, uint64_t *step); +void pcg64_cm_advance(pcg64_state *state, uint64_t *step); void pcg64_set_seed(pcg64_state *state, uint64_t *seed, uint64_t *inc); diff --git a/numpy/random/tests/data/pcg64dxsm-testset-1.csv b/numpy/random/tests/data/pcg64dxsm-testset-1.csv new file mode 100644 index 000000000000..39cef057f449 --- /dev/null +++ b/numpy/random/tests/data/pcg64dxsm-testset-1.csv @@ -0,0 +1,1001 @@ +seed, 0xdeadbeaf +0, 0xdf1ddcf1e22521fe +1, 0xc71b2f9c706cf151 +2, 0x6922a8cc24ad96b2 +3, 0x82738c549beccc30 +4, 0x5e8415cdb1f17580 +5, 0x64c54ad0c09cb43 +6, 0x361a17a607dce278 +7, 0x4346f6afb7acad68 +8, 0x6e9f14d4f6398d6b +9, 0xf818d4343f8ed822 +10, 0x6327647daf508ed6 +11, 0xe1d1dbe5496a262a +12, 0xfc081e619076b2e0 +13, 0x37126563a956ab1 +14, 0x8bb46e155db16b9 +15, 0x56449f006c9f3fb4 +16, 0x34a9273550941803 +17, 0x5b4df62660f99462 +18, 0xb8665cad532e3018 +19, 0x72fc3e5f7f84216a +20, 0x71d3c47f6fd59939 +21, 0xfd4218afa1de463b +22, 0xc84054c78e0a9a71 +23, 0xae59034726be61a8 +24, 0xa6a5f21de983654d +25, 0x3b633acf572009da +26, 0x6a0884f347ab54c8 +27, 0x7a907ebe9adcab50 +28, 0xbe779be53d7b8d4a +29, 0xf5976e8c69b9dcd1 +30, 0x1d8302f114699e11 +31, 0x7d37e43042c038a0 +32, 0x2cc1d4edc2a40f35 +33, 0x83e3347bb2d581f1 +34, 0x253f8698651a844d +35, 0x4312dea0dd4e32f6 +36, 0x10f106439964ea3a +37, 0x810eb374844868cc +38, 0x366342a54b1978cc +39, 0x9fb39b13aaddfb5e +40, 0xdb91fd0d9482bed7 +41, 0x89f6ea4ca9c68204 +42, 0x146b31ccca461792 +43, 0x203fd9724deb2486 +44, 0x58a84f23748e25cb +45, 0x2f20eb6aeb94e88 +46, 0x14d3581460e473c +47, 0xad5bd0d25f37d047 +48, 0x1cf88fa16de258b2 +49, 0x3bcab6485b7a341 +50, 0xb2433b37f227d90c +51, 0x2cffd7e0a8360cc8 +52, 0x5d2eeff7c9ebc847 +53, 0x6fd7c7ae23f9f64b +54, 0x381650b2d00f175d +55, 0x9d93edcedc873cae +56, 0x56e369a033d4cb49 +57, 0x7547997116a3bac +58, 0x11debaa897fd4665 +59, 0xdf799d2b73bd6fb8 +60, 0x3747d299c66624d +61, 0xac9346701afd0cfa +62, 0xac90e150fa13c7bf +63, 0x85c56ad2248c2871 +64, 0xdea66bf35c45f195 +65, 0x59cf910ea079fb74 +66, 0x2f841bb782274586 +67, 0x9814df4384d92bd9 +68, 0x15bc70824be09925 +69, 0x16d4d0524c0503a3 +70, 0xf04ea249135c0cc7 +71, 0xa707ab509b7e3032 +72, 0x465459efa869e372 +73, 0x64cbf70a783fab67 +74, 0x36b3541a14ca8ed7 +75, 0x9a4dfae8f4c596bf +76, 0x11d9a04224281be3 +77, 0xe09bbe6d5e98ec32 +78, 0xa6c60d908973aa0d +79, 0x7c524c57dd5915c8 +80, 0xa810c170b27f1fdc +81, 0xce5d409819621583 +82, 0xfe2ee3d5332a3525 +83, 0x162fb7c8b32045eb +84, 0x4a3327156b0b2d83 +85, 0x808d0282f971064 +86, 0x2e6f04cf5ed27e60 +87, 0xaf6800699cca67a9 +88, 0xc7590aae7244c3bf +89, 0x7824345f4713f5f9 +90, 0x8f713505f8fd059b +91, 0x3d5b5b9bb6b1e80e +92, 0x8674f45e5dc40d79 +93, 0xcb1e36846aa14773 +94, 0xe0ae45b2b9b778c1 +95, 0xd7254ce931eefcfb +96, 0xef34e15e4f55ac0a +97, 0xf17cc0ba15a99bc4 +98, 0x77bb0f7ffe7b31f1 +99, 0x6ee86438d2e71d38 +100, 0x584890f86829a455 +101, 0x7baf0d8d30ba70fe +102, 0xb1ac8f326b8403ae +103, 0xcc1963435c874ba7 +104, 0x9c483b953d1334ce +105, 0xc0924bcbf3e10941 +106, 0x21bcc581558717b1 +107, 0x2c5ad1623f8d292b +108, 0xa8ea110f6124557e +109, 0x15f24a6c5c4c591 +110, 0x40fe0d9cd7629126 +111, 0xcfe8f2b3b081484d +112, 0x891383f4b4cac284 +113, 0x76f2fcdef7fa845 +114, 0x4edd12133aed0584 +115, 0xd53c06d12308873d +116, 0xf7f22882c17f86bf +117, 0xfbaa4aad72f35e10 +118, 0x627610da2e3c0cc3 +119, 0x582b16a143634d9a +120, 0x9b4a7f69ed38f4a0 +121, 0x2df694974d1e1cbe +122, 0xe5be6eaafed5d4b +123, 0xc48e2a288ad6605e +124, 0xbcb088149ce27c2b +125, 0x3cb6a7fb06ceecbe +126, 0x516735fff3b9e3ac +127, 0x5cbafc551ee5008d +128, 0xee27d1ab855c5fd5 +129, 0xc99fb341f6baf846 +130, 0x7ad8891b92058e6d +131, 0xf50310d03c1ac6c7 +132, 0x947e281d998cbd3e +133, 0x1d4d94a93824fe80 +134, 0x5568b77289e7ee73 +135, 0x7d82d1b2b41e3c8b +136, 0x1af462c7abc787b +137, 0xcfd8dfe80bfae1ef +138, 0xd314caeb723a63ea +139, 0x1c63ddcfc1145429 +140, 0x3801b7cc6cbf2437 +141, 0xc327d5b9fdafddd3 +142, 0xe140278430ca3c78 +143, 0x4d0345a685cb6ef8 +144, 0x47640dc86e261ff9 +145, 0xab817f158523ebf4 +146, 0x37c51e35fbe65a6b +147, 0xab090f475d30a178 +148, 0x4d3ec225bf599fc1 +149, 0xefd517b0041679b1 +150, 0x20ad50bca4da32c5 +151, 0x75e1f7cd07fad86d +152, 0x348cf781ee655f4b +153, 0x9375f0e5ffc2d2ec +154, 0x7689082fd5f7279c +155, 0x633e56f763561e77 +156, 0x9d1752d70861f9fd +157, 0xa3c994b4e70b0b0f +158, 0xabf7276a58701b88 +159, 0xbfa18d1a0540d000 +160, 0xc6a28a2475646d26 +161, 0x7cdf108583f65085 +162, 0x82dcefb9f32104be +163, 0xc6baadd0adc6b446 +164, 0x7a63cff01075b1b4 +165, 0x67ac62e575c89919 +166, 0x96fa4320a0942035 +167, 0xc4658859385b325f +168, 0xde22c17ff47808f6 +169, 0xbb952c4d89e2f2ec +170, 0x638251fbc55bdc37 +171, 0x38918b307a03b3ea +172, 0xccb60f2cedbb570b +173, 0x3c06f4086a28f012 +174, 0x4e8d238388986e33 +175, 0x1760b7793514a143 +176, 0xa3f924efe49ee7d6 +177, 0xaf6be2dbaebc0bdf +178, 0x6782682090dffe09 +179, 0xb63a4d90d848e8ef +180, 0x5f649c7eaf4c54c5 +181, 0xbe57582426a085ba +182, 0xb5dd825aa52fb76d +183, 0x74cb4e6ca4039617 +184, 0x382e578bf0a49588 +185, 0xc043e8ea6e1dcdae +186, 0xf902addd5c04fa7c +187, 0xf3337994612528db +188, 0x4e8fd48d6d15b4e6 +189, 0x7190a509927c07ab +190, 0x864c2dee5b7108ae +191, 0xbb9972ddc196f467 +192, 0x1ea02ab3ca10a448 +193, 0xe50a8ffde35ddef9 +194, 0x7bd2f59a67183541 +195, 0x5a940b30d8fcd27a +196, 0x82b4cea62623d4d3 +197, 0x6fbda76d4afef445 +198, 0x8b1f6880f418328e +199, 0x8b69a025c72c54b7 +200, 0xb71e0f3986a3835f +201, 0xa4a7ddb8b9816825 +202, 0x945dcda28228b1d8 +203, 0xb471abf2f8044d72 +204, 0xf07d4af64742b1ba +205, 0xfca5190bc4dd6a2a +206, 0xd681497262e11bc5 +207, 0xbe95d5f00c577028 +208, 0x56313439fd8bde19 +209, 0x3f3d9ac9b5ee6522 +210, 0x7b8d457dd2b49bbe +211, 0xe76b5747885d214b +212, 0xa8a695b3deb493ea +213, 0x5292446548c95d71 +214, 0xbf5cdf0d436412df +215, 0x7936abaed779d28d +216, 0x659c6e8073b3a06d +217, 0x86c9ff28f5543b71 +218, 0x6faa748445a99146 +219, 0xdcc1e6ab57904fd7 +220, 0x770bd61233addc5f +221, 0x16963e041e46d94f +222, 0x158e6cb2934157ac +223, 0xb65088a8fd246441 +224, 0x2b12ced6ce8a68c3 +225, 0x59a18d02cd6082b3 +226, 0x4ddbc318cb5488ee +227, 0x3d4cf520b3ed20a1 +228, 0x7028b3a92e2b292d +229, 0xf141da264a250e4d +230, 0x9788d53e86041c37 +231, 0x1bb91238a7c97dbf +232, 0x81953d0ddb634309 +233, 0xfa39ccfe14d2d46 +234, 0xf7c7861c9b7e8399 +235, 0x18d27ca50d9dc249 +236, 0x258dfdf38510d0d9 +237, 0x9e72d8af910ea76f +238, 0x4f8ef24b96de50ad +239, 0xb9d9c12297e03dc9 +240, 0x91994e41b4a1929c +241, 0x8defa79b2ccc83b9 +242, 0x948566748706dac5 +243, 0x7b0454946e70e4cf +244, 0x340b7cb298c70ed7 +245, 0x6602005330cebd95 +246, 0xf71cb803aa61f722 +247, 0x4683fb07fc70ae8a +248, 0xc6db9f0c4de3ed88 +249, 0x3e8dfae2a593cef9 +250, 0x615f7c38e3862b33 +251, 0x676c7996550d857 +252, 0xc6d520d54a5c266a +253, 0x202b1e8eef14aa2e +254, 0xa3a84891a27a582 +255, 0x84dbee451658d47f +256, 0x254c7cd97e777e3a +257, 0xf50b6e977f0eba50 +258, 0x2898b1d3062a4798 +259, 0x4096f7cbbb019773 +260, 0x9fb8e75548062c50 +261, 0x4647071e5ca318ec +262, 0x2b4750bdb3b3b01 +263, 0x88ac41cc69a39786 +264, 0x705e25476ef46fa3 +265, 0xc0c1db19884a48a6 +266, 0x1364c0afdbb465e5 +267, 0x58e98534701272a6 +268, 0x746a5ea9701517c0 +269, 0x523a70bc6b300b67 +270, 0x9b1c098eda8564ad +271, 0xfbaeb28d3637067f +272, 0xddd9a13551fdba65 +273, 0x56461a670559e832 +274, 0xab4fd79be85570ad +275, 0xd4b691ecaff8ca55 +276, 0x11a4495939e7f004 +277, 0x40d069d19477eb47 +278, 0xe790783d285cd81e +279, 0xde8218b16d935bc7 +280, 0x2635e8c65cd4182d +281, 0xeae402623e3454 +282, 0x9f99c833184e0279 +283, 0x3d0f79a0d52d84e7 +284, 0xc1f8edb10c625b90 +285, 0x9b4546363d1f0489 +286, 0x98d86d0b1212a282 +287, 0x386b53863161200d +288, 0xbe1165c7fe48a135 +289, 0xb9658b04dbbfdc8c +290, 0xcea14eddfe84d71a +291, 0x55d03298be74abe7 +292, 0x5be3b50d961ffd7e +293, 0xc76b1045dc4b78e1 +294, 0x7830e3ff3f6c3d4c +295, 0xb617adb36ca3729 +296, 0x4a51bdb194f14aa9 +297, 0x246024e54e6b682a +298, 0x33d42fc9c6d33083 +299, 0xadccba149f31e1d +300, 0x5183e66b9002f8b +301, 0x70eb2416404d51b7 +302, 0x26c25eb225535351 +303, 0xbc2d5b0d23076561 +304, 0x5823019ddead1da +305, 0x85cfa109fca69f62 +306, 0x26017933e7e1efd9 +307, 0x3ec7be9a32212753 +308, 0x697e8a0697cd6f60 +309, 0x44735f6cca03920f +310, 0x8cc655eb94ee212e +311, 0x8b8b74eba84929a0 +312, 0x7708ccedd0c98c80 +313, 0x1b6f21f19777cbe1 +314, 0x363e564bd5fadedb +315, 0x5921543a641591fe +316, 0xc390786d68ea8a1b +317, 0x9b293138dc033fca +318, 0x45447ca8dc843345 +319, 0xee6ef6755bc49c5e +320, 0x70a3a1f5163c3be5 +321, 0xf05e25448b6343b0 +322, 0x4739f4f8717b7e69 +323, 0xb006141975bf957 +324, 0x31874a91b707f452 +325, 0x3a07f2c90bae2869 +326, 0xb73dae5499a55c5e +327, 0x489070893bb51575 +328, 0x7129acf423940575 +329, 0x38c41f4b90130972 +330, 0xc5260ca65f5a84a1 +331, 0x6e76194f39563932 +332, 0x62ca1f9ca3de3ca6 +333, 0xb4a97874e640853f +334, 0x38ed0f71e311cc02 +335, 0xde183b81099e8f47 +336, 0x9bb8bf8e6694346 +337, 0xd15497b6bf81e0f2 +338, 0xaaae52536c00111 +339, 0x4e4e60d1435aaafd +340, 0x5a15512e5d6ea721 +341, 0xff0f1ffabfc6664f +342, 0xba3ffcedc5f97fec +343, 0xef87f391c0c6bfb6 +344, 0x4a888c5d31eb0f98 +345, 0x559a3fbfd7946e95 +346, 0xe45b44a0db5a9bad +347, 0x9457898964190af1 +348, 0xd9357dfaab76cd9e +349, 0xa60e907178d965a1 +350, 0x76b2dc3032dc2f4a +351, 0x13549b9c2802120 +352, 0x8656b965a66a1800 +353, 0x16802e6e22456a23 +354, 0x23b62edc60efaa9 +355, 0x6832a366e1e4ea3b +356, 0x46b1b41093ff2b1e +357, 0x55c857128143f219 +358, 0x7fc35ddf5e138200 +359, 0x790abe78be67467e +360, 0xa4446fc08babd466 +361, 0xc23d70327999b855 +362, 0x2e019d1597148196 +363, 0xfefd98e560403ab8 +364, 0xbe5f0a33da330d58 +365, 0x3078a4e9d43ca395 +366, 0x511bfedd6f12f2b3 +367, 0x8bc138e335be987c +368, 0x24640f803465716d +369, 0xf6530b04d0bd618f +370, 0x9b7833e5aa782716 +371, 0x778cd35aea5841b1 +372, 0xecea3c458cefbc60 +373, 0x5107ae83fc527f46 +374, 0x278ad83d44bd2d1a +375, 0x7014a382295aeb16 +376, 0xf326dd762048743f +377, 0x858633d56279e553 +378, 0x76408154085f01bc +379, 0x3e77d3364d02e746 +380, 0x2f26cea26cadd50b +381, 0x6d6846a4ecb84273 +382, 0x4847e96f2df5f76 +383, 0x5a8610f46e13ff61 +384, 0x4e7a7cac403e10dd +385, 0x754bdf2e20c7bc90 +386, 0x8bdd80e6c51bd0be +387, 0x61c655fae2b4bc52 +388, 0x60873ef48e3d2f03 +389, 0x9d7d8d3698a0b4a4 +390, 0xdf48e9c355cd5d4b +391, 0x69ecf03e20be99ac +392, 0xc1a0c5a339bd1815 +393, 0x2e3263a6a3adccb +394, 0x23557459719adbdc +395, 0xd1b709a3b330e5a +396, 0xade5ab00a5d88b9d +397, 0x69a6bd644120cfad +398, 0x40187ecceee92342 +399, 0x1c41964ba1ac78da +400, 0x9ac5c51cbecabe67 +401, 0xbdc075781cf36d55 +402, 0xeaf5a32246ded56 +403, 0xcda0b67e39c0fb71 +404, 0x4839ee456ef7cc95 +405, 0xf17092fdd41d5658 +406, 0x2b5d422e60ae3253 +407, 0x3effe71102008551 +408, 0x20a47108e83934b7 +409, 0xd02da65fe768a88f +410, 0xeb046bd56afa4026 +411, 0x70c0509c08e0fbe0 +412, 0x1d35c38d4f8bac6c +413, 0x9aa8eb6466f392e0 +414, 0x587bd4a430740f30 +415, 0x82978fe4bad4195 +416, 0xdc4ebc4c0feb50ab +417, 0xd3b7164d0240c06f +418, 0x6e2ad6e5a5003a63 +419, 0xa24b430e2ee6b59c +420, 0x2905f49fd5073094 +421, 0x5f209e4de03aa941 +422, 0x57b7da3e0bedb1dc +423, 0x5e054018875b01f5 +424, 0xb2f2da6145658db3 +425, 0xbd9c94a69a8eb651 +426, 0x9c5f9a07cd6ac749 +427, 0x2296c4af4d529c38 +428, 0x522ed800fafdefab +429, 0xe2a447ced0c66791 +430, 0x937f10d45e455fef +431, 0xc882987d9e29a24 +432, 0x4610bfd6a247ee1a +433, 0x562ba3e50870059 +434, 0x59d8d58793602189 +435, 0xfe9a606e3e34abe +436, 0x6825f7932a5e9282 +437, 0xe77f7061bab476ad +438, 0xbf42001da340ace3 +439, 0x9c3e9230f5e47960 +440, 0x2c0f700d96d5ad58 +441, 0x330048b7cd18f1f9 +442, 0xffc08785eca5cca9 +443, 0xb5879046915f07a5 +444, 0xef51fe26f83c988e +445, 0xfa4c2968e7881a9a +446, 0xc0a9744455a4aad +447, 0xbd2ad686d6313928 +448, 0x6b9f0984c127682a +449, 0xc9aaa00a5da59ed8 +450, 0x762a0c4b98980dbf +451, 0x52d1a2393d3ca2d1 +452, 0x1e9308f2861db15c +453, 0xe7b3c74fe4b4a844 +454, 0x485e15704a7fc594 +455, 0x9e7f67ea44c221f6 +456, 0xbab9ad47fde916e0 +457, 0x50e383912b7fc1f4 +458, 0xaad63db8abcef62d +459, 0xc2f0c5699f47f013 +460, 0xee15b36ada826812 +461, 0x2a1b1cf1e1777142 +462, 0x8adb03ede79e937d +463, 0xf14105ef65643bf3 +464, 0x752bbaefc374a3c7 +465, 0xa4980a08a5a21d23 +466, 0x418a1c05194b2db7 +467, 0xdd6ff32efe1c3cd6 +468, 0x272473ed1f0d3aa2 +469, 0x1e7fdebadabe6c06 +470, 0xd1baa90c17b3842f +471, 0xd3d3a778e9c8404a +472, 0x781ae7fda49fa1a0 +473, 0x61c44fdbdacc672d +474, 0x6d447d0a1404f257 +475, 0x9303e8bdfbfb894d +476, 0x3b3482cdec016244 +477, 0xb149bf245d062e7b +478, 0x96f8d54b14cf992d +479, 0x4741549a01f8c3d0 +480, 0x48270811b2992af +481, 0x7b58f175cd25d147 +482, 0x8f19a840b56f4be9 +483, 0x84a77f43c0951a93 +484, 0x34e1a69381f0c374 +485, 0xb158383c9b4040f +486, 0x372f1abc7cf3a9fa +487, 0x5439819a84571763 +488, 0xabf8515e9084e2fa +489, 0xb02312b9387ff99 +490, 0x238a85bb47a68b12 +491, 0x2068cb83857c49bb +492, 0xc6170e743083664c +493, 0x745cf8470bcb8467 +494, 0xe3a759a301670300 +495, 0x292c7686ad3e67da +496, 0x359efedaff192a45 +497, 0x511f2c31a2d8c475 +498, 0x97fd041bf21c20b3 +499, 0x25ef1fe841b7b3f6 +500, 0xbb71739e656f262d +501, 0x2729b0e989b6b7b8 +502, 0xd2142702ec7dbabf +503, 0x7008decd2488ee3f +504, 0x69daa95e303298d7 +505, 0xc35eca4efb8baa5a +506, 0xf3f16d261cec3b6c +507, 0x22371c1d75396bd3 +508, 0x7aefa08eccae857e +509, 0x255b493c5e3c2a2f +510, 0x779474a077d34241 +511, 0x5199c42686bea241 +512, 0x16c83931e293b8d3 +513, 0xa57fe8db8c0302c7 +514, 0xd7ace619e5312eb1 +515, 0x8740f013306d217c +516, 0xb6a1ad5e29f4d453 +517, 0x31abf7c964688597 +518, 0xbc3d791daed71e7 +519, 0x31ee4ca67b7056ed +520, 0x1ab5416bfe290ea3 +521, 0x93db416f6d3b843a +522, 0xed83bbe5b1dd2fed +523, 0xece38271470d9b6d +524, 0x3a620f42663cd8ae +525, 0x50c87e02acafee5d +526, 0xcabeb8bedbc6dab5 +527, 0x2880a6d09970c729 +528, 0x4aba5dd3bfc81bc +529, 0xaba54edf41080cec +530, 0xb86bb916fc85a169 +531, 0x4c41de87bc79d8ca +532, 0xcce2a202622945fe +533, 0x513f086fad94c107 +534, 0x18b3960c11f8cc96 +535, 0x2f0d1cfd1896e236 +536, 0x1702ae3880d79b15 +537, 0x88923749029ae81 +538, 0x84810d4bdec668eb +539, 0xf85b0a123f4fc68d +540, 0x93efd68974b6e4d1 +541, 0x5d16d6d993a071c9 +542, 0x94436858f94ca43b +543, 0xb3dbb9ed0cb180b6 +544, 0x6447030a010b8c99 +545, 0xd7224897c62925d8 +546, 0xb0c13c1d50605d3a +547, 0xdff02c7cb9d45f30 +548, 0xe8103179f983570d +549, 0xbc552037d6d0a24e +550, 0x775e500b01486b0d +551, 0x2050ac632c694dd6 +552, 0x218910387c4d7ae7 +553, 0xf83e8b68ff885d5d +554, 0xe3374ec25fca51a3 +555, 0xfa750ffa3a60f3af +556, 0x29ee40ba6df5592e +557, 0x70e21a68f48260d2 +558, 0x3805ca72cd40886e +559, 0x2f23e73f8eabf062 +560, 0x2296f80cdf6531ae +561, 0x903099ed968db43a +562, 0xf044445cf9f2929f +563, 0xcd47fdc2de1b7a1 +564, 0xaab1cbd4f849da99 +565, 0x5fc990688da01acb +566, 0xa9cee52ea7dab392 +567, 0xecefc3a4349283a8 +568, 0xdd6b572972e3fafc +569, 0xc1f0b1a2ffb155da +570, 0xc30d53fc17bd25c8 +571, 0x8afa89c77834db28 +572, 0x5569a596fb32896c +573, 0x36f207fc8df3e3d4 +574, 0x57c2bd58517d81db +575, 0xb524693e73d0061c +576, 0xb69f6eb233f5c48b +577, 0x4f0fb23cab8dc695 +578, 0x492c1ad0a48df8df +579, 0xf6dcc348ec8dec1f +580, 0xa4d8708d6eb2e262 +581, 0x4c2072c2c9766ff1 +582, 0xa9bf27c4304875f0 +583, 0xfc8fb8066d4f9ae2 +584, 0x188095f6235fec3c +585, 0x1d8227a2938c2864 +586, 0x89ea50c599010378 +587, 0xcac86df0a7c6d56d +588, 0x47a8c5df84c7d78 +589, 0xe607ae24ea228bfa +590, 0x36624a7996efe104 +591, 0x5d72881c1227d810 +592, 0x78694a6750374c8 +593, 0x7b9a217d4ab5ff45 +594, 0xd53e5d6f7504becc +595, 0x197a72d3f4889a0e +596, 0xfdc70c4755a8df36 +597, 0xd0fda83748c77f74 +598, 0x7ddc919ac9d6dcc9 +599, 0x785c810a6a2dc08b +600, 0xba4be83e7e36896c +601, 0x379d6fe80cf2bffe +602, 0x74cae2dabc429206 +603, 0x1efac32d5d34c917 +604, 0x3cb64e2f98d36e70 +605, 0xc0a7c3cdc3c60aa7 +606, 0x699dfadd38790ebe +607, 0x4861e61b3ecfbeac +608, 0x531744826c345baa +609, 0x5ec26427ad450cba +610, 0xf2c1741479abdcae +611, 0xe9328a78b2595458 +612, 0x30cd1bdf087acd7f +613, 0x7491ced4e009adbe +614, 0xdcd942df1e2e7023 +615, 0xfe63f01689fee35 +616, 0x80282dfe5eaedc42 +617, 0x6ecdea86495f8427 +618, 0xe0adfdd5e9ed31c3 +619, 0xf32bd2a7418127e +620, 0x8aabba078db6ee2 +621, 0xa8a8e60499145aca +622, 0xf76b086ac4e8a0f2 +623, 0x6e55b3c452ff27f8 +624, 0xe18fa7cd025a71bf +625, 0xeed7b685fde0fa25 +626, 0xba9b6c95867fa721 +627, 0x4c2603bc69de2df2 +628, 0xaac87eee1b58cd66 +629, 0x3c9af6656e01282c +630, 0x2dfa05ce8ff476b6 +631, 0xeae9143fcf92f23d +632, 0x3f0699f631be3bc8 +633, 0xa0f5f79f2492bd67 +634, 0x59c47722388131ed +635, 0x5f6e9d2941cef1de +636, 0xe9ad915c09788b7b +637, 0x92c6d37e4f9482f5 +638, 0x57d301b7fdadd911 +639, 0x7e952d23d2a8443 +640, 0xbb2fa5e0704b3871 +641, 0xe5642199be36e2d5 +642, 0x5020b60d54358291 +643, 0xa0b6317ec3f60343 +644, 0xb57b08b99540bc5c +645, 0x21f1890adc997a88 +646, 0xfcf824200dd9da2d +647, 0x8146293d83d425d1 +648, 0xdadfbf5fbb99d420 +649, 0x1eb9bbc5e6482b7d +650, 0xd40ff44f1bbd0f1c +651, 0xa9f948ba2d08afa5 +652, 0x638cc07c5301e601 +653, 0x1f984baa606e14e8 +654, 0x44e153671081f398 +655, 0xb17882eeb1d77a5d +656, 0x5fd8dbee995f14c +657, 0xff3533e87f81b7fe +658, 0x2f44124293c49795 +659, 0x3bf6b51e9360248 +660, 0x72d615edf1436371 +661, 0x8fc5cf4a38adab9d +662, 0xfa517e9022078374 +663, 0xf356733f3e26f4d8 +664, 0x20ea099cdc6aad40 +665, 0xe15b977deb37637d +666, 0xcc85601b89dae88d +667, 0x5768c62f8dd4905c +668, 0xa43cc632b4e56ea +669, 0xc4240cf980e82458 +670, 0xb194e8ffb4b3eeb6 +671, 0xee753cf2219c5fa1 +672, 0xfe2500192181d44d +673, 0x2d03d7d6493dd821 +674, 0xff0e787bb98e7f9b +675, 0xa05cf8d3bd810ce7 +676, 0x718d5d6dcbbdcd65 +677, 0x8d0b5343a06931c +678, 0xae3a00a932e7eaf9 +679, 0x7ed3d8f18f983e18 +680, 0x3bb778ee466dc143 +681, 0x711c685c4e9062c0 +682, 0x104c3af5d7ac9834 +683, 0x17bdbb671fb5d5cf +684, 0xabf26caead4d2292 +685, 0xa45f02866467c005 +686, 0xf3769a32dc945d2d +687, 0xe78d0007f6aabb66 +688, 0x34b60be4acbd8d4b +689, 0x58c0b04b69359084 +690, 0x3a8bb354c212b1 +691, 0x6b82a8f3d70058d5 +692, 0x405bdef80a276a4a +693, 0xe20ca40ee9195cad +694, 0xf5dd96ba2446fefd +695, 0xc1e180c55fe55e3c +696, 0xa329caf6daa952b3 +697, 0xb4809dd0c84a6b0a +698, 0xd27f82661070cee7 +699, 0xa7121f15ee2b0d8a +700, 0x4bdaea70d6b34583 +701, 0xe821dc2f310f7a49 +702, 0x4c00a5a68e76f647 +703, 0x331065b064a2d5ea +704, 0xac0c2ce3dc04fa37 +705, 0x56b32b37b8229008 +706, 0xe757cdb51534fcfa +707, 0xd3ff183576b2fad7 +708, 0x179e1f4190f197a7 +709, 0xf874c626a7c9aae5 +710, 0xd58514ffc37c80e4 +711, 0xc65de31d33fa7fd3 +712, 0x6f6637052025769b +713, 0xca1c6bdadb519cc0 +714, 0xd1f3534cde37828a +715, 0xc858c339eee4830a +716, 0x2371eacc215e02f4 +717, 0x84e5022db85bbbe9 +718, 0x5f71c50bba48610e +719, 0xe420192dad9c323f +720, 0x2889342721fca003 +721, 0x83e64f63334f501d +722, 0xac2617172953f2c +723, 0xfa1f78d8433938ff +724, 0x5578382760051462 +725, 0x375d7a2e3b90af16 +726, 0xb93ff44e6c07552d +727, 0xded1d5ad811e818c +728, 0x7cf256b3b29e3a8c +729, 0x78d581b8e7bf95e8 +730, 0x5b69192f2caa6ad3 +731, 0xa9e25855a52de3ce +732, 0x69d8e8fc45cc188d +733, 0x5dd012c139ad347d +734, 0xfcb01c07b77db606 +735, 0x56253e36ab3d1cce +736, 0x1181edbb3ea2192 +737, 0x325bef47ff19a08d +738, 0xd3e231ceb27e5f7 +739, 0x8e819dd2de7956d2 +740, 0x34a9689fe6f84a51 +741, 0x3e4eeb719a9c2927 +742, 0x5c3b3440581d0aaf +743, 0x57caf51897d7c920 +744, 0xec6a458130464b40 +745, 0xe98f044e0da40e9b +746, 0xbe38662020eeb8e7 +747, 0x7b8c407c632724ae +748, 0x16c7cfa97b33a544 +749, 0xd23359e2e978ae5a +750, 0x4fdba458250933dd +751, 0x3c9e0713cfe616ba +752, 0x6f0df87b13163b42 +753, 0xc460902cb852cc97 +754, 0x289df8fefd6b0bce +755, 0x4ac2a2a1c3fb8029 +756, 0x2fc3e24d8b68eef7 +757, 0x34564386a59aab9a +758, 0x31047391ebd67ce4 +759, 0x6c23d070a0564d41 +760, 0xba6387b2b72545f7 +761, 0xcdcf1008058387af +762, 0xc9308fa98db05192 +763, 0xdbdbb5abd01a9d84 +764, 0x937088275c7804ab +765, 0x6f6accfefe34ee81 +766, 0x5c33c74c49cfdb2c +767, 0x5e1a771edfb92bd3 +768, 0x6e89b009069ecae7 +769, 0x34d64e17ec0e8968 +770, 0x841203d0cde0c330 +771, 0x7642cc9d7eb9e9cb +772, 0xca01d2e8c128b97e +773, 0x5b8390617b3304ab +774, 0x52ec4ed10de1eb2d +775, 0xb90f288b9616f237 +776, 0x5bd43cd49617b2e2 +777, 0x1a53e21d25230596 +778, 0x36ccd15207a21cd6 +779, 0xc8263d780618fd3c +780, 0x6eb520598c6ce1cb +781, 0x493c99a3b341564f +782, 0xab999e9c5aa8764f +783, 0xab2fa4ceaba84b +784, 0xbbd2f17e5cb2331b +785, 0xc8b4d377c0cc4e81 +786, 0x31f71a6e165c4b1e +787, 0xd1011e55fb3addaa +788, 0x5f7ec34728dfa59 +789, 0x2aef59e60a84eb0f +790, 0x5dde6f09aec9ad5f +791, 0x968c6cdbc0ef0438 +792, 0x1957133afa15b13a +793, 0xbaf28f27573a64c2 +794, 0xc6f6ddd543ebf862 +795, 0xdd7534315ec9ae1e +796, 0xd2b80cd2758dd3b +797, 0xa38c3da00cc81538 +798, 0x15c95b82d3f9b0f9 +799, 0x6704930287ce2571 +800, 0x9c40cc2f6f4ecb0c +801, 0xc8de91f50b22e94e +802, 0x39272e8fddbfdf0a +803, 0x879e0aa810a117d +804, 0xa312fff4e9e5f3bd +805, 0x10dd747f2835dfec +806, 0xeb8466db7171cdae +807, 0xaa808d87b9ad040a +808, 0xab4d2229a329243a +809, 0x7c622f70d46f789c +810, 0x5d41cef5965b2a8e +811, 0xce97ec4702410d99 +812, 0x5beba2812c91211b +813, 0xf134b46c93a3fec7 +814, 0x76401d5630127226 +815, 0xc55fc9d9eacd4ec1 +816, 0xaec8cefaa12f813f +817, 0x2f845dcfd7b00722 +818, 0x3380ab4c20885921 +819, 0xdb68ad2597691b74 +820, 0x8a7e4951455f563f +821, 0x2372d007ed761c53 +822, 0xcab691907714c4f1 +823, 0x16bc31d6f3abec1a +824, 0x7dff639fbcf1824 +825, 0x6666985fbcff543d +826, 0xb618948e3d8e6d0c +827, 0x77b87837c794e068 +828, 0xcd48288d54fcb5a8 +829, 0x47a773ed6ae30dc3 +830, 0xba85ae44e203c942 +831, 0xa7a7b21791a25b2d +832, 0x4029dd92e63f19e0 +833, 0xc2ad66ab85e7d5aa +834, 0xa0f237c96fdab0db +835, 0xffefb0ab1ca18ed +836, 0x90cb4500785fd7d5 +837, 0xa7dd3120f4876435 +838, 0x53f7872624694300 +839, 0xea111326ff0040d9 +840, 0x5f83cb4cce40c83b +841, 0x918e04936c3b504d +842, 0x87a8db4c0e15e87c +843, 0x7cff39da6a0dedd0 +844, 0x36f7de2037f85381 +845, 0xd1d8d94022a1e9a7 +846, 0x2c9930127dc33ec9 +847, 0x6cb4719dcd0101c6 +848, 0xc01868cde76935f7 +849, 0x6b86f2ec1ab50143 +850, 0x68af607d8d94ae61 +851, 0xe216c5b95feedf34 +852, 0x4b866bd91efe2e4b +853, 0x4bff79df08f92c99 +854, 0x6ff664ea806acfd1 +855, 0x7fce0b3f9ece39bc +856, 0x29bc90b59cb3db97 +857, 0x833c4b419198607d +858, 0xf3573e36ca4d4768 +859, 0x50d71c0a3c2a3fa8 +860, 0xd754591aea2017e7 +861, 0x3f9126f1ee1ebf3 +862, 0xe775d7f4b1e43de8 +863, 0xe93d51628c263060 +864, 0x83e77f6fb32d6d82 +865, 0x43dd7eef823408e4 +866, 0x1c843c2c90180662 +867, 0xe924dafb9a16066b +868, 0x6af3ee96e7b7fbd9 +869, 0x94d5c4f37befcd1f +870, 0x40ffb04bedef4236 +871, 0x71c17bbc20e553e +872, 0x101f7a0a6208729f +873, 0x5ca34570cf923548 +874, 0x8e3139db2e96e814 +875, 0x3ab96d96263d048d +876, 0x97f3c0bbc6755c3c +877, 0x31fc72daedaef3dc +878, 0x71f8d7855d10789b +879, 0xce6dc97b4662333b +880, 0xfddc2aabd342bc61 +881, 0xefbd4007ff8c7d2e +882, 0xf72cd6c689ef8758 +883, 0x932c8b0c0e755137 +884, 0x94cc4dedd58ff69 +885, 0xde4dfd6890535979 +886, 0xdb00dcd2dcb4a50a +887, 0xb0466240b4548107 +888, 0x9cb9264c7b90d1a3 +889, 0x357e378e9be5766b +890, 0x6e0316ef03367bbf +891, 0x201ea18839544ca +892, 0x803ff3406be5f338 +893, 0xf9d5e82fd4144bb2 +894, 0x1b6b88ca701e9f47 +895, 0xd1fe5ab8e1f89cc0 +896, 0x14171fe176c4bece +897, 0x887948bdef78beaa +898, 0x80449ddc3eb9b977 +899, 0x5f4e1f900fb4bcf3 +900, 0xbe30f8701909f8e2 +901, 0xd1f2a2fb5503306d +902, 0x6b1c77238dc23803 +903, 0x102156a6c9860f66 +904, 0x4cd446e099edf4c1 +905, 0xc79ac6cbc911f33b +906, 0x3ee096ffe3384f1c +907, 0xb58f83b18a306dc7 +908, 0x9f76582141de56b2 +909, 0x9ddfa85e02c13866 +910, 0x4d9a19d4ce90a543 +911, 0xbf81ab39fd17d376 +912, 0x5327e5054c6a74f1 +913, 0xd5062dd31db1a9b7 +914, 0x645853735527edc +915, 0x485393967f91af08 +916, 0xeff9667dcf77ca68 +917, 0xd012313f5fbec464 +918, 0xbeae35bdfae55144 +919, 0x302c41ebac8444a0 +920, 0x9ccdb6c2fe58fba8 +921, 0x567753af68ed23f8 +922, 0xff90f790e43efec3 +923, 0x970cc756fb799696 +924, 0xe59239d1c44915 +925, 0x4d2d189fb3941f05 +926, 0x96f23085db165a9c +927, 0xa1202dec7a37b1a5 +928, 0xc0c1ee74bcd7dc1a +929, 0x9edcf2048b30333a +930, 0xd848588ba7e865fb +931, 0x8d9f0897317cab40 +932, 0x67b96f15e25924fb +933, 0xefc8d8536619ee42 +934, 0xf3f621d22bdde0c2 +935, 0x68610a0de862ae32 +936, 0xa22ca5142de24cbd +937, 0x8815452f4e6b4801 +938, 0x4e9c1b607b2750e5 +939, 0x19b3c09ba6fc9b25 +940, 0x9b2543c8836780ac +941, 0xe702b8f950e56431 +942, 0xb357cc329cac3917 +943, 0x387bf86a17a31e08 +944, 0x9940b983d331b163 +945, 0xf5d89d7fe9095e18 +946, 0x4362682329e5c4d1 +947, 0xd2132573f6ae7b42 +948, 0xc0a5849e23a61606 +949, 0xdadbddf47265bc02 +950, 0x1b96f00339a705f7 +951, 0x94e6642329288913 +952, 0x825ab3f10e6d330b +953, 0x1a1c31ac9d883ea0 +954, 0xb49076b7155c6f47 +955, 0x920cf3085dfe3ccb +956, 0x9743407c9f28e825 +957, 0x6ce8a28622402719 +958, 0xce2fe67e06baf8a6 +959, 0x3a16b34784ecf5e6 +960, 0x140467cc1d162a0c +961, 0x32d4772692ab625 +962, 0xa4f4b28562f43336 +963, 0x885b4335457bd84a +964, 0x499d3ed26c87ad8a +965, 0xc7328bcedb9a545e +966, 0xc6dd76a6cbf5d2b2 +967, 0xba9c22be404ee1aa +968, 0x70e6aee45f23521d +969, 0x61e03a798593c177 +970, 0x171671f809c68213 +971, 0x28d54872fc1d914c +972, 0x43c2fcd9bd098b53 +973, 0x172ad4c4a98b9d37 +974, 0x330860c9460f2516 +975, 0x49547f472df984f4 +976, 0x873b2436d3f0e114 +977, 0x6f99accf4ea050b6 +978, 0x5968ac874ed51613 +979, 0x4939d70d29a3c611 +980, 0x11f381ed28738d3d +981, 0xa97430d36ab3a869 +982, 0xe6fa880801129e22 +983, 0xf84decbd8f48c913 +984, 0x4425c0ed1e9a82a5 +985, 0x7a1f9485e9929d5a +986, 0xc7c51f155dfce1c6 +987, 0x9619a39501d74f2b +988, 0x7c7035955dbf4c1b +989, 0xc61ee569cf57c2c9 +990, 0x3eaf7c5b0df734e1 +991, 0xe71cb4064d1ede05 +992, 0x356e3cec80e418b2 +993, 0xca04306243a15be6 +994, 0x941cf3881fa18896 +995, 0x30dbb0e819d644e0 +996, 0xaae22c0bef02859a +997, 0x7bd30917bbaa8a94 +998, 0x2672547bc8d7d329 +999, 0x4955c92aaa231578 diff --git a/numpy/random/tests/data/pcg64dxsm-testset-2.csv b/numpy/random/tests/data/pcg64dxsm-testset-2.csv new file mode 100644 index 000000000000..878c5ea7c3a5 --- /dev/null +++ b/numpy/random/tests/data/pcg64dxsm-testset-2.csv @@ -0,0 +1,1001 @@ +seed, 0x0 +0, 0xd97e4a147f788a70 +1, 0x8dfa7bce56e3a253 +2, 0x13556ed9f53d3c10 +3, 0x55dbf1c241341e98 +4, 0xa2cd98f722eb0e0a +5, 0x83dfc407203ade8 +6, 0xeaa083df518f030d +7, 0x44968c87e432852b +8, 0x573107b9cb8d9ecc +9, 0x9eedd1da50b9daca +10, 0xb33a6735ca451e3c +11, 0x72830d2b39677262 +12, 0x9da8c512fd0207e8 +13, 0x1fc5c91954a2672b +14, 0xd33479437116e08 +15, 0x9ccdd9390cee46f3 +16, 0x1fd39bb01acd9e76 +17, 0xedc1869a42ff7fe5 +18, 0xbd68ca0b42a6e7e9 +19, 0x620b67df09621b1f +20, 0xfa11d51bd6950221 +21, 0xc8c45b36e7d28d08 +22, 0xe9c91272fbaad777 +23, 0x2dc87a143f220e90 +24, 0x6376a7c82361f49d +25, 0x552c5e434232fe75 +26, 0x468f7f872ac195bc +27, 0x32bed6858125cf89 +28, 0xe4f06111494d09d3 +29, 0xa5c166ffea248b80 +30, 0x4e26605b97064a3f +31, 0xceafd9f6fc5569d +32, 0xb772f2f9eed9e106 +33, 0x672c65e6a93534e2 +34, 0xcdc5e1a28d1bd6a0 +35, 0x1ed9c96daeebd3e3 +36, 0x4d189dcfc0c93c3f +37, 0x50df5a95c62f4b43 +38, 0xcccf4949fa65bbb8 +39, 0x19b8073d53cdc984 +40, 0x6fb40bba35483703 +41, 0xb02de4aef86b515a +42, 0x4d90c63655350310 +43, 0xea44e4089825b16c +44, 0x8d676958b1f9da2b +45, 0x6d313940917ae195 +46, 0x1b1d35a4c1dd19f4 +47, 0x117720f8397337ef +48, 0xcc073cf3ac11eeaa +49, 0x8331ec58a9ff8acb +50, 0xf3dc2a308b6b866f +51, 0x7eba1202663382b6 +52, 0x8269839debeb4e5a +53, 0x87fd3dc0f9181a8e +54, 0xabe62ddd3c925f03 +55, 0x7f56f146944fe8d4 +56, 0xc535972150852068 +57, 0x60b252d453bd3a68 +58, 0x4251f0134634490a +59, 0x338950da210dfeb2 +60, 0xcadfe932971c9471 +61, 0xfb7049457fab470e +62, 0x9bfb8145a4459dff +63, 0x4a89dda3898f9d8a +64, 0x88cc560151483929 +65, 0x277dc820f4b6796e +66, 0x3524bd07ea0afb88 +67, 0x92eb6ffb2bf14311 +68, 0xf6559be0783f3fe9 +69, 0xf0844f9af54af00d +70, 0xdd5e0b59adcef8a +71, 0x4ff7e4f2ab18554c +72, 0x3fa22c8a02634587 +73, 0x1db8e1a9442fe300 +74, 0x40cf15953ad3d3e7 +75, 0x92af15fe1a9f6f0a +76, 0xab4a0e466fb0cfd +77, 0x944f1555a06cca82 +78, 0x10cf48412f1f6066 +79, 0x7f51f9a455f9e8e1 +80, 0x47ee93530f024c7e +81, 0x36cf2f0413e0f6f2 +82, 0xa315e23731969407 +83, 0xd8e2796327cf5f87 +84, 0xa86072696a555c34 +85, 0xee3f0b8804feaab7 +86, 0x41e80dc858f8360b +87, 0x31ec2e9b78f5b29 +88, 0xd397fb9b8561344c +89, 0x28081e724e649b74 +90, 0x5c135fc3fc672348 +91, 0x9a276ca70ce9caa0 +92, 0x9216da059229050a +93, 0xcf7d375ed68007b0 +94, 0xa68ad1963724a770 +95, 0xd4350de8d3b6787c +96, 0xee7d2c2cc275b6d2 +97, 0x71645ec738749735 +98, 0x45abdf8c68d33dbb +99, 0xe71cadb692c705ea +100, 0x60af6f061fd90622 +101, 0x1eabe2072632c99d +102, 0x947dda995a402cb6 +103, 0xbb19f49a3454f3b +104, 0xe6e43e907407758c +105, 0xfe2b67016bd6873a +106, 0x7fdb4dd8ab30a722 +107, 0x39d3265b0ff1a45b +108, 0xed24c0e4fce8d0c2 +109, 0xf6e074f86faf669d +110, 0x9142040df8dc2a79 +111, 0x9682ab16bc939a9c +112, 0x6a4e80c378d971c8 +113, 0x31309c2c7fc2d3d6 +114, 0xb7237ec682993339 +115, 0x6a30c06bb83dccd9 +116, 0x21c8e9b6d8e7c382 +117, 0x258a24ae6f086a19 +118, 0xb76edb5be7df5c35 +119, 0x3c11d7d5c16e7175 +120, 0xbdfc34c31eff66e1 +121, 0x8af66e44be8bf3a2 +122, 0x3053292e193dec28 +123, 0xd0cc44545b454995 +124, 0x408ac01a9289d56 +125, 0x4e02d34318ec2e85 +126, 0x9413ff3777c6eb6b +127, 0xa3a301f8e37eb3df +128, 0x14e6306bd8d8f9f9 +129, 0xd3ea06ce16c4a653 +130, 0x170abe5429122982 +131, 0x7f9e6fddc6cacb85 +132, 0xa41b93e10a10a4c8 +133, 0x239216f9d5b6d0b5 +134, 0x985fcb6cb4190d98 +135, 0xb45e3e7c68f480c6 +136, 0xc1b2fc2e0446211c +137, 0x4596adb28858c498 +138, 0x2dd706f3458ddc75 +139, 0x29c988c86f75464 +140, 0xac33a65aa679a60 +141, 0xa28fef762d39d938 +142, 0x541e6fa48647f53 +143, 0x27838d56b2649735 +144, 0x8e143d318a796212 +145, 0xaea6097745f586b8 +146, 0x636143330f8ee2e6 +147, 0xc2d05fd8b945b172 +148, 0x6e355f9eb4353055 +149, 0xeb64ca42e8bf282e +150, 0xe8202dfd9da0fe5 +151, 0x7305689c9d790cba +152, 0xf122f8b1bef32970 +153, 0x9562887e38c32ba5 +154, 0xf9cd9be121b738d +155, 0x6238e0c398307913 +156, 0x5f2e79bb07c30f47 +157, 0x8ce8e45c465006e +158, 0x39281fe1e99e2441 +159, 0xafb10c2ca2874fea +160, 0x6e52f91633f83cf +161, 0x8ff12c1ac73c4494 +162, 0xe48608a09365af59 +163, 0xefd9bbc7e76e6a33 +164, 0xbe16a39d5c38ec92 +165, 0x6a6ffbcaf5a2330f +166, 0xdd5d6ac7d998d43d +167, 0x207bf978226d4f11 +168, 0xf8eec56bd2a0f62e +169, 0xa5bccf05dce0d975 +170, 0x93cf3ec1afe457a6 +171, 0x38651466d201f736 +172, 0x3ad21473985c9184 +173, 0xc6407a3bd38c92a6 +174, 0xb1ec42c7afa90a25 +175, 0xbdeca984df8b7dd3 +176, 0xb6926b1d00aa6c55 +177, 0x86141d0022352d49 +178, 0x169316256135ee09 +179, 0xffb1c7767af02a5c +180, 0x502af38ad19f5c91 +181, 0xfbf6cbc080086658 +182, 0x33cf9b219edae501 +183, 0x46e69bebd77b8862 +184, 0xf11e0cc91125d041 +185, 0xb4cd1649f85e078f +186, 0xb49be408db4e952 +187, 0xb0b8db46140cce3c +188, 0xba647f2174012be7 +189, 0x4f0a09e406970ac9 +190, 0xf868c7aec9890a5c +191, 0xde4c8fa7498ea090 +192, 0x872ceb197978c1d4 +193, 0x1eb5cd9c3269b258 +194, 0x3ea189f91724f014 +195, 0x41379656f7746f2c +196, 0x7bd18493aca60e51 +197, 0x5380c23b0cbbf15e +198, 0x920b72835f88246b +199, 0x24d7f734a4548b8e +200, 0x9944edb57e5aa145 +201, 0x4628e136ebb8afe1 +202, 0xb4ee6a776356e2a7 +203, 0x481cbe9744ccf7d7 +204, 0x7e8d67e8b0b995d9 +205, 0xeeacde100af7b47e +206, 0x103da08f2487dab7 +207, 0x6b9890a91d831459 +208, 0xd0c5beae37b572c7 +209, 0xfdccc371ee73fcc +210, 0x65438f0a367a2003 +211, 0x5d23b2c818a7e943 +212, 0x9a8ed45ac04b58b3 +213, 0xdaf3c3f1695dce10 +214, 0x5960eec706fa2bc0 +215, 0x98ca652facb80d40 +216, 0x72970ae5e2194143 +217, 0x18c6374d878c5c94 +218, 0x20fa51f997381900 +219, 0x3af253dba26d6e1d +220, 0x1b23d65db15c7f78 +221, 0x9f53ae976259b0e3 +222, 0x9a6addb28dc92d49 +223, 0x1e085c4accd0a7d7 +224, 0xe9d3f4cc9bad6ce5 +225, 0xe018fad78b5b1059 +226, 0x5ef7682232b4b95 +227, 0xb2242aa649f5de80 +228, 0x8f3e6d8dd99b9e4e +229, 0xb9be6cc22949d62a +230, 0xecbdc7beaa5ff1fe +231, 0xd388db43a855bdf0 +232, 0xd71ee3238852568d +233, 0x85ab3056304c04b5 +234, 0x2ed7ae7ad3cfc3cb +235, 0x781d1b03d40b6c48 +236, 0x7d3c740886657e6d +237, 0x982cfa6828daa6b0 +238, 0x278579599c529464 +239, 0x773adecfae9f0e08 +240, 0x63a243ea4b85c5d7 +241, 0x59940074fc3709e1 +242, 0xc914a2eed58a6363 +243, 0x2602b04274dd724c +244, 0xdf636eb7636c2c42 +245, 0x891a334d0d26c547 +246, 0xde8cd586d499e22d +247, 0x3ea1aa4d9b7035b6 +248, 0xd085cff6f9501523 +249, 0xe82a872f374959e +250, 0x55cb495bbd42cc53 +251, 0x5f42b3226e56ca97 +252, 0xea463f6f203493a3 +253, 0xeef3718e57731737 +254, 0x1bd4f9d62b7f9f3c +255, 0x19284f5e74817511 +256, 0xaf6e842c7450ca87 +257, 0x1d27d2b08a6b3600 +258, 0xfb4b912b396a52e3 +259, 0x30804d4c5c710121 +260, 0x4907e82564e36338 +261, 0x6441cf3b2900ddb7 +262, 0xd76de6f51988dc66 +263, 0x4f298ef96fd5e6d2 +264, 0x65432960c009f83d +265, 0x65ebed07e1d2e3df +266, 0xf83ee8078febca20 +267, 0x7bb18e9d74fc5b29 +268, 0x597b5fbc2261d91 +269, 0xea4f8ed0732b15b2 +270, 0xba2267f74f458268 +271, 0x3f304acabd746bbb +272, 0x7bd187af85659a82 +273, 0x88e20dbdb7a08ea3 +274, 0x2a2dc948c772fcb4 +275, 0x87784fec2993c867 +276, 0x89163933cd362d4e +277, 0xfd7b24f04302f957 +278, 0x9bdd544405dfb153 +279, 0xddee0fac58ffc611 +280, 0xa8e8993417e71ec1 +281, 0x55e0ab46ff7757af +282, 0x53e7645f08d3d7df +283, 0xbf78e563bc656ba2 +284, 0x1d162253b45ee2de +285, 0x15e2bfefedf29eb4 +286, 0x4e2a4584aa394702 +287, 0xa89fb12b01525897 +288, 0x825bd98f0544e4df +289, 0xfc6c50da6750700 +290, 0xc24aaabde7d28423 +291, 0x79d6f4660fcb19e5 +292, 0xee7d4fb40c8d659f +293, 0x70bc281b462e811d +294, 0x23ed4dc9636519a7 +295, 0xcb7c3f5a5711b935 +296, 0xe73090e0508c5d9d +297, 0xb25a331f375952a6 +298, 0xa64c86e0c04740f6 +299, 0xb8f3ffc8d56ac124 +300, 0x2479266fc5ee6b15 +301, 0x8d5792d27f5ffbcb +302, 0xb064298be946cd52 +303, 0xf0934a98912ffe26 +304, 0xbe805682c6634d98 +305, 0xe0e6e2c010012b4f +306, 0x58c47d475f75976 +307, 0x358c9a6e646b2b4a +308, 0x7e7c4ffca5b17ba7 +309, 0x43585c8c9a24a04c +310, 0x5154ddbcd68d5c2c +311, 0x4a2b062d3742a5e +312, 0xca5691191da2b946 +313, 0x696a542109457466 +314, 0x9eb5d658a5022ba5 +315, 0x8158cf6b599ab8dc +316, 0x1b95391eaa4af4a6 +317, 0x9953e79bd0fc3107 +318, 0x8639690086748123 +319, 0x2d35781c287c6842 +320, 0x393ef0001cd7bc8f +321, 0xe3a61be8c5f2c22a +322, 0x5e4ff21b847cc29b +323, 0x4c9c9389a370eb84 +324, 0xd43a25a8fc3635fa +325, 0xf6790e4a85385508 +326, 0x37edf0c81cb95e1d +327, 0x52db00d6e6e79af8 +328, 0x3b202bceeb7f096 +329, 0x2a164a1c776136bb +330, 0x73e03ee3fd80fd1b +331, 0xd2c58c0746b8d858 +332, 0x2ed2cb0038153d22 +333, 0x98996d0fc8ceeacc +334, 0xa4ed0589936b37f +335, 0x5f61cf41a6d2c172 +336, 0xa6d4afb538c110d7 +337, 0xe85834541baadf1a +338, 0x4c8967107fd49212 +339, 0x49bafb762ab1a8c1 +340, 0x45d540e2a834bf17 +341, 0x1c0ec8b4ed671dac +342, 0x3d503ce2c83fe883 +343, 0x437bfffd95f42022 +344, 0xc82d1e3d5c2bc8d2 +345, 0x7a0a9cbfcb0d3f24 +346, 0xc0a4f00251b7a3be +347, 0xb5be24e74bb6a1c6 +348, 0xa3104b94b57545b1 +349, 0x86de7d0c4b97b361 +350, 0x879c1483f26538a6 +351, 0xd74c87557f6accfb +352, 0x2f9be40dbf0fe8a1 +353, 0x445a93398f608d89 +354, 0x7b3cb8a7211d7fdc +355, 0xe86cc51290d031e7 +356, 0x33ef3594052ad79f +357, 0xc61911d241dbb590 +358, 0x37cccb0c0e3de461 +359, 0xb75259124080b48b +360, 0xd81e8961beb4abe5 +361, 0xf4542deb84a754e +362, 0x6ea036d00385f02e +363, 0xa7b60b0ac3b88681 +364, 0x108a6c36ca30baf5 +365, 0x4a2adc5bbfe2bf07 +366, 0x4079501f892a5342 +367, 0x55e113963c5448f0 +368, 0x8019ff4903b37242 +369, 0x109c6dcdb7ec6618 +370, 0x1239ac50944da450 +371, 0xe1399c7f94c651c1 +372, 0x5a6bbbae388d365a +373, 0x4d72be57b8810929 +374, 0x3f067df24384e1fb +375, 0x4f8b9e0f7f6c7be +376, 0x202492c342a3b08 +377, 0x250753192af93a3 +378, 0xfba1159d9de2cb8e +379, 0xba964497ab05505c +380, 0x1329ec5d8a709dca +381, 0x32927cacb6cd22bb +382, 0x6b4d7db904187d56 +383, 0xe76adccf8e841e02 +384, 0x8c4bf4b6a788202 +385, 0x3013a3b409831651 +386, 0x7427d125c475412f +387, 0x84dcc4bb2bf43202 +388, 0x117526f1101372a5 +389, 0xfe95d64b8984bd72 +390, 0x524e129934cc55c1 +391, 0xc3db4b0418c36d30 +392, 0xe1cb2047e9c19f7a +393, 0xea43d6c8d8982795 +394, 0xe80ac8a37df89ed +395, 0xfecc2104329ed306 +396, 0xa5c38aac9c1d51ea +397, 0x3abe5d1c01e4fe17 +398, 0x717a805d97fcc7ac +399, 0x94441f8207a1fb78 +400, 0x22d7869c5f002607 +401, 0x349e899f28c3a1b9 +402, 0x5639950cdea92b75 +403, 0x7e08450497c375b +404, 0x94bf898b475d211d +405, 0x75c761a402375104 +406, 0x1930920ec9d2a1e7 +407, 0xb774ba1bc6f6e4e2 +408, 0xf715602412e5d900 +409, 0x87bb995f4a13f0ba +410, 0xa3c787868dfa9c8d +411, 0xa17fd42a5a4f0987 +412, 0x4a9f7d435242b86 +413, 0x240364aff88f8aef +414, 0xe7cd4cf4bf39f144 +415, 0xd030f313ca4c2692 +416, 0xc46696f4e03ec1e9 +417, 0x22c60f1ec21060b3 +418, 0x16c88058fd68986f +419, 0x69ca448e8e6bde3f +420, 0x3466c2cdec218abd +421, 0x837ac4d05e6b117d +422, 0x911210e154690191 +423, 0x9ece851d6fa358b7 +424, 0x42f79cb0c45e7897 +425, 0xbf7583babd7c499b +426, 0x2059fe8031c6e0b9 +427, 0xabbec8fc00f7e51d +428, 0x88809d86a3a256e1 +429, 0xd36056df829fdcb5 +430, 0x515632b6cb914c64 +431, 0xba76d06c2558874 +432, 0x632c54ca4214d253 +433, 0xadec487adf2cb215 +434, 0x521e663e1940513d +435, 0xb1b638b548806694 +436, 0xbe2d5bfbe57d2c72 +437, 0x8b89e7719db02f7 +438, 0x90ba5281c1d56e63 +439, 0x899e1b92fceea102 +440, 0xf90d918e15182fa6 +441, 0x94a489ce96c948c4 +442, 0xad34db453517fcd4 +443, 0xc5264eb2de15930f +444, 0x101b4e6603a21cee +445, 0xef9b6258d6e85fff +446, 0x6075c7d6c048bd7a +447, 0x6f03232c64e438aa +448, 0x18c983d7105ee469 +449, 0x3ffc23f5c1375879 +450, 0xbc1b4a00afb1f9f +451, 0x5afa6b2bb8c6b46e +452, 0xe7fce4af2f2c152a +453, 0x5b00ab5c4b3982c7 +454, 0x2d4b0c9c0eb4bd0c +455, 0x61d926270642f1f2 +456, 0x7219c485c23a2377 +457, 0x7e471c752fecd895 +458, 0x23c4d30a4d17ba1f +459, 0x65cb277fe565ca22 +460, 0xcbb56ed9c701363b +461, 0xfd04ab3a6eba8282 +462, 0x19c9e5c8bab38500 +463, 0xea4c15227676b65b +464, 0x20f3412606c8da6f +465, 0xb06782d3bf61a239 +466, 0xf96e02d5276a9a31 +467, 0x835d256b42aa52a6 +468, 0x25b09151747f39c1 +469, 0x64507386e1103eda +470, 0x51cbc05716ef88e4 +471, 0x998cd9b7989e81cc +472, 0x9d7115416bec28d1 +473, 0xc992ca39de97906b +474, 0xd571e6f7ca598214 +475, 0xafc7fb6ccd9abbf8 +476, 0x88ef456febff7bf4 +477, 0xdbe87ccc55b157d2 +478, 0xaab95e405f8a4f6d +479, 0xad586a385e74af4f +480, 0x23cd15225c8485aa +481, 0x370940bf47900ac7 +482, 0xefd6afda1a4b0ead +483, 0x9cb1a4c90993dd7a +484, 0xff7893e8b2f70b11 +485, 0xb09e1807c0638e8e +486, 0xb10915dcb4978f74 +487, 0x88212ab0051a85eb +488, 0x7af41b76e1ec793f +489, 0x2e5c486406d3fefd +490, 0xebe54eff67f513cc +491, 0xab6c90d0876a79b8 +492, 0x224df82f93fe9089 +493, 0xc51c1ce053dc9cd2 +494, 0x5ef35a4d8a633ee7 +495, 0x4aca033459c2585f +496, 0xd066932c6eefb23d +497, 0x5309768aab9a7591 +498, 0xa2a3e33823df37f9 +499, 0xcec77ff6a359ee9 +500, 0x784dc62d999d3483 +501, 0x84e789fb8acc985d +502, 0xd590237e86aa60f +503, 0x737e2ffe1c8ad600 +504, 0xc019c3a39a99eab8 +505, 0x6a39e9836964c516 +506, 0xe0fe43129535d9da +507, 0xdfc5f603d639d4de +508, 0x7b9a7d048a9c03b6 +509, 0xbb5aa520faa27fdd +510, 0x2a09b4200f398fa2 +511, 0x38cc88107904064e +512, 0xa9a90d0b2d92bb25 +513, 0x9419762f87e987e3 +514, 0x1a52c525153dedcd +515, 0xc26d9973dd65ae99 +516, 0x8e89bd9d0dc6e6a1 +517, 0x2f30868dc01bfb53 +518, 0x20f09d99b46501c4 +519, 0x78b468a563b8f1e9 +520, 0xcccf34b0b6c380c7 +521, 0xf554e7dc815297e6 +522, 0x332a585cfb4a50ef +523, 0xa9fb64a2b6da41d7 +524, 0xdcd2a5a337391ce0 +525, 0x8a9bd3e324c6463d +526, 0x9f4487d725503bdd +527, 0xf72282d82f1d0ff +528, 0x308f4160abb72d42 +529, 0x648de1db3a601b08 +530, 0x36cab5192e7ebd39 +531, 0x7975fbe4ab6a1c66 +532, 0xd515b4d72243864e +533, 0x43a568f8b915e895 +534, 0x15fa9f2057bdb91d +535, 0x7a43858ef7a222dc +536, 0x17b4a9175ac074fe +537, 0xa932c833b8d0f8f8 +538, 0x1d2db93a9a587678 +539, 0x98abd1d146124d27 +540, 0xf0ab0431671740aa +541, 0xa9d182467540ad33 +542, 0x41c8a6cfc331b7fc +543, 0xa52c6bd0fcd1d228 +544, 0x2773c29a34dc6fa3 +545, 0x3098230746fc1f37 +546, 0xd63311bb4f23fabe +547, 0x6712bf530cd2faec +548, 0x342e8f342e42c4dd +549, 0xfbd83331851cdcad +550, 0xe903be1361bbc34d +551, 0xd94372e5077e3ef9 +552, 0x95aaa234f194bd8 +553, 0x20c0c8fb11e27538 +554, 0xfaf47dc90462b30b +555, 0x8ddc6d144147682a +556, 0xf626833fd926af55 +557, 0x5df93c34290d1793 +558, 0xb06a903e6e9fca5e +559, 0x10c792dc851d77ca +560, 0xd9b1b817b18e56cb +561, 0x3a81730c408eb408 +562, 0x65052c04a8d4b63c +563, 0x3328546598e33742 +564, 0xeca44a13f62d156d +565, 0x69f83d1d86b20170 +566, 0x937764200412027d +567, 0xc57eb1b58df0f191 +568, 0xa1c7d67dce81bc41 +569, 0x8e709c59a6a579ce +570, 0x776a2f5155d46c70 +571, 0xd92906fbbc373aa5 +572, 0xe97ad478a2a98bf6 +573, 0xc296c8819ac815f +574, 0x613ede67ba70e93e +575, 0xe145222498f99cde +576, 0xafcdfa7a3c1cf9bf +577, 0x1c89252176db670d +578, 0xad245eda5c0865ff +579, 0x249463d3053eb917 +580, 0xc9be16d337517c0b +581, 0xefcc82bf67b8f731 +582, 0x1e01577d029e0d00 +583, 0xad9c24b2a4f3d418 +584, 0xed2cceb510db4d0f +585, 0xbddadcdb92400c70 +586, 0x67d6b0476ef82186 +587, 0xbc7662ff7bf19f73 +588, 0x9d94452a729e6e92 +589, 0x6b278d8594f55428 +590, 0x6c4b31cceb1b2109 +591, 0xccc6c3a726701e9 +592, 0x6bc28ece07df8925 +593, 0xc0422b7bf150ccc4 +594, 0xab7158f044e73479 +595, 0xdf3347546d9ed83f +596, 0x3b3235a02c70dff4 +597, 0x2551c49c14ea8d77 +598, 0xee2f7f5bb3cc228e +599, 0x39b87bfe8c882d39 +600, 0x7dd420fad380b51c +601, 0xffe64976af093f96 +602, 0x4a4f48dc6e7eaa5f +603, 0x85f2514d32fdc8cc +604, 0x1ab1215fd7f94801 +605, 0x4cd1200fc795b774 +606, 0xcf8af463a38942ee +607, 0x319caa7ce3022721 +608, 0x8cd9798a76d1aea4 +609, 0x2bd3933ac7afd34e +610, 0x85d4c323403cf811 +611, 0xd7b956d3064efa30 +612, 0x67a078dbf1f13068 +613, 0x665fa6c83e87c290 +614, 0x9333ac2416d2469b +615, 0xdfb1fd21a0094977 +616, 0xa1962a6e2c25f8ff +617, 0x1f3b10a7ed5287cf +618, 0x70641efb3d362713 +619, 0xe527a2cf85d00918 +620, 0x9741e45d3f9890a3 +621, 0x6cb74b5d4d36db4b +622, 0xf24734d622bd2209 +623, 0xadd6d94f78e9d378 +624, 0xc3bbdb59225cca7f +625, 0x5ad36614275b30cd +626, 0x495568dd74eea434 +627, 0xf35de47e0ffe1f2d +628, 0xefa209dca719ab18 +629, 0x844ddcaeb5b99ae8 +630, 0x37449670a1dc7b19 +631, 0x5a4612c166f845c1 +632, 0xe70f7782f2087947 +633, 0x98d484deac365721 +634, 0x705302198cf52457 +635, 0x7135ae0f5b77df41 +636, 0x342ac6e44a9b6fc3 +637, 0x2713fd2a59af5826 +638, 0x6e1a3f90f84efa75 +639, 0x9fb3b4dd446ca040 +640, 0x530044ae91e6bd49 +641, 0xe984c4183974dc3e +642, 0x40c1fa961997d066 +643, 0xb7868250d8c21559 +644, 0x8bc929fa085fd1de +645, 0x7bdb63288dc8733e +646, 0xac4faad24326a468 +647, 0x1c6e799833aea0b1 +648, 0xcc8a749e94f20f36 +649, 0x4e7abfd0443547c5 +650, 0xb661c73bb8caa358 +651, 0x4a800f5728ff2351 +652, 0x8c15e15189b9f7ed +653, 0xab367846b811362c +654, 0x4ba7508f0851ca2a +655, 0xe9af891acbafc356 +656, 0xbdebe183989601f8 +657, 0x4c665ea496afc061 +658, 0x3ca1d14a5f2ed7c +659, 0xfbdff10a1027dd21 +660, 0xdfd28f77c8cff968 +661, 0xc4fbaadf8a3e9c77 +662, 0xdac7e448b218c589 +663, 0xb26390b5befd19e2 +664, 0xd2ef14916c66dba9 +665, 0xfab600284b0ff86b +666, 0xf04a1c229b58dabb +667, 0xc21c45637e452476 +668, 0xd1435966f75e0791 +669, 0xc1f28522eda4a2d0 +670, 0x52332ae8f1222185 +671, 0x81c6c0790c0bf47e +672, 0xfebd215e7d8ffb86 +673, 0x68c5dce55dbe962b +674, 0x231d09cb0d2531d1 +675, 0x3218fba199dbbc6b +676, 0x8f23c535f8ea0bf6 +677, 0x6c228963e1df8bd9 +678, 0x9843c7722ed153e3 +679, 0xd032d99e419bddec +680, 0xe2dca88aa7814cab +681, 0x4d53fb8c6a59cdc2 +682, 0x8fb3abc46157b68b +683, 0xa3e733087e09b8e +684, 0x6bdc1aee029d6b96 +685, 0x4089667a8906d65b +686, 0x8f3026a52d39dd03 +687, 0x6d2e0ccb567bae84 +688, 0x74bad450199e464 +689, 0xf114fb68a8f300d5 +690, 0xc7a5cc7b374c7d10 +691, 0xf0e93da639b279d1 +692, 0xb9943841ad493166 +693, 0x77a69290455a3664 +694, 0x41530da2ebea054b +695, 0xe8f9fab03ea24abf +696, 0xaa931f0c9f55a57a +697, 0xb4d68a75d56f97ae +698, 0x3d58ff898b6ba297 +699, 0x49d81e08faf5a3f5 +700, 0xfc5207b9f3697f3b +701, 0xa25911abb3cf19b7 +702, 0x6b8908eb67c3a41 +703, 0xd63ef402e2e3fa33 +704, 0x728e75d3f33b14c5 +705, 0x248cb1b8bc6f379a +706, 0x3aa3d6d2b8c72996 +707, 0x49cc50bd2d3d2860 +708, 0xb4e1387647c72075 +709, 0x435a1630a4a81ed3 +710, 0xa5ea13005d2460cf +711, 0xc7a613df37d159ec +712, 0x95721ccc218b857e +713, 0xd4b70d8c86b124d3 +714, 0x2b82bcc4b612d494 +715, 0xaf13062885276050 +716, 0xcbd8fcf571a33d9c +717, 0x3f7f67ca1125fc15 +718, 0xddf4bb45aac81b4c +719, 0x23606da62de9c040 +720, 0xa3a172375666b636 +721, 0x292f87387a6c6c3c +722, 0xd1d10d00c5496fe1 +723, 0x86b0411ce8a25550 +724, 0x38e0487872e33976 +725, 0x363e49f88ddfd42c +726, 0x45bdf1e9f6b66b0a +727, 0x8a6fff3de394f9b5 +728, 0x8502158bb03f6209 +729, 0x22e24d16dba42907 +730, 0x3fe3ba427cc2b779 +731, 0x77144793f66b3d7e +732, 0xcf8912ccb29b8af9 +733, 0xdc856caff2abd670 +734, 0xe6d3ae0b0d9d4c8b +735, 0xb8f5d40e454c539f +736, 0x79ca953114fbc6b7 +737, 0x478d6f4bbfa38837 +738, 0x9babae1a3ffdc340 +739, 0x40edd56802bae613 +740, 0x97a56c2dcccf0641 +741, 0xafc250257f027f8e +742, 0x8da41ef1edf69125 +743, 0x6574b0280ff9d309 +744, 0x197c776151b8f820 +745, 0x6b03e077c9dac3b6 +746, 0x24a40ebbc5c341c5 +747, 0x50e585169a6a1c4b +748, 0x37783a5a6a3e4e02 +749, 0xb3de81ee6fbad647 +750, 0xf4f292f57ca4591e +751, 0x6214e9e7d44d30a +752, 0x5920190c56d21c12 +753, 0x9ac163419b5e0c9b +754, 0xfc2328761ae8ed93 +755, 0xc68f945b545508c6 +756, 0x687c49a17ce0a5e2 +757, 0x276d8f53d30d4ab4 +758, 0x8201804970343ce1 +759, 0x1b5d323cc2e7fb7e +760, 0x6f351ef04fd904b +761, 0x6c793a7d455d5198 +762, 0x46f5d108430ae91f +763, 0xac16a15b2a0cf77f +764, 0xa0d479d9e4122b9d +765, 0x3afd94604307f19 +766, 0x2573ed6d39d38dbf +767, 0xa58e14ba60b4294b +768, 0xe69c1aed5840d156 +769, 0x4cf6fda7f04855c2 +770, 0x2fb65a56ef5f22da +771, 0xf95819434d5dc220 +772, 0x29c65133623dafba +773, 0x8e997bd018467523 +774, 0xfd08ba9d498461a7 +775, 0xdd52243bc78a5592 +776, 0x39c30108f6db88b3 +777, 0x38af8e1894f259b9 +778, 0x97eedf3b4ae5f6de +779, 0x757825add80c5ece +780, 0xf0fdd90ac14edb14 +781, 0xbbb19d4cc8cac6d4 +782, 0x9a82234edfae05e3 +783, 0x704401c61d1edf1c +784, 0x8b0eb481fb3a1fb2 +785, 0xef6f36e7cc06c002 +786, 0x7a208b17e04b8cd7 +787, 0xf20e33d498838fe9 +788, 0xc2bdb22117058326 +789, 0x6ec31939eb4ca543 +790, 0x6f1654838f507a21 +791, 0xc65ab81a955d2b93 +792, 0x40b1420fdd9531b8 +793, 0xe31f221cab9f4f40 +794, 0x798cdd414c1deb7a +795, 0x9c84e9c7d41cd983 +796, 0x63d6b1ae3b60b7fa +797, 0xb42bfdd1a2f78ffa +798, 0x37e431eaccaaa8e9 +799, 0x7508142a0f73eac9 +800, 0x91662a023df5893a +801, 0x59782070e2fe3031 +802, 0xb2acd589a8ce7961 +803, 0xa224743fa877b292 +804, 0xaa5362aa27e6ed9e +805, 0xa394a4e520c0c1c7 +806, 0xe49b16d2018ffb6f +807, 0xb8074b9f2f1e762b +808, 0xcf5f86143d5c23a7 +809, 0xfd838785db987087 +810, 0x31b1889df389aff8 +811, 0x30aaca876a4383b +812, 0x1731bb71c4c38d4f +813, 0x9a83a65395e05458 +814, 0x99cd0c8d67c8f4fc +815, 0xfbd9fdc849b761a5 +816, 0x82c04834fc466889 +817, 0xdeef9d6e715e8c97 +818, 0x549c281c16da6078 +819, 0x2d70661254ad599d +820, 0x57995793a72acac +821, 0xf1727005116183ba +822, 0xa22bb38945285de3 +823, 0x4f2d687fe45131ff +824, 0x5666c87ddbbc981f +825, 0xbcb4b2d4e7a517d0 +826, 0x5e794dd2e20b785d +827, 0x449ad020149e093c +828, 0x7704ee0412d106f5 +829, 0x83cbdf257b072ac1 +830, 0xae5c4fc9f638b0da +831, 0x7b9e5a64e372ed47 +832, 0x7eddbbb22c2cdf57 +833, 0x3f19ebfa155b08e +834, 0x91d991154dfd7177 +835, 0x611ae74b952d387f +836, 0x3fdf7a335bda36ee +837, 0xdf182433fc7a7c05 +838, 0x62c78598d1f8db0a +839, 0xc3750c69d2c5c1f0 +840, 0xf1318024709efdee +841, 0xaa3fd360d224dc29 +842, 0x62af53b2f307c19 +843, 0xdf527683c58120c2 +844, 0x3281deecc496f93d +845, 0x4f704ad31527ef08 +846, 0x127a14a5e07cfdfc +847, 0x90d0b1f549255c92 +848, 0xbc3406b212c5e1fc +849, 0x4e89f39379dba91d +850, 0x1290ef43c4998e6e +851, 0xecfeb1a1cb1c6e1b +852, 0x2067e90403003bf1 +853, 0x38ae04be30bdbeba +854, 0x8a3537f298baedda +855, 0xd07f3b825cdb2936 +856, 0xea020b5aebae8b45 +857, 0xfcd614ab031132b0 +858, 0x5fb682a4ff2268f5 +859, 0xd1c4662ce65596f4 +860, 0x7026b8270dd0b8dc +861, 0x8101ec4b4beae45a +862, 0xa0e9dc87940610a6 +863, 0x83ec33679d83165b +864, 0x981847ca82e86d41 +865, 0xda84c188a304a0b7 +866, 0x3c37529c5a5bbbb8 +867, 0x34a8491ce3e19a5a +868, 0xd36ad716a2fa6cb8 +869, 0xfd1d1d6a5189a15c +870, 0x9716eb47851e8d8d +871, 0x7dfb13ea3b15c5aa +872, 0xbdf6e707f45113a5 +873, 0xb8118261b04bd097 +874, 0x6191f9895881bec6 +875, 0x7aac257ae11acf9b +876, 0x35a491e1537ff120 +877, 0xe078943432efa71c +878, 0xb3338485dd3dc2b9 +879, 0x456060975d2bb3b5 +880, 0xaddc4c451bdfc44c +881, 0x18bfa7beacf96430 +882, 0x8802ebcaf0f67498 +883, 0xad922a5a825bd780 +884, 0x9fb4587d748f4efa +885, 0xdb2a445136cd5e7 +886, 0xb98b3676ea8e96ac +887, 0xb02d8d244d784878 +888, 0xa1a8442b18860abb +889, 0x6a3029ba1361e5d1 +890, 0xf426d5fac161eb1 +891, 0xfa5ac2b87acecb23 +892, 0xaa659896e50535df +893, 0xf40dd7a3d3c5c8ed +894, 0x3f8367abecb705bc +895, 0x2d60e7525873358f +896, 0xc4a9d3948a0c3937 +897, 0x5ecc04fef6003909 +898, 0x7a865004918cba2 +899, 0x47ae110a678ec10b +900, 0xa0f02f629d91aa67 +901, 0x4848b99e7fac9347 +902, 0xaa858346d63b80ac +903, 0xeb5bf42ee161eeef +904, 0x4d35d723d3c6ba37 +905, 0xdf22ca6ca93b64a7 +906, 0x9d198520f97b25b1 +907, 0x3068415350778efe +908, 0xf3709f2e8793c2fe +909, 0xd1517bac8dd9f16f +910, 0xfb99bccaa15861dc +911, 0xa9ad607d796a2521 +912, 0x55d3793d36bd22e4 +913, 0xf99270d891ff7401 +914, 0x401750a5c4aa8238 +915, 0xd84b3003e6f28309 +916, 0x8a23798b5fa7c98b +917, 0xadd58bbc8f43e399 +918, 0xbd8c741ada62c6a8 +919, 0xbdc6937bc55b49fa +920, 0x4aefa82201b8502 +921, 0x17adf29a717b303 +922, 0xa6ed2197be168f6c +923, 0x1ba47543f4359a95 +924, 0xe34299949ac01ae9 +925, 0x711c76cffc9b62f3 +926, 0xbac259895508a4b7 +927, 0x3c8b3b3626b0d900 +928, 0x1a8d23fbe2ae71bf +929, 0xca984fa3b5a5c3a1 +930, 0xb1986ab7521a9c93 +931, 0xd6b5b2c8d47a75b5 +932, 0xc7f1c4a88afb4957 +933, 0xdeb58033a3acd6cc +934, 0xabe49ddfe1167e67 +935, 0x8d559c10205c06e3 +936, 0xea07a1a7de67a651 +937, 0xcbef60db15b6fef8 +938, 0xbfca142cff280e7 +939, 0x362693eba0732221 +940, 0x7463237e134db103 +941, 0x45574ddb5035e17a +942, 0xfc65e0cb9b94a1aa +943, 0x3154c55f1d86b36d +944, 0x2d93a96dd6ab2d8b +945, 0xbe3bc1d1f2542a25 +946, 0xdd4b541f7385bdaa +947, 0x3b56b919d914e3f8 +948, 0x82fd51468a21895f +949, 0x8988cf120731b916 +950, 0xa06a61db5fb93e32 +951, 0x6ed66c1b36f68623 +952, 0x875ae844d2f01c59 +953, 0x17ccd7ac912e5925 +954, 0x12fe2a66b8e40cb1 +955, 0xf843e5e3923ad791 +956, 0xa17560f2fd4ef48 +957, 0x27a2968191a8ee07 +958, 0xa9aab4d22ff44a3c +959, 0x63cd0dcc3bb083ae +960, 0x7a30b48c6160bf85 +961, 0x956160fb572503b3 +962, 0xc47f6b7546640257 +963, 0xaf4b625f7f49153 +964, 0x2f5c86a790e0c7e8 +965, 0xb52e0610ae07f0b8 +966, 0x38a589292c3d849e +967, 0xc3e9ef655d30b4ef +968, 0xb5695f765cda998a +969, 0xde5d5e692a028e91 +970, 0x839476721555f72e +971, 0x48b20679b17d9ebf +972, 0xe3d4c6b2c26fb0df +973, 0xce5a9834f0b4e71f +974, 0x533abb253d5d420e +975, 0x9eac5ad9aed34627 +976, 0xc0f2a01ab3c90dbb +977, 0x6528eda93f6a066c +978, 0xc16a1b625e467ade +979, 0x1a4a320fb5e8b098 +980, 0x8819cccd8b4ab32f +981, 0x42daa88531fd0bfd +982, 0xcf732226409be17c +983, 0xfddcdb25ccbf378c +984, 0x9b15b603bf589fc1 +985, 0x2436066b95d366fe +986, 0x8d42eff2e9cbda90 +987, 0x694b2fc8a4e8303c +988, 0x8e207f98aaea3ccd +989, 0x4730d7a620f822d9 +990, 0x468dc9ca30fe2fd4 +991, 0x74b36d8a1c0f031b +992, 0x3c1aac1c488c1a94 +993, 0x19d0101042444585 +994, 0x8ec50c56d0c8adf4 +995, 0x721ec629e4d66394 +996, 0x3ca5ad93abeac4a4 +997, 0xaaebc76e71592623 +998, 0x969cc319e3ed6058 +999, 0xc0a277e3b2bfc3de diff --git a/numpy/random/tests/test_direct.py b/numpy/random/tests/test_direct.py index d602b36b4169..29054b70b95a 100644 --- a/numpy/random/tests/test_direct.py +++ b/numpy/random/tests/test_direct.py @@ -8,8 +8,8 @@ import pytest from numpy.random import ( - Generator, MT19937, PCG64, Philox, RandomState, SeedSequence, SFC64, - default_rng + Generator, MT19937, PCG64, PCG64DXSM, Philox, RandomState, SeedSequence, + SFC64, default_rng ) from numpy.random._common import interface @@ -359,6 +359,34 @@ def test_advance_symmetry(self): assert val_big == val_pos +class TestPCG64DXSM(Base): + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64DXSM + cls.bits = 64 + cls.dtype = np.uint64 + cls.data1 = cls._read_csv(join(pwd, './data/pcg64dxsm-testset-1.csv')) + cls.data2 = cls._read_csv(join(pwd, './data/pcg64dxsm-testset-2.csv')) + cls.seed_error_type = (ValueError, TypeError) + cls.invalid_init_types = [(3.2,), ([None],), (1, None)] + cls.invalid_init_values = [(-1,)] + + def test_advance_symmetry(self): + rs = Generator(self.bit_generator(*self.data1['seed'])) + state = rs.bit_generator.state + step = -0x9e3779b97f4a7c150000000000000000 + rs.bit_generator.advance(step) + val_neg = rs.integers(10) + rs.bit_generator.state = state + rs.bit_generator.advance(2**128 + step) + val_pos = rs.integers(10) + rs.bit_generator.state = state + rs.bit_generator.advance(10 * 2**128 + step) + val_big = rs.integers(10) + assert val_neg == val_pos + assert val_big == val_pos + + class TestMT19937(Base): @classmethod def setup_class(cls): diff --git a/numpy/random/tests/test_generator_mt19937.py b/numpy/random/tests/test_generator_mt19937.py index b69cd38d4a4a..4abcf6fe4694 100644 --- a/numpy/random/tests/test_generator_mt19937.py +++ b/numpy/random/tests/test_generator_mt19937.py @@ -10,7 +10,7 @@ assert_warns, assert_no_warnings, assert_array_equal, assert_array_almost_equal, suppress_warnings) -from numpy.random import Generator, MT19937, SeedSequence +from numpy.random import Generator, MT19937, SeedSequence, RandomState random = Generator(MT19937()) @@ -142,6 +142,14 @@ def test_multidimensional_pvals(self): assert_raises(ValueError, random.multinomial, 10, [[[0], [1]], [[1], [0]]]) assert_raises(ValueError, random.multinomial, 10, np.array([[0, 1], [1, 0]])) + def test_multinomial_pvals_float32(self): + x = np.array([9.9e-01, 9.9e-01, 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09, + 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09], dtype=np.float32) + pvals = x / x.sum() + random = Generator(MT19937(1432985819)) + match = r"[\w\s]*pvals array is cast to 64-bit floating" + with pytest.raises(ValueError, match=match): + random.multinomial(1, pvals) class TestMultivariateHypergeometric: @@ -960,6 +968,14 @@ def test_shuffle_custom_axis(self): random.shuffle(actual, axis=-1) assert_array_equal(actual, desired) + def test_shuffle_custom_axis_empty(self): + random = Generator(MT19937(self.seed)) + desired = np.array([]).reshape((0, 6)) + for axis in (0, 1): + actual = np.array([]).reshape((0, 6)) + random.shuffle(actual, axis=axis) + assert_array_equal(actual, desired) + def test_shuffle_axis_nonsquare(self): y1 = np.arange(20).reshape(2, 10) y2 = y1.copy() @@ -993,6 +1009,11 @@ def test_shuffle_exceptions(self): arr = [[1, 2, 3], [4, 5, 6]] assert_raises(NotImplementedError, random.shuffle, arr, 1) + arr = np.array(3) + assert_raises(TypeError, random.shuffle, arr) + arr = np.ones((3, 2)) + assert_raises(np.AxisError, random.shuffle, arr, 2) + def test_permutation(self): random = Generator(MT19937(self.seed)) alist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] @@ -1004,7 +1025,7 @@ def test_permutation(self): arr_2d = np.atleast_2d([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]).T actual = random.permutation(arr_2d) assert_array_equal(actual, np.atleast_2d(desired).T) - + bad_x_str = "abcd" assert_raises(np.AxisError, random.permutation, bad_x_str) @@ -1231,9 +1252,9 @@ def test_gamma_0(self): def test_geometric(self): random = Generator(MT19937(self.seed)) actual = random.geometric(.123456789, size=(3, 2)) - desired = np.array([[ 1, 10], - [ 1, 12], - [ 9, 10]]) + desired = np.array([[1, 11], + [1, 12], + [11, 17]]) assert_array_equal(actual, desired) def test_geometric_exceptions(self): @@ -1536,9 +1557,9 @@ def test_power(self): def test_rayleigh(self): random = Generator(MT19937(self.seed)) actual = random.rayleigh(scale=10, size=(3, 2)) - desired = np.array([[ 4.51734079831581, 15.6802442485758 ], - [ 4.19850651287094, 17.08718809823704], - [14.7907457708776 , 15.85545333419775]]) + desired = np.array([[4.19494429102666, 16.66920198906598], + [3.67184544902662, 17.74695521962917], + [16.27935397855501, 21.08355560691792]]) assert_array_almost_equal(actual, desired, decimal=14) def test_rayleigh_0(self): @@ -1666,6 +1687,21 @@ def test_uniform_range_bounds(self): # DBL_MAX by increasing fmin a bit random.uniform(low=np.nextafter(fmin, 1), high=fmax / 1e17) + def test_uniform_zero_range(self): + func = random.uniform + result = func(1.5, 1.5) + assert_allclose(result, 1.5) + result = func([0.0, np.pi], [0.0, np.pi]) + assert_allclose(result, [0.0, np.pi]) + result = func([[2145.12], [2145.12]], [2145.12, 2145.12]) + assert_allclose(result, 2145.12 + np.zeros((2, 2))) + + def test_uniform_neg_range(self): + func = random.uniform + assert_raises(ValueError, func, 2, 1) + assert_raises(ValueError, func, [1, 2], [1, 1]) + assert_raises(ValueError, func, [[0, 1],[2, 3]], 2) + def test_scalar_exception_propagation(self): # Tests that exceptions are correctly propagated in distributions # when called with objects that throw exceptions when converted to @@ -1707,6 +1743,26 @@ def test_vonmises_nan(self): r = random.vonmises(mu=0., kappa=np.nan) assert_(np.isnan(r)) + @pytest.mark.parametrize("kappa", [1e4, 1e15]) + def test_vonmises_large_kappa(self, kappa): + random = Generator(MT19937(self.seed)) + rs = RandomState(random.bit_generator) + state = random.bit_generator.state + + random_state_vals = rs.vonmises(0, kappa, size=10) + random.bit_generator.state = state + gen_vals = random.vonmises(0, kappa, size=10) + if kappa < 1e6: + assert_allclose(random_state_vals, gen_vals) + else: + assert np.all(random_state_vals != gen_vals) + + @pytest.mark.parametrize("mu", [-7., -np.pi, -3.1, np.pi, 3.2]) + @pytest.mark.parametrize("kappa", [1e-9, 1e-6, 1, 1e3, 1e15]) + def test_vonmises_large_kappa_range(self, mu, kappa): + r = random.vonmises(mu, kappa, 50) + assert_(np.all(r > -np.pi) and np.all(r <= np.pi)) + def test_wald(self): random = Generator(MT19937(self.seed)) actual = random.wald(mean=1.23, scale=1.54, size=(3, 2)) @@ -2058,7 +2114,11 @@ def test_lognormal(self): def test_rayleigh(self): scale = [1] bad_scale = [-1] - desired = np.array([0.60439534475066, 0.66120048396359, 1.67873398389499]) + desired = np.array( + [1.1597068009872629, + 0.6539188836253857, + 1.1981526554349398] + ) random = Generator(MT19937(self.seed)) actual = random.rayleigh(scale * 3) @@ -2498,3 +2558,49 @@ def test_broadcast_size_scalar(): random.normal(mu, sigma, size=3) with pytest.raises(ValueError): random.normal(mu, sigma, size=2) + + +def test_ragged_shuffle(): + # GH 18142 + seq = [[], [], 1] + gen = Generator(MT19937(0)) + assert_no_warnings(gen.shuffle, seq) + assert seq == [1, [], []] + + +@pytest.mark.parametrize("high", [-2, [-2]]) +@pytest.mark.parametrize("endpoint", [True, False]) +def test_single_arg_integer_exception(high, endpoint): + # GH 14333 + gen = Generator(MT19937(0)) + msg = 'high < 0' if endpoint else 'high <= 0' + with pytest.raises(ValueError, match=msg): + gen.integers(high, endpoint=endpoint) + msg = 'low > high' if endpoint else 'low >= high' + with pytest.raises(ValueError, match=msg): + gen.integers(-1, high, endpoint=endpoint) + with pytest.raises(ValueError, match=msg): + gen.integers([-1], high, endpoint=endpoint) + + +@pytest.mark.parametrize("dtype", ["f4", "f8"]) +def test_c_contig_req_out(dtype): + # GH 18704 + out = np.empty((2, 3), order="F", dtype=dtype) + shape = [1, 2, 3] + with pytest.raises(ValueError, match="Supplied output array"): + random.standard_gamma(shape, out=out, dtype=dtype) + with pytest.raises(ValueError, match="Supplied output array"): + random.standard_gamma(shape, out=out, size=out.shape, dtype=dtype) + + +@pytest.mark.parametrize("dtype", ["f4", "f8"]) +@pytest.mark.parametrize("order", ["F", "C"]) +@pytest.mark.parametrize("dist", [random.standard_normal, random.random]) +def test_contig_req_out(dist, order, dtype): + # GH 18704 + out = np.empty((2, 3), dtype=dtype, order=order) + variates = dist(out=out, dtype=dtype) + assert variates is out + variates = dist(out=out, dtype=dtype, size=out.shape) + assert variates is out diff --git a/numpy/random/tests/test_generator_mt19937_regressions.py b/numpy/random/tests/test_generator_mt19937_regressions.py index 2ef6b0631920..9f6dcdc6bae8 100644 --- a/numpy/random/tests/test_generator_mt19937_regressions.py +++ b/numpy/random/tests/test_generator_mt19937_regressions.py @@ -1,14 +1,14 @@ from numpy.testing import (assert_, assert_array_equal) import numpy as np import pytest -from numpy.random import Generator, MT19937 +from numpy.random import Generator, MT19937, RandomState mt19937 = Generator(MT19937()) class TestRegression: - def test_VonMises_range(self): + def test_vonmises_range(self): # Make sure generated random variables are in [-pi, pi]. # Regression test for ticket #986. for mu in np.linspace(-7., 7., 5): diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index c13fc39e3339..6a584a511e1c 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -1,5 +1,7 @@ import warnings +import pytest + import numpy as np from numpy.testing import ( assert_, assert_raises, assert_equal, assert_warns, @@ -510,6 +512,58 @@ def test_shuffle_masked(self): assert_equal( sorted(b.data[~b.mask]), sorted(b_orig.data[~b_orig.mask])) + @pytest.mark.parametrize("random", + [np.random, np.random.RandomState(), np.random.default_rng()]) + def test_shuffle_untyped_warning(self, random): + # Create a dict works like a sequence but isn't one + values = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6} + with pytest.warns(UserWarning, + match="you are shuffling a 'dict' object") as rec: + random.shuffle(values) + assert "test_random" in rec[0].filename + + @pytest.mark.parametrize("random", + [np.random, np.random.RandomState(), np.random.default_rng()]) + @pytest.mark.parametrize("use_array_like", [True, False]) + def test_shuffle_no_object_unpacking(self, random, use_array_like): + class MyArr(np.ndarray): + pass + + items = [ + None, np.array([3]), np.float64(3), np.array(10), np.float64(7) + ] + arr = np.array(items, dtype=object) + item_ids = {id(i) for i in items} + if use_array_like: + arr = arr.view(MyArr) + + # The array was created fine, and did not modify any objects: + assert all(id(i) in item_ids for i in arr) + + if use_array_like and not isinstance(random, np.random.Generator): + # The old API gives incorrect results, but warns about it. + with pytest.warns(UserWarning, + match="Shuffling a one dimensional array.*"): + random.shuffle(arr) + else: + random.shuffle(arr) + assert all(id(i) in item_ids for i in arr) + + def test_shuffle_memoryview(self): + # gh-18273 + # allow graceful handling of memoryviews + # (treat the same as arrays) + np.random.seed(self.seed) + a = np.arange(5).data + np.random.shuffle(a) + assert_equal(np.asarray(a), [0, 1, 4, 3, 2]) + rng = np.random.RandomState(self.seed) + rng.shuffle(a) + assert_equal(np.asarray(a), [0, 1, 2, 3, 4]) + rng = np.random.default_rng(self.seed) + rng.shuffle(a) + assert_equal(np.asarray(a), [4, 1, 0, 3, 2]) + def test_beta(self): np.random.seed(self.seed) actual = np.random.beta(.1, .9, size=(3, 2)) diff --git a/numpy/random/tests/test_randomstate.py b/numpy/random/tests/test_randomstate.py index b70a043472cc..861813a95d1f 100644 --- a/numpy/random/tests/test_randomstate.py +++ b/numpy/random/tests/test_randomstate.py @@ -167,6 +167,14 @@ def test_p_non_contiguous(self): contig = random.multinomial(100, pvals=np.ascontiguousarray(pvals)) assert_array_equal(non_contig, contig) + def test_multinomial_pvals_float32(self): + x = np.array([9.9e-01, 9.9e-01, 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09, + 1.0e-09, 1.0e-09, 1.0e-09, 1.0e-09], dtype=np.float32) + pvals = x / x.sum() + match = r"[\w\s]*pvals array is cast to 64-bit floating" + with pytest.raises(ValueError, match=match): + random.multinomial(1, pvals) + class TestSetState: def setup(self): @@ -642,7 +650,7 @@ def test_choice_nan_probabilities(self): a = np.array([42, 1, 2]) p = [None, None, None] assert_raises(ValueError, random.choice, a, p=p) - + def test_choice_p_non_contiguous(self): p = np.ones(10) / 5 p[1::2] = 3.0 @@ -699,6 +707,10 @@ def test_shuffle_masked(self): assert_equal( sorted(b.data[~b.mask]), sorted(b_orig.data[~b_orig.mask])) + def test_shuffle_invalid_objects(self): + x = np.array(3) + assert_raises(TypeError, random.shuffle, x) + def test_permutation(self): random.seed(self.seed) alist = [1, 2, 3, 4, 5, 6, 7, 8, 9, 0] @@ -1234,6 +1246,15 @@ def test_vonmises_small(self): r = random.vonmises(mu=0., kappa=1.1e-8, size=10**6) assert_(np.isfinite(r).all()) + def test_vonmises_large(self): + # guard against changes in RandomState when Generator is fixed + random.seed(self.seed) + actual = random.vonmises(mu=0., kappa=1e7, size=3) + desired = np.array([4.634253748521111e-04, + 3.558873596114509e-04, + -2.337119622577433e-04]) + assert_array_almost_equal(actual, desired, decimal=8) + def test_vonmises_nan(self): random.seed(self.seed) r = random.vonmises(mu=0., kappa=np.nan) diff --git a/numpy/random/tests/test_smoke.py b/numpy/random/tests/test_smoke.py index 909bfaa8dab6..9becc434d0d1 100644 --- a/numpy/random/tests/test_smoke.py +++ b/numpy/random/tests/test_smoke.py @@ -4,7 +4,7 @@ import numpy as np import pytest from numpy.testing import assert_equal, assert_, assert_array_equal -from numpy.random import (Generator, MT19937, PCG64, Philox, SFC64) +from numpy.random import (Generator, MT19937, PCG64, PCG64DXSM, Philox, SFC64) @pytest.fixture(scope='module', params=(np.bool_, np.int8, np.int16, np.int32, np.int64, @@ -774,6 +774,18 @@ def setup_class(cls): cls._extra_setup() +class TestPCG64DXSM(RNG): + @classmethod + def setup_class(cls): + cls.bit_generator = PCG64DXSM + cls.advance = 2**63 + 2**31 + 2**15 + 1 + cls.seed = [12345] + cls.rg = Generator(cls.bit_generator(*cls.seed)) + cls.initial_state = cls.rg.bit_generator.state + cls.seed_vector_bits = 64 + cls._extra_setup() + + class TestDefaultRNG(RNG): @classmethod def setup_class(cls): diff --git a/numpy/rec.pyi b/numpy/rec.pyi index c70ee5374ec4..198636058a27 100644 --- a/numpy/rec.pyi +++ b/numpy/rec.pyi @@ -1,5 +1,65 @@ -from typing import Any +from typing import List -record: Any -recarray: Any -format_parser: Any +from numpy import ( + format_parser as format_parser, + record as record, + recarray as recarray, +) + +__all__: List[str] + +def fromarrays( + arrayList, + dtype=..., + shape=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., +): ... +def fromrecords( + recList, + dtype=..., + shape=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., +): ... +def fromstring( + datastring, + dtype=..., + shape=..., + offset=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., +): ... +def fromfile( + fd, + dtype=..., + shape=..., + offset=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., +): ... +def array( + obj, + dtype=..., + shape=..., + offset=..., + strides=..., + formats=..., + names=..., + titles=..., + aligned=..., + byteorder=..., + copy=..., +): ... diff --git a/numpy/testing/__init__.pyi b/numpy/testing/__init__.pyi index c394a387d5df..395626f6bb5a 100644 --- a/numpy/testing/__init__.pyi +++ b/numpy/testing/__init__.pyi @@ -1,44 +1,113 @@ -from typing import Any - -assert_equal: Any -assert_almost_equal: Any -assert_approx_equal: Any -assert_array_equal: Any -assert_array_less: Any -assert_string_equal: Any -assert_array_almost_equal: Any -assert_raises: Any -build_err_msg: Any -decorate_methods: Any -jiffies: Any -memusage: Any -print_assert_equal: Any -raises: Any -rundocs: Any -runstring: Any -verbose: Any -measure: Any -assert_: Any -assert_array_almost_equal_nulp: Any -assert_raises_regex: Any -assert_array_max_ulp: Any -assert_warns: Any -assert_no_warnings: Any -assert_allclose: Any -IgnoreException: Any -clear_and_catch_warnings: Any -SkipTest: Any -KnownFailureException: Any -temppath: Any -tempdir: Any -IS_PYPY: Any -HAS_REFCOUNT: Any -suppress_warnings: Any -assert_array_compare: Any -_assert_valid_refcount: Any -_gen_alignment_data: Any -assert_no_gc_cycles: Any -break_cycles: Any -HAS_LAPACK64: Any -TestCase: Any -run_module_suite: Any +import sys +import warnings +from typing import Any, List, ClassVar, Tuple, Set + +if sys.version_info >= (3, 8): + from typing import Final +else: + from typing_extensions import Final + +from unittest import ( + TestCase as TestCase, +) + +from unittest.case import ( + SkipTest as SkipTest, +) + +__all__: List[str] + +def run_module_suite(file_to_run=..., argv=...): ... + +class KnownFailureException(Exception): ... +class IgnoreException(Exception): ... + +class clear_and_catch_warnings(warnings.catch_warnings): + class_modules: ClassVar[Tuple[str, ...]] + modules: Set[str] + def __init__(self, record=..., modules=...): ... + def __enter__(self): ... + def __exit__(self, *exc_info): ... + +class suppress_warnings: + log: List[warnings.WarningMessage] + def __init__(self, forwarding_rule=...): ... + def filter(self, category=..., message=..., module=...): ... + def record(self, category=..., message=..., module=...): ... + def __enter__(self): ... + def __exit__(self, *exc_info): ... + def __call__(self, func): ... + +verbose: int +IS_PYPY: Final[bool] +HAS_REFCOUNT: Final[bool] +HAS_LAPACK64: Final[bool] + +def assert_(val, msg=...): ... +def memusage(processName=..., instance=...): ... +def jiffies(_proc_pid_stat=..., _load_time=...): ... +def build_err_msg( + arrays, + err_msg, + header=..., + verbose=..., + names=..., + precision=..., +): ... +def assert_equal(actual, desired, err_msg=..., verbose=...): ... +def print_assert_equal(test_string, actual, desired): ... +def assert_almost_equal( + actual, + desired, + decimal=..., + err_msg=..., + verbose=..., +): ... +def assert_approx_equal( + actual, + desired, + significant=..., + err_msg=..., + verbose=..., +): ... +def assert_array_compare( + comparison, + x, + y, + err_msg=..., + verbose=..., + header=..., + precision=..., + equal_nan=..., + equal_inf=..., +): ... +def assert_array_equal(x, y, err_msg=..., verbose=...): ... +def assert_array_almost_equal(x, y, decimal=..., err_msg=..., verbose=...): ... +def assert_array_less(x, y, err_msg=..., verbose=...): ... +def runstring(astr, dict): ... +def assert_string_equal(actual, desired): ... +def rundocs(filename=..., raise_on_error=...): ... +def raises(*args): ... +def assert_raises(*args, **kwargs): ... +def assert_raises_regex(exception_class, expected_regexp, *args, **kwargs): ... +def decorate_methods(cls, decorator, testmatch=...): ... +def measure(code_str, times=..., label=...): ... +def assert_allclose( + actual, + desired, + rtol=..., + atol=..., + equal_nan=..., + err_msg=..., + verbose=..., +): ... +def assert_array_almost_equal_nulp(x, y, nulp=...): ... +def assert_array_max_ulp(a, b, maxulp=..., dtype=...): ... +def assert_warns(warning_class, *args, **kwargs): ... +def assert_no_warnings(*args, **kwargs): ... +def tempdir(*args, **kwargs): ... +def temppath(*args, **kwargs): ... +def assert_no_gc_cycles(*args, **kwargs): ... +def break_cycles(): ... +def _assert_valid_refcount(op): ... +def _gen_alignment_data(dtype=..., type=..., max_size=...): ... diff --git a/numpy/testing/_private/decorators.py b/numpy/testing/_private/decorators.py index 4c87d1a4919c..cb49d9a73473 100644 --- a/numpy/testing/_private/decorators.py +++ b/numpy/testing/_private/decorators.py @@ -14,6 +14,7 @@ """ import collections.abc +import warnings from .utils import SkipTest, assert_warns, HAS_REFCOUNT @@ -23,6 +24,10 @@ def slow(t): """ + .. deprecated:: 1.21 + This decorator is retained for compatibility with the nose testing framework, which is being phased out. + Please use the nose2 or pytest frameworks instead. + Label a test as 'slow'. The exact definition of a slow test is obviously both subjective and @@ -52,12 +57,19 @@ def test_big(self): print('Big, slow test') """ + # Numpy 1.21, 2020-12-20 + warnings.warn('the np.testing.dec decorators are included for nose support, and are ' + 'deprecated since NumPy v1.21. Use the nose2 or pytest frameworks instead.', DeprecationWarning, stacklevel=2) t.slow = True return t def setastest(tf=True): """ + .. deprecated:: 1.21 + This decorator is retained for compatibility with the nose testing framework, which is being phased out. + Please use the nose2 or pytest frameworks instead. + Signals to nose that this function is or is not a test. Parameters @@ -84,6 +96,9 @@ def func_with_test_in_name(arg1, arg2): pass """ + # Numpy 1.21, 2020-12-20 + warnings.warn('the np.testing.dec decorators are included for nose support, and are ' + 'deprecated since NumPy v1.21. Use the nose2 or pytest frameworks instead.', DeprecationWarning, stacklevel=2) def set_test(t): t.__test__ = tf return t @@ -91,6 +106,10 @@ def set_test(t): def skipif(skip_condition, msg=None): """ + .. deprecated:: 1.21 + This decorator is retained for compatibility with the nose testing framework, which is being phased out. + Please use the nose2 or pytest frameworks instead. + Make function raise SkipTest exception if a given condition is true. If the condition is a callable, it is used at runtime to dynamically @@ -123,6 +142,10 @@ def skip_decorator(f): # import time overhead at actual test-time. import nose + # Numpy 1.21, 2020-12-20 + warnings.warn('the np.testing.dec decorators are included for nose support, and are ' + 'deprecated since NumPy v1.21. Use the nose2 or pytest frameworks instead.', DeprecationWarning, stacklevel=2) + # Allow for both boolean or callable skip conditions. if isinstance(skip_condition, collections.abc.Callable): skip_val = lambda: skip_condition() @@ -167,6 +190,10 @@ def skipper_gen(*args, **kwargs): def knownfailureif(fail_condition, msg=None): """ + .. deprecated:: 1.21 + This decorator is retained for compatibility with the nose testing framework, which is being phased out. + Please use the nose2 or pytest frameworks instead. + Make function raise KnownFailureException exception if given condition is true. If the condition is a callable, it is used at runtime to dynamically @@ -195,6 +222,10 @@ def knownfailureif(fail_condition, msg=None): function in order to transmit function name, and various other metadata. """ + # Numpy 1.21, 2020-12-20 + warnings.warn('the np.testing.dec decorators are included for nose support, and are ' + 'deprecated since NumPy v1.21. Use the nose2 or pytest frameworks instead.', DeprecationWarning, stacklevel=2) + if msg is None: msg = 'Test skipped due to known failure' @@ -221,6 +252,10 @@ def knownfailer(*args, **kwargs): def deprecated(conditional=True): """ + .. deprecated:: 1.21 + This decorator is retained for compatibility with the nose testing framework, which is being phased out. + Please use the nose2 or pytest frameworks instead. + Filter deprecation warnings while running the test suite. This decorator can be used to filter DeprecationWarning's, to avoid @@ -249,6 +284,10 @@ def deprecate_decorator(f): # import time overhead at actual test-time. import nose + # Numpy 1.21, 2020-12-20 + warnings.warn('the np.testing.dec decorators are included for nose support, and are ' + 'deprecated since NumPy v1.21. Use the nose2 or pytest frameworks instead.', DeprecationWarning, stacklevel=2) + def _deprecated_imp(*args, **kwargs): # Poor man's replacement for the with statement with assert_warns(DeprecationWarning): @@ -267,6 +306,10 @@ def _deprecated_imp(*args, **kwargs): def parametrize(vars, input): """ + .. deprecated:: 1.21 + This decorator is retained for compatibility with the nose testing framework, which is being phased out. + Please use the nose2 or pytest frameworks instead. + Pytest compatibility class. This implements the simplest level of pytest.mark.parametrize for use in nose as an aid in making the transition to pytest. It achieves that by adding a dummy var parameter and ignoring @@ -279,6 +322,10 @@ def parametrize(vars, input): """ from .parameterized import parameterized + # Numpy 1.21, 2020-12-20 + warnings.warn('the np.testing.dec decorators are included for nose support, and are ' + 'deprecated since NumPy v1.21. Use the nose2 or pytest frameworks instead.', DeprecationWarning, stacklevel=2) + return parameterized(input) _needs_refcount = skipif(not HAS_REFCOUNT, "python has no sys.getrefcount") diff --git a/numpy/testing/_private/parameterized.py b/numpy/testing/_private/parameterized.py index ac7db6c4041f..db9629a94680 100644 --- a/numpy/testing/_private/parameterized.py +++ b/numpy/testing/_private/parameterized.py @@ -37,11 +37,6 @@ from types import MethodType from collections import namedtuple -try: - from collections import OrderedDict as MaybeOrderedDict -except ImportError: - MaybeOrderedDict = dict - from unittest import TestCase _param = namedtuple("param", "args kwargs") @@ -113,13 +108,6 @@ def __repr__(self): return "param(*%r, **%r)" %self -class QuietOrderedDict(MaybeOrderedDict): - """ When OrderedDict is available, use it to make sure that the kwargs in - doc strings are consistently ordered. """ - __str__ = dict.__str__ - __repr__ = dict.__repr__ - - def parameterized_argument_value_pairs(func, p): """Return tuples of parameterized arguments and their values. @@ -165,7 +153,7 @@ def parameterized_argument_value_pairs(func, p): ]) seen_arg_names = {n for (n, _) in result} - keywords = QuietOrderedDict(sorted([ + keywords = dict(sorted([ (name, p.kwargs[name]) for name in p.kwargs if name not in seen_arg_names @@ -339,7 +327,7 @@ def assert_not_in_testcase_subclass(self): "'@parameterized.expand' instead.") def _terrible_magic_get_defining_classes(self): - """ Returns the set of parent classes of the class currently being defined. + """ Returns the list of parent classes of the class currently being defined. Will likely only work if called from the ``parameterized`` decorator. This function is entirely @brandon_rhodes's fault, as he suggested the implementation: http://stackoverflow.com/a/8793684/71522 diff --git a/numpy/testing/_private/utils.py b/numpy/testing/_private/utils.py index fb33bdcbd7f7..393fedc2705f 100644 --- a/numpy/testing/_private/utils.py +++ b/numpy/testing/_private/utils.py @@ -17,6 +17,7 @@ from warnings import WarningMessage import pprint +import numpy as np from numpy.core import( intp, float32, empty, arange, array_repr, ndarray, isnat, array) import numpy.linalg.lapack_lite @@ -378,7 +379,8 @@ def assert_equal(actual, desired, err_msg='', verbose=True): try: isdesnat = isnat(desired) isactnat = isnat(actual) - dtypes_match = array(desired).dtype.type == array(actual).dtype.type + dtypes_match = (np.asarray(desired).dtype.type == + np.asarray(actual).dtype.type) if isdesnat and isactnat: # If both are NaT (and have the same dtype -- datetime or # timedelta) they are considered equal. @@ -398,8 +400,8 @@ def assert_equal(actual, desired, err_msg='', verbose=True): return # both nan, so equal # handle signed zero specially for floats - array_actual = array(actual) - array_desired = array(desired) + array_actual = np.asarray(actual) + array_desired = np.asarray(desired) if (array_actual.dtype.char in 'Mm' or array_desired.dtype.char in 'Mm'): # version 1.18 @@ -481,7 +483,7 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True): instead of this function for more consistent floating point comparisons. - The test verifies that the elements of ``actual`` and ``desired`` satisfy. + The test verifies that the elements of `actual` and `desired` satisfy. ``abs(desired-actual) < 1.5 * 10**(-decimal)`` @@ -516,9 +518,9 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True): Examples -------- - >>> import numpy.testing as npt - >>> npt.assert_almost_equal(2.3333333333333, 2.33333334) - >>> npt.assert_almost_equal(2.3333333333333, 2.33333334, decimal=10) + >>> from numpy.testing import assert_almost_equal + >>> assert_almost_equal(2.3333333333333, 2.33333334) + >>> assert_almost_equal(2.3333333333333, 2.33333334, decimal=10) Traceback (most recent call last): ... AssertionError: @@ -526,8 +528,8 @@ def assert_almost_equal(actual,desired,decimal=7,err_msg='',verbose=True): ACTUAL: 2.3333333333333 DESIRED: 2.33333334 - >>> npt.assert_almost_equal(np.array([1.0,2.3333333333333]), - ... np.array([1.0,2.33333334]), decimal=9) + >>> assert_almost_equal(np.array([1.0,2.3333333333333]), + ... np.array([1.0,2.33333334]), decimal=9) Traceback (most recent call last): ... AssertionError: @@ -701,8 +703,8 @@ def assert_array_compare(comparison, x, y, err_msg='', verbose=True, header='', __tracebackhide__ = True # Hide traceback for py.test from numpy.core import array, array2string, isnan, inf, bool_, errstate, all, max, object_ - x = array(x, copy=False, subok=True) - y = array(y, copy=False, subok=True) + x = np.asanyarray(x) + y = np.asanyarray(y) # original array for output formatting ox, oy = x, y @@ -745,7 +747,7 @@ def func_assert_same_pos(x, y, func=isnan, hasval='nan'): # flag as it everywhere, so we should return the scalar flag. if isinstance(x_id, bool) or x_id.ndim == 0: return bool_(x_id) - elif isinstance(x_id, bool) or y_id.ndim == 0: + elif isinstance(y_id, bool) or y_id.ndim == 0: return bool_(y_id) else: return y_id @@ -1033,7 +1035,7 @@ def compare(x, y): # make sure y is an inexact type to avoid abs(MIN_INT); will cause # casting of x later. dtype = result_type(y, 1.) - y = array(y, dtype=dtype, copy=False, subok=True) + y = np.asanyarray(y, dtype) z = abs(x - y) if not issubdtype(z.dtype, number): @@ -1678,11 +1680,11 @@ def nulp_diff(x, y, dtype=None): """ import numpy as np if dtype: - x = np.array(x, dtype=dtype) - y = np.array(y, dtype=dtype) + x = np.asarray(x, dtype=dtype) + y = np.asarray(y, dtype=dtype) else: - x = np.array(x) - y = np.array(y) + x = np.asarray(x) + y = np.asarray(y) t = np.common_type(x, y) if np.iscomplexobj(x) or np.iscomplexobj(y): @@ -1699,7 +1701,7 @@ def nulp_diff(x, y, dtype=None): (x.shape, y.shape)) def _diff(rx, ry, vdt): - diff = np.array(rx-ry, dtype=vdt) + diff = np.asarray(rx-ry, dtype=vdt) return np.abs(diff) rx = integer_repr(x) @@ -2006,7 +2008,7 @@ class clear_and_catch_warnings(warnings.catch_warnings): def __init__(self, record=False, modules=()): self.modules = set(modules).union(self.class_modules) self._warnreg_copies = {} - super(clear_and_catch_warnings, self).__init__(record=record) + super().__init__(record=record) def __enter__(self): for mod in self.modules: @@ -2014,10 +2016,10 @@ def __enter__(self): mod_reg = mod.__warningregistry__ self._warnreg_copies[mod] = mod_reg.copy() mod_reg.clear() - return super(clear_and_catch_warnings, self).__enter__() + return super().__enter__() def __exit__(self, *exc_info): - super(clear_and_catch_warnings, self).__exit__(*exc_info) + super().__exit__(*exc_info) for mod in self.modules: if hasattr(mod, '__warningregistry__'): mod.__warningregistry__.clear() diff --git a/numpy/testing/tests/test_decorators.py b/numpy/testing/tests/test_decorators.py deleted file mode 100644 index b60d6dfbce90..000000000000 --- a/numpy/testing/tests/test_decorators.py +++ /dev/null @@ -1,210 +0,0 @@ -""" -Test the decorators from ``testing.decorators``. - -""" -import warnings -import pytest - -from numpy.testing import ( - assert_, assert_raises, dec, SkipTest, KnownFailureException, - ) - - -try: - with warnings.catch_warnings(): - warnings.simplefilter("always") - import nose # noqa: F401 -except ImportError: - HAVE_NOSE = False -else: - HAVE_NOSE = True - - -@pytest.mark.skipif(not HAVE_NOSE, reason="Needs nose") -class TestNoseDecorators: - # These tests are run in a class for simplicity while still - # getting a report on each, skipped or success. - - class DidntSkipException(Exception): - pass - - def test_slow(self): - @dec.slow - def slow_func(x, y, z): - pass - - assert_(slow_func.slow) - - def test_setastest(self): - @dec.setastest() - def f_default(a): - pass - - @dec.setastest(True) - def f_istest(a): - pass - - @dec.setastest(False) - def f_isnottest(a): - pass - - assert_(f_default.__test__) - assert_(f_istest.__test__) - assert_(not f_isnottest.__test__) - - def test_skip_functions_hardcoded(self): - @dec.skipif(True) - def f1(x): - raise self.DidntSkipException - - try: - f1('a') - except self.DidntSkipException: - raise Exception('Failed to skip') - except SkipTest().__class__: - pass - - @dec.skipif(False) - def f2(x): - raise self.DidntSkipException - - try: - f2('a') - except self.DidntSkipException: - pass - except SkipTest().__class__: - raise Exception('Skipped when not expected to') - - def test_skip_functions_callable(self): - def skip_tester(): - return skip_flag == 'skip me!' - - @dec.skipif(skip_tester) - def f1(x): - raise self.DidntSkipException - - try: - skip_flag = 'skip me!' - f1('a') - except self.DidntSkipException: - raise Exception('Failed to skip') - except SkipTest().__class__: - pass - - @dec.skipif(skip_tester) - def f2(x): - raise self.DidntSkipException - - try: - skip_flag = 'five is right out!' - f2('a') - except self.DidntSkipException: - pass - except SkipTest().__class__: - raise Exception('Skipped when not expected to') - - def test_skip_generators_hardcoded(self): - @dec.knownfailureif(True, "This test is known to fail") - def g1(x): - yield from range(x) - - try: - for j in g1(10): - pass - except KnownFailureException().__class__: - pass - else: - raise Exception('Failed to mark as known failure') - - @dec.knownfailureif(False, "This test is NOT known to fail") - def g2(x): - yield from range(x) - raise self.DidntSkipException('FAIL') - - try: - for j in g2(10): - pass - except KnownFailureException().__class__: - raise Exception('Marked incorrectly as known failure') - except self.DidntSkipException: - pass - - def test_skip_generators_callable(self): - def skip_tester(): - return skip_flag == 'skip me!' - - @dec.knownfailureif(skip_tester, "This test is known to fail") - def g1(x): - yield from range(x) - - try: - skip_flag = 'skip me!' - for j in g1(10): - pass - except KnownFailureException().__class__: - pass - else: - raise Exception('Failed to mark as known failure') - - @dec.knownfailureif(skip_tester, "This test is NOT known to fail") - def g2(x): - yield from range(x) - raise self.DidntSkipException('FAIL') - - try: - skip_flag = 'do not skip' - for j in g2(10): - pass - except KnownFailureException().__class__: - raise Exception('Marked incorrectly as known failure') - except self.DidntSkipException: - pass - - def test_deprecated(self): - @dec.deprecated(True) - def non_deprecated_func(): - pass - - @dec.deprecated() - def deprecated_func(): - import warnings - warnings.warn("TEST: deprecated func", DeprecationWarning) - - @dec.deprecated() - def deprecated_func2(): - import warnings - warnings.warn("AHHHH") - raise ValueError - - @dec.deprecated() - def deprecated_func3(): - import warnings - warnings.warn("AHHHH") - - # marked as deprecated, but does not raise DeprecationWarning - assert_raises(AssertionError, non_deprecated_func) - # should be silent - deprecated_func() - with warnings.catch_warnings(record=True): - warnings.simplefilter("always") # do not propagate unrelated warnings - # fails if deprecated decorator just disables test. See #1453. - assert_raises(ValueError, deprecated_func2) - # warning is not a DeprecationWarning - assert_raises(AssertionError, deprecated_func3) - - def test_parametrize(self): - # dec.parametrize assumes that it is being run by nose. Because - # we are running under pytest, we need to explicitly check the - # results. - @dec.parametrize('base, power, expected', - [(1, 1, 1), - (2, 1, 2), - (2, 2, 4)]) - def check_parametrize(base, power, expected): - assert_(base**power == expected) - - count = 0 - for test in check_parametrize(): - test[0](*test[1:]) - count += 1 - assert_(count == 3) diff --git a/numpy/testing/tests/test_utils.py b/numpy/testing/tests/test_utils.py index c3b9e04b693f..31d2cdc76b3e 100644 --- a/numpy/testing/tests/test_utils.py +++ b/numpy/testing/tests/test_utils.py @@ -434,10 +434,10 @@ def test_subclass_that_cannot_be_bool(self): # (which, e.g., astropy Quantity cannot usefully do). See gh-8452. class MyArray(np.ndarray): def __eq__(self, other): - return super(MyArray, self).__eq__(other).view(np.ndarray) + return super().__eq__(other).view(np.ndarray) def __lt__(self, other): - return super(MyArray, self).__lt__(other).view(np.ndarray) + return super().__lt__(other).view(np.ndarray) def all(self, *args, **kwargs): raise NotImplementedError @@ -585,10 +585,10 @@ def test_subclass_that_cannot_be_bool(self): # (which, e.g., astropy Quantity cannot usefully do). See gh-8452. class MyArray(np.ndarray): def __eq__(self, other): - return super(MyArray, self).__eq__(other).view(np.ndarray) + return super().__eq__(other).view(np.ndarray) def __lt__(self, other): - return super(MyArray, self).__lt__(other).view(np.ndarray) + return super().__lt__(other).view(np.ndarray) def all(self, *args, **kwargs): raise NotImplementedError @@ -904,6 +904,11 @@ def test_report_max_relative_error(self): msg = str(exc_info.value) assert_('Max relative difference: 0.5' in msg) + def test_timedelta(self): + # see gh-18286 + a = np.array([[1, 2, 3, "NaT"]], dtype="m8[ns]") + assert_allclose(a, a) + class TestArrayAlmostEqualNulp: diff --git a/numpy/tests/test_numpy_version.py b/numpy/tests/test_numpy_version.py index 916ab93830df..bccbcb8e9cf7 100644 --- a/numpy/tests/test_numpy_version.py +++ b/numpy/tests/test_numpy_version.py @@ -1,3 +1,20 @@ +""" +Check the numpy version is valid. + +Note that a development version is marked by the presence of 'dev0' or '+' +in the version string, all else is treated as a release. The version string +itself is set from the output of ``git describe`` which relies on tags. + +Examples +-------- + +Valid Development: 1.22.0.dev0 1.22.0.dev0+5-g7999db4df2 1.22.0+5-g7999db4df2 +Valid Release: 1.21.0.rc1, 1.21.0.b1, 1.21.0 +Invalid: 1.22.0.dev, 1.22.0.dev0-5-g7999db4dfB, 1.21.0.d1, 1.21.a + +Note that a release is determined by the version string, which in turn +is controlled by the result of the ``git describe`` command. +""" import re import numpy as np @@ -7,11 +24,21 @@ def test_valid_numpy_version(): # Verify that the numpy version is a valid one (no .post suffix or other # nonsense). See gh-6431 for an issue caused by an invalid version. - version_pattern = r"^[0-9]+\.[0-9]+\.[0-9]+(|a[0-9]|b[0-9]|rc[0-9])" - dev_suffix = r"(\.dev0\+([0-9a-f]{7}|Unknown))" + version_pattern = r"^[0-9]+\.[0-9]+\.[0-9]+(a[0-9]|b[0-9]|rc[0-9]|)" + dev_suffix = r"(\.dev0|)(\+[0-9]*\.g[0-9a-f]+|)" if np.version.release: - res = re.match(version_pattern, np.__version__) + res = re.match(version_pattern + '$', np.__version__) else: - res = re.match(version_pattern + dev_suffix, np.__version__) + res = re.match(version_pattern + dev_suffix + '$', np.__version__) assert_(res is not None, np.__version__) + + +def test_short_version(): + # Check numpy.short_version actually exists + if np.version.release: + assert_(np.__version__ == np.version.short_version, + "short_version mismatch in release version") + else: + assert_(np.__version__.split("+")[0] == np.version.short_version, + "short_version mismatch in development version") diff --git a/numpy/tests/test_public_api.py b/numpy/tests/test_public_api.py index a7bd0f115d15..6e4a8dee0a7c 100644 --- a/numpy/tests/test_public_api.py +++ b/numpy/tests/test_public_api.py @@ -40,7 +40,7 @@ def test_numpy_namespace(): 'byte_bounds': 'numpy.lib.utils.byte_bounds', 'compare_chararrays': 'numpy.core._multiarray_umath.compare_chararrays', 'deprecate': 'numpy.lib.utils.deprecate', - 'deprecate_with_doc': 'numpy.lib.utils.', + 'deprecate_with_doc': 'numpy.lib.utils.deprecate_with_doc', 'disp': 'numpy.lib.function_base.disp', 'fastCopyAndTranspose': 'numpy.core._multiarray_umath._fastCopyAndTranspose', 'get_array_wrap': 'numpy.lib.shape_base.get_array_wrap', @@ -154,6 +154,7 @@ def test_NPY_NO_EXPORT(): "lib.mixins", "lib.recfunctions", "lib.scimath", + "lib.stride_tricks", "linalg", "ma", "ma.extras", @@ -166,10 +167,10 @@ def test_NPY_NO_EXPORT(): "polynomial.laguerre", "polynomial.legendre", "polynomial.polynomial", - "polynomial.polyutils", "random", "testing", "typing", + "typing.mypy_plugin", "version", ]] @@ -243,6 +244,7 @@ def test_NPY_NO_EXPORT(): "distutils.fcompiler.nv", "distutils.fcompiler.sun", "distutils.fcompiler.vast", + "distutils.fcompiler.fujitsu", "distutils.from_template", "distutils.intelccompiler", "distutils.lib2def", @@ -278,7 +280,6 @@ def test_NPY_NO_EXPORT(): "lib.npyio", "lib.polynomial", "lib.shape_base", - "lib.stride_tricks", "lib.twodim_base", "lib.type_check", "lib.ufunclike", @@ -292,6 +293,7 @@ def test_NPY_NO_EXPORT(): "ma.timer_comparison", "matrixlib", "matrixlib.defmatrix", + "polynomial.polyutils", "random.mtrand", "random.bit_generator", "testing.print_coercion_tables", diff --git a/numpy/tests/test_reloading.py b/numpy/tests/test_reloading.py index 61ae91b00d55..5c4309f4ab88 100644 --- a/numpy/tests/test_reloading.py +++ b/numpy/tests/test_reloading.py @@ -1,4 +1,4 @@ -from numpy.testing import assert_raises, assert_, assert_equal +from numpy.testing import assert_raises, assert_warns, assert_, assert_equal from numpy.compat import pickle import sys @@ -16,13 +16,15 @@ def test_numpy_reloading(): VisibleDeprecationWarning = np.VisibleDeprecationWarning ModuleDeprecationWarning = np.ModuleDeprecationWarning - reload(np) + with assert_warns(UserWarning): + reload(np) assert_(_NoValue is np._NoValue) assert_(ModuleDeprecationWarning is np.ModuleDeprecationWarning) assert_(VisibleDeprecationWarning is np.VisibleDeprecationWarning) assert_raises(RuntimeError, reload, numpy._globals) - reload(np) + with assert_warns(UserWarning): + reload(np) assert_(_NoValue is np._NoValue) assert_(ModuleDeprecationWarning is np.ModuleDeprecationWarning) assert_(VisibleDeprecationWarning is np.VisibleDeprecationWarning) @@ -45,13 +47,15 @@ def test_full_reimport(): # This is generally unsafe, especially, since we also reload the C-modules. code = textwrap.dedent(r""" import sys + from pytest import warns import numpy as np for k in list(sys.modules.keys()): if "numpy" in k: del sys.modules[k] - import numpy as np + with warns(UserWarning): + import numpy as np """) p = subprocess.run([sys.executable, '-c', code]) - assert p.returncode == 0 + diff --git a/numpy/tests/test_scripts.py b/numpy/tests/test_scripts.py index a0f2ba70a4a1..e67a829471dc 100644 --- a/numpy/tests/test_scripts.py +++ b/numpy/tests/test_scripts.py @@ -38,9 +38,9 @@ def find_f2py_commands(): def test_f2py(f2py_cmd): # test that we can run f2py script stdout = subprocess.check_output([f2py_cmd, '-v']) - assert_equal(stdout.strip(), b'2') + assert_equal(stdout.strip(), np.__version__.encode('ascii')) def test_pep338(): stdout = subprocess.check_output([sys.executable, '-mnumpy.f2py', '-v']) - assert_equal(stdout.strip(), b'2') + assert_equal(stdout.strip(), np.__version__.encode('ascii')) diff --git a/numpy/typing/__init__.py b/numpy/typing/__init__.py index e72e8fb4d60d..1bfdf07ae74e 100644 --- a/numpy/typing/__init__.py +++ b/numpy/typing/__init__.py @@ -19,6 +19,33 @@ .. _typing-extensions: https://pypi.org/project/typing-extensions/ +Mypy plugin +----------- + +A mypy_ plugin is distributed in `numpy.typing` for managing a number of +platform-specific annotations. Its function can be split into to parts: + +* Assigning the (platform-dependent) precisions of certain `~numpy.number` subclasses, + including the likes of `~numpy.int_`, `~numpy.intp` and `~numpy.longlong`. + See the documentation on :ref:`scalar types ` for a + comprehensive overview of the affected classes. without the plugin the precision + of all relevant classes will be inferred as `~typing.Any`. +* Removing all extended-precision `~numpy.number` subclasses that are unavailable + for the platform in question. Most notable this includes the likes of + `~numpy.float128` and `~numpy.complex256`. Without the plugin *all* + extended-precision types will, as far as mypy is concerned, be available + to all platforms. + +To enable the plugin, one must add it to their mypy `configuration file`_: + +.. code-block:: ini + + [mypy] + plugins = numpy.typing.mypy_plugin + +.. _mypy: http://mypy-lang.org/ +.. _configuration file: https://mypy.readthedocs.io/en/stable/config_file.html + Differences from the runtime NumPy API -------------------------------------- @@ -69,7 +96,7 @@ >>> x.dtype = np.bool_ This sort of mutation is not allowed by the types. Users who want to -write statically typed code should insted use the `numpy.ndarray.view` +write statically typed code should instead use the `numpy.ndarray.view` method to create a view of the array with a different dtype. DTypeLike @@ -82,15 +109,15 @@ >>> x = np.dtype({"field1": (float, 1), "field2": (int, 3)}) -Although this is valid Numpy code, the type checker will complain about it, +Although this is valid NumPy code, the type checker will complain about it, since its usage is discouraged. Please see : :ref:`Data type objects ` -Number Precision +Number precision ~~~~~~~~~~~~~~~~ The precision of `numpy.number` subclasses is treated as a covariant generic -parameter (see :class:`~NBitBase`), simplifying the annoting of proccesses +parameter (see :class:`~NBitBase`), simplifying the annotating of processes involving precision-based casting. .. code-block:: python @@ -113,6 +140,20 @@ The `~numpy.timedelta64` class is not considered a subclass of `~numpy.signedinteger`, the former only inheriting from `~numpy.generic` while static type checking. +0D arrays +~~~~~~~~~ + +During runtime numpy aggressively casts any passed 0D arrays into their +corresponding `~numpy.generic` instance. Until the introduction of shape +typing (see :pep:`646`) it is unfortunately not possible to make the +necessary distinction between 0D and >0D arrays. While thus not strictly +correct, all operations are that can potentially perform a 0D-array -> scalar +cast are currently annotated as exclusively returning an `ndarray`. + +If it is known in advance that an operation _will_ perform a +0D-array -> scalar cast, then one can consider manually remedying the +situation with either `typing.cast` or a ``# type: ignore`` comment. + API --- @@ -132,7 +173,7 @@ def final(f): return f if not TYPE_CHECKING: - __all__ = ["ArrayLike", "DTypeLike", "NBitBase"] + __all__ = ["ArrayLike", "DTypeLike", "NBitBase", "NDArray"] else: # Ensure that all objects within this module are accessible while # static type checking. This includes private ones, as we need them @@ -149,7 +190,7 @@ class NBitBase: An object representing `numpy.number` precision during static type checking. Used exclusively for the purpose static type checking, `NBitBase` - represents the base of a hierachieral set of subclasses. + represents the base of a hierarchical set of subclasses. Each subsequent subclass is herein used for representing a lower level of precision, *e.g.* ``64Bit > 32Bit > 16Bit``. @@ -162,13 +203,15 @@ class NBitBase: .. code-block:: python - >>> from typing import TypeVar, TYPE_CHECKING + >>> from __future__ import annotations + >>> from typing import TypeVar, Union, TYPE_CHECKING >>> import numpy as np >>> import numpy.typing as npt - >>> T = TypeVar("T", bound=npt.NBitBase) + >>> T1 = TypeVar("T1", bound=npt.NBitBase) + >>> T2 = TypeVar("T2", bound=npt.NBitBase) - >>> def add(a: "np.floating[T]", b: "np.integer[T]") -> "np.floating[T]": + >>> def add(a: np.floating[T1], b: np.integer[T2]) -> np.floating[Union[T1, T2]]: ... return a + b >>> a = np.float16() @@ -204,22 +247,131 @@ class _32Bit(_64Bit): ... # type: ignore[misc] class _16Bit(_32Bit): ... # type: ignore[misc] class _8Bit(_16Bit): ... # type: ignore[misc] -# Clean up the namespace -del TYPE_CHECKING, final, List - +from ._nbit import ( + _NBitByte, + _NBitShort, + _NBitIntC, + _NBitIntP, + _NBitInt, + _NBitLongLong, + _NBitHalf, + _NBitSingle, + _NBitDouble, + _NBitLongDouble, +) +from ._char_codes import ( + _BoolCodes, + _UInt8Codes, + _UInt16Codes, + _UInt32Codes, + _UInt64Codes, + _Int8Codes, + _Int16Codes, + _Int32Codes, + _Int64Codes, + _Float16Codes, + _Float32Codes, + _Float64Codes, + _Complex64Codes, + _Complex128Codes, + _ByteCodes, + _ShortCodes, + _IntCCodes, + _IntPCodes, + _IntCodes, + _LongLongCodes, + _UByteCodes, + _UShortCodes, + _UIntCCodes, + _UIntPCodes, + _UIntCodes, + _ULongLongCodes, + _HalfCodes, + _SingleCodes, + _DoubleCodes, + _LongDoubleCodes, + _CSingleCodes, + _CDoubleCodes, + _CLongDoubleCodes, + _DT64Codes, + _TD64Codes, + _StrCodes, + _BytesCodes, + _VoidCodes, + _ObjectCodes, +) from ._scalars import ( - _CharLike, - _BoolLike, - _IntLike, - _FloatLike, - _ComplexLike, - _NumberLike, - _ScalarLike, - _VoidLike, + _CharLike_co, + _BoolLike_co, + _UIntLike_co, + _IntLike_co, + _FloatLike_co, + _ComplexLike_co, + _TD64Like_co, + _NumberLike_co, + _ScalarLike_co, + _VoidLike_co, ) -from ._array_like import _SupportsArray, ArrayLike from ._shape import _Shape, _ShapeLike -from ._dtype_like import _SupportsDType, _VoidDTypeLike, DTypeLike +from ._dtype_like import ( + DTypeLike as DTypeLike, + _SupportsDType, + _VoidDTypeLike, + _DTypeLikeBool, + _DTypeLikeUInt, + _DTypeLikeInt, + _DTypeLikeFloat, + _DTypeLikeComplex, + _DTypeLikeTD64, + _DTypeLikeDT64, + _DTypeLikeObject, + _DTypeLikeVoid, + _DTypeLikeStr, + _DTypeLikeBytes, + _DTypeLikeComplex_co, +) +from ._array_like import ( + ArrayLike as ArrayLike, + _ArrayLike, + _NestedSequence, + _RecursiveSequence, + _SupportsArray, + _ArrayLikeInt, + _ArrayLikeBool_co, + _ArrayLikeUInt_co, + _ArrayLikeInt_co, + _ArrayLikeFloat_co, + _ArrayLikeComplex_co, + _ArrayLikeNumber_co, + _ArrayLikeTD64_co, + _ArrayLikeDT64_co, + _ArrayLikeObject_co, + _ArrayLikeVoid_co, + _ArrayLikeStr_co, + _ArrayLikeBytes_co, +) +from ._generic_alias import ( + NDArray as NDArray, + _GenericAlias, +) + +if TYPE_CHECKING: + from ._ufunc import ( + _UFunc_Nin1_Nout1, + _UFunc_Nin2_Nout1, + _UFunc_Nin1_Nout2, + _UFunc_Nin2_Nout2, + _GUFunc_Nin2_Nout1, + ) +else: + _UFunc_Nin1_Nout1 = NotImplemented + _UFunc_Nin2_Nout1 = NotImplemented + _UFunc_Nin1_Nout2 = NotImplemented + _UFunc_Nin2_Nout2 = NotImplemented + _GUFunc_Nin2_Nout1 = NotImplemented + +# Clean up the namespace +del TYPE_CHECKING, final, List if __doc__ is not None: from ._add_docstring import _docstrings diff --git a/numpy/typing/_add_docstring.py b/numpy/typing/_add_docstring.py index 8e39fe2c6175..56ef41cfdb3c 100644 --- a/numpy/typing/_add_docstring.py +++ b/numpy/typing/_add_docstring.py @@ -3,14 +3,32 @@ import re import textwrap +from ._generic_alias import NDArray + _docstrings_list = [] -def add_newdoc(name, value, doc): +def add_newdoc(name: str, value: str, doc: str) -> None: + """Append ``_docstrings_list`` with a docstring for `name`. + + Parameters + ---------- + name : str + The name of the object. + value : str + A string-representation of the object. + doc : str + The docstring of the object. + + """ _docstrings_list.append((name, value, doc)) -def _parse_docstrings(): +def _parse_docstrings() -> str: + """Convert all docstrings in ``_docstrings_list`` into a single + sphinx-legible text block. + + """ type_list_ret = [] for name, value, doc in _docstrings_list: s = textwrap.dedent(doc).replace("\n", "\n ") @@ -93,4 +111,33 @@ def _parse_docstrings(): """) +add_newdoc('NDArray', repr(NDArray), + """ + A :term:`generic ` version of + `np.ndarray[Any, np.dtype[+ScalarType]] `. + + Can be used during runtime for typing arrays with a given dtype + and unspecified shape. + + Examples + -------- + .. code-block:: python + + >>> import numpy as np + >>> import numpy.typing as npt + + >>> print(npt.NDArray) + numpy.ndarray[typing.Any, numpy.dtype[+ScalarType]] + + >>> print(npt.NDArray[np.float64]) + numpy.ndarray[typing.Any, numpy.dtype[numpy.float64]] + + >>> NDArrayInt = npt.NDArray[np.int_] + >>> a: NDArrayInt = np.arange(10) + + >>> def func(a: npt.ArrayLike) -> npt.NDArray[Any]: + ... return np.array(a) + + """) + _docstrings = _parse_docstrings() diff --git a/numpy/typing/_array_like.py b/numpy/typing/_array_like.py index a1a604239390..9f57b22956cc 100644 --- a/numpy/typing/_array_like.py +++ b/numpy/typing/_array_like.py @@ -1,8 +1,25 @@ +from __future__ import annotations + import sys -from typing import Any, overload, Sequence, TYPE_CHECKING, Union +from typing import Any, overload, Sequence, TYPE_CHECKING, Union, TypeVar -from numpy import ndarray -from ._scalars import _ScalarLike +from numpy import ( + ndarray, + dtype, + generic, + bool_, + unsignedinteger, + integer, + floating, + complexfloating, + number, + timedelta64, + datetime64, + object_, + void, + str_, + bytes_, +) from ._dtype_like import DTypeLike if sys.version_info >= (3, 8): @@ -16,15 +33,40 @@ else: HAVE_PROTOCOL = True +_T = TypeVar("_T") +_ScalarType = TypeVar("_ScalarType", bound=generic) +_DType = TypeVar("_DType", bound="dtype[Any]") +_DType_co = TypeVar("_DType_co", covariant=True, bound="dtype[Any]") + if TYPE_CHECKING or HAVE_PROTOCOL: - class _SupportsArray(Protocol): - @overload - def __array__(self, __dtype: DTypeLike = ...) -> ndarray: ... - @overload - def __array__(self, dtype: DTypeLike = ...) -> ndarray: ... + # The `_SupportsArray` protocol only cares about the default dtype + # (i.e. `dtype=None` or no `dtype` parameter at all) of the to-be returned + # array. + # Concrete implementations of the protocol are responsible for adding + # any and all remaining overloads + class _SupportsArray(Protocol[_DType_co]): + def __array__(self) -> ndarray[Any, _DType_co]: ... else: _SupportsArray = Any +# TODO: Wait for support for recursive types +_NestedSequence = Union[ + _T, + Sequence[_T], + Sequence[Sequence[_T]], + Sequence[Sequence[Sequence[_T]]], + Sequence[Sequence[Sequence[Sequence[_T]]]], +] +_RecursiveSequence = Sequence[Sequence[Sequence[Sequence[Sequence[Any]]]]] + +# A union representing array-like objects; consists of two typevars: +# One representing types that can be parametrized w.r.t. `np.dtype` +# and another one for the rest +_ArrayLike = Union[ + _NestedSequence[_SupportsArray[_DType]], + _NestedSequence[_T], +] + # TODO: support buffer protocols once # # https://bugs.python.org/issue27501 @@ -33,8 +75,57 @@ def __array__(self, dtype: DTypeLike = ...) -> ndarray: ... # # https://github.com/python/typing/issues/593 ArrayLike = Union[ - _ScalarLike, - Sequence[_ScalarLike], - Sequence[Sequence[Any]], # TODO: Wait for support for recursive types - _SupportsArray, + _RecursiveSequence, + _ArrayLike[ + "dtype[Any]", + Union[bool, int, float, complex, str, bytes] + ], +] + +# `ArrayLike_co`: array-like objects that can be coerced into `X` +# given the casting rules `same_kind` +_ArrayLikeBool_co = _ArrayLike[ + "dtype[bool_]", + bool, +] +_ArrayLikeUInt_co = _ArrayLike[ + "dtype[Union[bool_, unsignedinteger[Any]]]", + bool, +] +_ArrayLikeInt_co = _ArrayLike[ + "dtype[Union[bool_, integer[Any]]]", + Union[bool, int], +] +_ArrayLikeFloat_co = _ArrayLike[ + "dtype[Union[bool_, integer[Any], floating[Any]]]", + Union[bool, int, float], +] +_ArrayLikeComplex_co = _ArrayLike[ + "dtype[Union[bool_, integer[Any], floating[Any], complexfloating[Any, Any]]]", + Union[bool, int, float, complex], +] +_ArrayLikeNumber_co = _ArrayLike[ + "dtype[Union[bool_, number[Any]]]", + Union[bool, int, float, complex], +] +_ArrayLikeTD64_co = _ArrayLike[ + "dtype[Union[bool_, integer[Any], timedelta64]]", + Union[bool, int], +] +_ArrayLikeDT64_co = _NestedSequence[_SupportsArray["dtype[datetime64]"]] +_ArrayLikeObject_co = _NestedSequence[_SupportsArray["dtype[object_]"]] + +_ArrayLikeVoid_co = _NestedSequence[_SupportsArray["dtype[void]"]] +_ArrayLikeStr_co = _ArrayLike[ + "dtype[str_]", + str, +] +_ArrayLikeBytes_co = _ArrayLike[ + "dtype[bytes_]", + bytes, +] + +_ArrayLikeInt = _ArrayLike[ + "dtype[integer[Any]]", + int, ] diff --git a/numpy/typing/_callable.py b/numpy/typing/_callable.py index 91b7a4ec21c0..d9cb0f1571a5 100644 --- a/numpy/typing/_callable.py +++ b/numpy/typing/_callable.py @@ -8,6 +8,8 @@ """ +from __future__ import annotations + import sys from typing import ( Union, @@ -20,6 +22,8 @@ ) from numpy import ( + ndarray, + dtype, generic, bool_, timedelta64, @@ -28,19 +32,23 @@ unsignedinteger, signedinteger, int8, + int_, floating, float64, complexfloating, complex128, ) +from ._nbit import _NBitInt, _NBitDouble from ._scalars import ( - _BoolLike, - _IntLike, - _FloatLike, - _ComplexLike, - _NumberLike, + _BoolLike_co, + _IntLike_co, + _FloatLike_co, + _ComplexLike_co, + _NumberLike_co, ) from . import NBitBase +from ._array_like import ArrayLike +from ._generic_alias import NDArray if sys.version_info >= (3, 8): from typing import Protocol @@ -54,11 +62,12 @@ HAVE_PROTOCOL = True if TYPE_CHECKING or HAVE_PROTOCOL: - _T = TypeVar("_T") - _2Tuple = Tuple[_T, _T] + _T1 = TypeVar("_T1") + _T2 = TypeVar("_T2") + _2Tuple = Tuple[_T1, _T1] - _NBit_co = TypeVar("_NBit_co", covariant=True, bound=NBitBase) - _NBit = TypeVar("_NBit", bound=NBitBase) + _NBit1 = TypeVar("_NBit1", bound=NBitBase) + _NBit2 = TypeVar("_NBit2", bound=NBitBase) _IntType = TypeVar("_IntType", bound=integer) _FloatType = TypeVar("_FloatType", bound=floating) @@ -68,9 +77,9 @@ class _BoolOp(Protocol[_GenericType_co]): @overload - def __call__(self, __other: _BoolLike) -> _GenericType_co: ... + def __call__(self, __other: _BoolLike_co) -> _GenericType_co: ... @overload # platform dependent - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> int_: ... @overload def __call__(self, __other: float) -> float64: ... @overload @@ -80,9 +89,9 @@ def __call__(self, __other: _NumberType) -> _NumberType: ... class _BoolBitOp(Protocol[_GenericType_co]): @overload - def __call__(self, __other: _BoolLike) -> _GenericType_co: ... + def __call__(self, __other: _BoolLike_co) -> _GenericType_co: ... @overload # platform dependent - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> int_: ... @overload def __call__(self, __other: _IntType) -> _IntType: ... @@ -91,7 +100,7 @@ class _BoolSub(Protocol): @overload def __call__(self, __other: bool) -> NoReturn: ... @overload # platform dependent - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> int_: ... @overload def __call__(self, __other: float) -> float64: ... @overload @@ -101,7 +110,7 @@ def __call__(self, __other: _NumberType) -> _NumberType: ... class _BoolTrueDiv(Protocol): @overload - def __call__(self, __other: Union[float, _IntLike, _BoolLike]) -> float64: ... + def __call__(self, __other: Union[float, _IntLike_co]) -> float64: ... @overload def __call__(self, __other: complex) -> complex128: ... @overload @@ -109,9 +118,9 @@ def __call__(self, __other: _NumberType) -> _NumberType: ... class _BoolMod(Protocol): @overload - def __call__(self, __other: _BoolLike) -> int8: ... + def __call__(self, __other: _BoolLike_co) -> int8: ... @overload # platform dependent - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> int_: ... @overload def __call__(self, __other: float) -> float64: ... @overload @@ -121,11 +130,11 @@ def __call__(self, __other: _FloatType) -> _FloatType: ... class _BoolDivMod(Protocol): @overload - def __call__(self, __other: _BoolLike) -> _2Tuple[int8]: ... + def __call__(self, __other: _BoolLike_co) -> _2Tuple[int8]: ... @overload # platform dependent - def __call__(self, __other: int) -> _2Tuple[signedinteger[Any]]: ... + def __call__(self, __other: int) -> _2Tuple[int_]: ... @overload - def __call__(self, __other: float) -> _2Tuple[float64]: ... + def __call__(self, __other: float) -> _2Tuple[floating[Union[_NBit1, _NBitDouble]]]: ... @overload def __call__(self, __other: _IntType) -> _2Tuple[_IntType]: ... @overload @@ -135,182 +144,200 @@ class _TD64Div(Protocol[_NumberType_co]): @overload def __call__(self, __other: timedelta64) -> _NumberType_co: ... @overload - def __call__(self, __other: _FloatLike) -> timedelta64: ... + def __call__(self, __other: _BoolLike_co) -> NoReturn: ... + @overload + def __call__(self, __other: _FloatLike_co) -> timedelta64: ... - class _IntTrueDiv(Protocol[_NBit_co]): + class _IntTrueDiv(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> floating[_NBit_co]: ... + def __call__(self, __other: bool) -> floating[_NBit1]: ... @overload - def __call__(self, __other: int) -> floating[Any]: ... + def __call__(self, __other: int) -> floating[Union[_NBit1, _NBitInt]]: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload - def __call__(self, __other: complex) -> complex128: ... + def __call__( + self, __other: complex + ) -> complexfloating[Union[_NBit1, _NBitDouble], Union[_NBit1, _NBitDouble]]: ... @overload - def __call__(self, __other: integer[_NBit]) -> floating[Union[_NBit_co, _NBit]]: ... + def __call__(self, __other: integer[_NBit2]) -> floating[Union[_NBit1, _NBit2]]: ... - class _UnsignedIntOp(Protocol[_NBit_co]): + class _UnsignedIntOp(Protocol[_NBit1]): # NOTE: `uint64 + signedinteger -> float64` @overload - def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + def __call__(self, __other: bool) -> unsignedinteger[_NBit1]: ... @overload def __call__( self, __other: Union[int, signedinteger[Any]] - ) -> Union[signedinteger[Any], float64]: ... + ) -> Any: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload - def __call__(self, __other: complex) -> complex128: ... + def __call__( + self, __other: complex + ) -> complexfloating[Union[_NBit1, _NBitDouble], Union[_NBit1, _NBitDouble]]: ... @overload def __call__( - self, __other: unsignedinteger[_NBit] - ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + self, __other: unsignedinteger[_NBit2] + ) -> unsignedinteger[Union[_NBit1, _NBit2]]: ... - class _UnsignedIntBitOp(Protocol[_NBit_co]): + class _UnsignedIntBitOp(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + def __call__(self, __other: bool) -> unsignedinteger[_NBit1]: ... @overload def __call__(self, __other: int) -> signedinteger[Any]: ... @overload def __call__(self, __other: signedinteger[Any]) -> signedinteger[Any]: ... @overload def __call__( - self, __other: unsignedinteger[_NBit] - ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + self, __other: unsignedinteger[_NBit2] + ) -> unsignedinteger[Union[_NBit1, _NBit2]]: ... - class _UnsignedIntMod(Protocol[_NBit_co]): + class _UnsignedIntMod(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> unsignedinteger[_NBit_co]: ... + def __call__(self, __other: bool) -> unsignedinteger[_NBit1]: ... @overload def __call__( self, __other: Union[int, signedinteger[Any]] - ) -> Union[signedinteger[Any], float64]: ... + ) -> Any: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload def __call__( - self, __other: unsignedinteger[_NBit] - ) -> unsignedinteger[Union[_NBit_co, _NBit]]: ... + self, __other: unsignedinteger[_NBit2] + ) -> unsignedinteger[Union[_NBit1, _NBit2]]: ... - class _UnsignedIntDivMod(Protocol[_NBit_co]): + class _UnsignedIntDivMod(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit_co]]: ... + def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit1]]: ... @overload def __call__( self, __other: Union[int, signedinteger[Any]] - ) -> Union[_2Tuple[signedinteger[Any]], _2Tuple[float64]]: ... + ) -> _2Tuple[Any]: ... @overload - def __call__(self, __other: float) -> _2Tuple[float64]: ... + def __call__(self, __other: float) -> _2Tuple[floating[Union[_NBit1, _NBitDouble]]]: ... @overload def __call__( - self, __other: unsignedinteger[_NBit] - ) -> _2Tuple[unsignedinteger[Union[_NBit_co, _NBit]]]: ... + self, __other: unsignedinteger[_NBit2] + ) -> _2Tuple[unsignedinteger[Union[_NBit1, _NBit2]]]: ... - class _SignedIntOp(Protocol[_NBit_co]): + class _SignedIntOp(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + def __call__(self, __other: bool) -> signedinteger[_NBit1]: ... @overload - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> signedinteger[Union[_NBit1, _NBitInt]]: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload - def __call__(self, __other: complex) -> complex128: ... + def __call__( + self, __other: complex + ) -> complexfloating[Union[_NBit1, _NBitDouble], Union[_NBit1, _NBitDouble]]: ... @overload def __call__( - self, __other: signedinteger[_NBit] - ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + self, __other: signedinteger[_NBit2] + ) -> signedinteger[Union[_NBit1, _NBit2]]: ... - class _SignedIntBitOp(Protocol[_NBit_co]): + class _SignedIntBitOp(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + def __call__(self, __other: bool) -> signedinteger[_NBit1]: ... @overload - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> signedinteger[Union[_NBit1, _NBitInt]]: ... @overload def __call__( - self, __other: signedinteger[_NBit] - ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + self, __other: signedinteger[_NBit2] + ) -> signedinteger[Union[_NBit1, _NBit2]]: ... - class _SignedIntMod(Protocol[_NBit_co]): + class _SignedIntMod(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> signedinteger[_NBit_co]: ... + def __call__(self, __other: bool) -> signedinteger[_NBit1]: ... @overload - def __call__(self, __other: int) -> signedinteger[Any]: ... + def __call__(self, __other: int) -> signedinteger[Union[_NBit1, _NBitInt]]: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload def __call__( - self, __other: signedinteger[_NBit] - ) -> signedinteger[Union[_NBit_co, _NBit]]: ... + self, __other: signedinteger[_NBit2] + ) -> signedinteger[Union[_NBit1, _NBit2]]: ... - class _SignedIntDivMod(Protocol[_NBit_co]): + class _SignedIntDivMod(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit_co]]: ... + def __call__(self, __other: bool) -> _2Tuple[signedinteger[_NBit1]]: ... @overload - def __call__(self, __other: int) -> _2Tuple[signedinteger[Any]]: ... + def __call__(self, __other: int) -> _2Tuple[signedinteger[Union[_NBit1, _NBitInt]]]: ... @overload - def __call__(self, __other: float) -> _2Tuple[float64]: ... + def __call__(self, __other: float) -> _2Tuple[floating[Union[_NBit1, _NBitDouble]]]: ... @overload def __call__( - self, __other: signedinteger[_NBit] - ) -> _2Tuple[signedinteger[Union[_NBit_co, _NBit]]]: ... + self, __other: signedinteger[_NBit2] + ) -> _2Tuple[signedinteger[Union[_NBit1, _NBit2]]]: ... - class _FloatOp(Protocol[_NBit_co]): + class _FloatOp(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> floating[_NBit_co]: ... + def __call__(self, __other: bool) -> floating[_NBit1]: ... @overload - def __call__(self, __other: int) -> floating[Any]: ... + def __call__(self, __other: int) -> floating[Union[_NBit1, _NBitInt]]: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload - def __call__(self, __other: complex) -> complex128: ... + def __call__( + self, __other: complex + ) -> complexfloating[Union[_NBit1, _NBitDouble], Union[_NBit1, _NBitDouble]]: ... @overload def __call__( - self, __other: Union[integer[_NBit], floating[_NBit]] - ) -> floating[Union[_NBit_co, _NBit]]: ... + self, __other: Union[integer[_NBit2], floating[_NBit2]] + ) -> floating[Union[_NBit1, _NBit2]]: ... - class _FloatMod(Protocol[_NBit_co]): + class _FloatMod(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> floating[_NBit_co]: ... + def __call__(self, __other: bool) -> floating[_NBit1]: ... @overload - def __call__(self, __other: int) -> floating[Any]: ... + def __call__(self, __other: int) -> floating[Union[_NBit1, _NBitInt]]: ... @overload - def __call__(self, __other: float) -> float64: ... + def __call__(self, __other: float) -> floating[Union[_NBit1, _NBitDouble]]: ... @overload def __call__( - self, __other: Union[integer[_NBit], floating[_NBit]] - ) -> floating[Union[_NBit_co, _NBit]]: ... + self, __other: Union[integer[_NBit2], floating[_NBit2]] + ) -> floating[Union[_NBit1, _NBit2]]: ... - class _FloatDivMod(Protocol[_NBit_co]): + class _FloatDivMod(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> _2Tuple[floating[_NBit_co]]: ... + def __call__(self, __other: bool) -> _2Tuple[floating[_NBit1]]: ... @overload - def __call__(self, __other: int) -> _2Tuple[floating[Any]]: ... + def __call__(self, __other: int) -> _2Tuple[floating[Union[_NBit1, _NBitInt]]]: ... @overload - def __call__(self, __other: float) -> _2Tuple[float64]: ... + def __call__(self, __other: float) -> _2Tuple[floating[Union[_NBit1, _NBitDouble]]]: ... @overload def __call__( - self, __other: Union[integer[_NBit], floating[_NBit]] - ) -> _2Tuple[floating[Union[_NBit_co, _NBit]]]: ... + self, __other: Union[integer[_NBit2], floating[_NBit2]] + ) -> _2Tuple[floating[Union[_NBit1, _NBit2]]]: ... - class _ComplexOp(Protocol[_NBit_co]): + class _ComplexOp(Protocol[_NBit1]): @overload - def __call__(self, __other: bool) -> complexfloating[_NBit_co, _NBit_co]: ... + def __call__(self, __other: bool) -> complexfloating[_NBit1, _NBit1]: ... @overload - def __call__(self, __other: int) -> complexfloating[Any, Any]: ... + def __call__(self, __other: int) -> complexfloating[Union[_NBit1, _NBitInt], Union[_NBit1, _NBitInt]]: ... @overload - def __call__(self, __other: Union[float, complex]) -> complex128: ... + def __call__( + self, __other: Union[float, complex] + ) -> complexfloating[Union[_NBit1, _NBitDouble], Union[_NBit1, _NBitDouble]]: ... @overload def __call__( self, __other: Union[ - integer[_NBit], - floating[_NBit], - complexfloating[_NBit, _NBit], + integer[_NBit2], + floating[_NBit2], + complexfloating[_NBit2, _NBit2], ] - ) -> complexfloating[Union[_NBit_co, _NBit], Union[_NBit_co, _NBit]]: ... + ) -> complexfloating[Union[_NBit1, _NBit2], Union[_NBit1, _NBit2]]: ... class _NumberOp(Protocol): - def __call__(self, __other: _NumberLike) -> number: ... + def __call__(self, __other: _NumberLike_co) -> Any: ... + + class _ComparisonOp(Protocol[_T1, _T2]): + @overload + def __call__(self, __other: _T1) -> bool_: ... + @overload + def __call__(self, __other: _T2) -> NDArray[bool_]: ... else: _BoolOp = Any @@ -334,3 +361,4 @@ def __call__(self, __other: _NumberLike) -> number: ... _FloatDivMod = Any _ComplexOp = Any _NumberOp = Any + _ComparisonOp = Any diff --git a/numpy/typing/_char_codes.py b/numpy/typing/_char_codes.py new file mode 100644 index 000000000000..6b6f7ae88946 --- /dev/null +++ b/numpy/typing/_char_codes.py @@ -0,0 +1,175 @@ +import sys +from typing import Any, TYPE_CHECKING + +if sys.version_info >= (3, 8): + from typing import Literal + HAVE_LITERAL = True +else: + try: + from typing_extensions import Literal + except ImportError: + HAVE_LITERAL = False + else: + HAVE_LITERAL = True + +if TYPE_CHECKING or HAVE_LITERAL: + _BoolCodes = Literal["?", "=?", "?", "bool", "bool_", "bool8"] + + _UInt8Codes = Literal["uint8", "u1", "=u1", "u1"] + _UInt16Codes = Literal["uint16", "u2", "=u2", "u2"] + _UInt32Codes = Literal["uint32", "u4", "=u4", "u4"] + _UInt64Codes = Literal["uint64", "u8", "=u8", "u8"] + + _Int8Codes = Literal["int8", "i1", "=i1", "i1"] + _Int16Codes = Literal["int16", "i2", "=i2", "i2"] + _Int32Codes = Literal["int32", "i4", "=i4", "i4"] + _Int64Codes = Literal["int64", "i8", "=i8", "i8"] + + _Float16Codes = Literal["float16", "f2", "=f2", "f2"] + _Float32Codes = Literal["float32", "f4", "=f4", "f4"] + _Float64Codes = Literal["float64", "f8", "=f8", "f8"] + + _Complex64Codes = Literal["complex64", "c8", "=c8", "c8"] + _Complex128Codes = Literal["complex128", "c16", "=c16", "c16"] + + _ByteCodes = Literal["byte", "b", "=b", "b"] + _ShortCodes = Literal["short", "h", "=h", "h"] + _IntCCodes = Literal["intc", "i", "=i", "i"] + _IntPCodes = Literal["intp", "int0", "p", "=p", "p"] + _IntCodes = Literal["long", "int", "int_", "l", "=l", "l"] + _LongLongCodes = Literal["longlong", "q", "=q", "q"] + + _UByteCodes = Literal["ubyte", "B", "=B", "B"] + _UShortCodes = Literal["ushort", "H", "=H", "H"] + _UIntCCodes = Literal["uintc", "I", "=I", "I"] + _UIntPCodes = Literal["uintp", "uint0", "P", "=P", "P"] + _UIntCodes = Literal["uint", "L", "=L", "L"] + _ULongLongCodes = Literal["ulonglong", "Q", "=Q", "Q"] + + _HalfCodes = Literal["half", "e", "=e", "e"] + _SingleCodes = Literal["single", "f", "=f", "f"] + _DoubleCodes = Literal["double", "float", "float_", "d", "=d", "d"] + _LongDoubleCodes = Literal["longdouble", "longfloat", "g", "=g", "g"] + + _CSingleCodes = Literal["csingle", "singlecomplex", "F", "=F", "F"] + _CDoubleCodes = Literal["cdouble", "complex", "complex_", "cfloat", "D", "=D", "D"] + _CLongDoubleCodes = Literal["clongdouble", "clongfloat", "longcomplex", "G", "=G", "G"] + + _StrCodes = Literal["str", "str_", "str0", "unicode", "unicode_", "U", "=U", "U"] + _BytesCodes = Literal["bytes", "bytes_", "bytes0", "S", "=S", "S"] + _VoidCodes = Literal["void", "void0", "V", "=V", "V"] + _ObjectCodes = Literal["object", "object_", "O", "=O", "O"] + + _DT64Codes = Literal[ + "datetime64", "=datetime64", "datetime64", + "datetime64[Y]", "=datetime64[Y]", "datetime64[Y]", + "datetime64[M]", "=datetime64[M]", "datetime64[M]", + "datetime64[W]", "=datetime64[W]", "datetime64[W]", + "datetime64[D]", "=datetime64[D]", "datetime64[D]", + "datetime64[h]", "=datetime64[h]", "datetime64[h]", + "datetime64[m]", "=datetime64[m]", "datetime64[m]", + "datetime64[s]", "=datetime64[s]", "datetime64[s]", + "datetime64[ms]", "=datetime64[ms]", "datetime64[ms]", + "datetime64[us]", "=datetime64[us]", "datetime64[us]", + "datetime64[ns]", "=datetime64[ns]", "datetime64[ns]", + "datetime64[ps]", "=datetime64[ps]", "datetime64[ps]", + "datetime64[fs]", "=datetime64[fs]", "datetime64[fs]", + "datetime64[as]", "=datetime64[as]", "datetime64[as]", + "M", "=M", "M", + "M8", "=M8", "M8", + "M8[Y]", "=M8[Y]", "M8[Y]", + "M8[M]", "=M8[M]", "M8[M]", + "M8[W]", "=M8[W]", "M8[W]", + "M8[D]", "=M8[D]", "M8[D]", + "M8[h]", "=M8[h]", "M8[h]", + "M8[m]", "=M8[m]", "M8[m]", + "M8[s]", "=M8[s]", "M8[s]", + "M8[ms]", "=M8[ms]", "M8[ms]", + "M8[us]", "=M8[us]", "M8[us]", + "M8[ns]", "=M8[ns]", "M8[ns]", + "M8[ps]", "=M8[ps]", "M8[ps]", + "M8[fs]", "=M8[fs]", "M8[fs]", + "M8[as]", "=M8[as]", "M8[as]", + ] + _TD64Codes = Literal[ + "timedelta64", "=timedelta64", "timedelta64", + "timedelta64[Y]", "=timedelta64[Y]", "timedelta64[Y]", + "timedelta64[M]", "=timedelta64[M]", "timedelta64[M]", + "timedelta64[W]", "=timedelta64[W]", "timedelta64[W]", + "timedelta64[D]", "=timedelta64[D]", "timedelta64[D]", + "timedelta64[h]", "=timedelta64[h]", "timedelta64[h]", + "timedelta64[m]", "=timedelta64[m]", "timedelta64[m]", + "timedelta64[s]", "=timedelta64[s]", "timedelta64[s]", + "timedelta64[ms]", "=timedelta64[ms]", "timedelta64[ms]", + "timedelta64[us]", "=timedelta64[us]", "timedelta64[us]", + "timedelta64[ns]", "=timedelta64[ns]", "timedelta64[ns]", + "timedelta64[ps]", "=timedelta64[ps]", "timedelta64[ps]", + "timedelta64[fs]", "=timedelta64[fs]", "timedelta64[fs]", + "timedelta64[as]", "=timedelta64[as]", "timedelta64[as]", + "m", "=m", "m", + "m8", "=m8", "m8", + "m8[Y]", "=m8[Y]", "m8[Y]", + "m8[M]", "=m8[M]", "m8[M]", + "m8[W]", "=m8[W]", "m8[W]", + "m8[D]", "=m8[D]", "m8[D]", + "m8[h]", "=m8[h]", "m8[h]", + "m8[m]", "=m8[m]", "m8[m]", + "m8[s]", "=m8[s]", "m8[s]", + "m8[ms]", "=m8[ms]", "m8[ms]", + "m8[us]", "=m8[us]", "m8[us]", + "m8[ns]", "=m8[ns]", "m8[ns]", + "m8[ps]", "=m8[ps]", "m8[ps]", + "m8[fs]", "=m8[fs]", "m8[fs]", + "m8[as]", "=m8[as]", "m8[as]", + ] + +else: + _BoolCodes = Any + + _UInt8Codes = Any + _UInt16Codes = Any + _UInt32Codes = Any + _UInt64Codes = Any + + _Int8Codes = Any + _Int16Codes = Any + _Int32Codes = Any + _Int64Codes = Any + + _Float16Codes = Any + _Float32Codes = Any + _Float64Codes = Any + + _Complex64Codes = Any + _Complex128Codes = Any + + _ByteCodes = Any + _ShortCodes = Any + _IntCCodes = Any + _IntPCodes = Any + _IntCodes = Any + _LongLongCodes = Any + + _UByteCodes = Any + _UShortCodes = Any + _UIntCCodes = Any + _UIntPCodes = Any + _UIntCodes = Any + _ULongLongCodes = Any + + _HalfCodes = Any + _SingleCodes = Any + _DoubleCodes = Any + _LongDoubleCodes = Any + + _CSingleCodes = Any + _CDoubleCodes = Any + _CLongDoubleCodes = Any + + _StrCodes = Any + _BytesCodes = Any + _VoidCodes = Any + _ObjectCodes = Any + + _DT64Codes = Any + _TD64Codes = Any diff --git a/numpy/typing/_dtype_like.py b/numpy/typing/_dtype_like.py index 1953bd5fcfcc..a41e2f358d97 100644 --- a/numpy/typing/_dtype_like.py +++ b/numpy/typing/_dtype_like.py @@ -1,7 +1,7 @@ import sys -from typing import Any, List, Sequence, Tuple, Union, TYPE_CHECKING +from typing import Any, List, Sequence, Tuple, Union, Type, TypeVar, TYPE_CHECKING -from numpy import dtype +import numpy as np from ._shape import _ShapeLike if sys.version_info >= (3, 8): @@ -15,6 +15,48 @@ else: HAVE_PROTOCOL = True +from ._char_codes import ( + _BoolCodes, + _UInt8Codes, + _UInt16Codes, + _UInt32Codes, + _UInt64Codes, + _Int8Codes, + _Int16Codes, + _Int32Codes, + _Int64Codes, + _Float16Codes, + _Float32Codes, + _Float64Codes, + _Complex64Codes, + _Complex128Codes, + _ByteCodes, + _ShortCodes, + _IntCCodes, + _IntPCodes, + _IntCodes, + _LongLongCodes, + _UByteCodes, + _UShortCodes, + _UIntCCodes, + _UIntPCodes, + _UIntCodes, + _ULongLongCodes, + _HalfCodes, + _SingleCodes, + _DoubleCodes, + _LongDoubleCodes, + _CSingleCodes, + _CDoubleCodes, + _CLongDoubleCodes, + _DT64Codes, + _TD64Codes, + _StrCodes, + _BytesCodes, + _VoidCodes, + _ObjectCodes, +) + _DTypeLikeNested = Any # TODO: wait for support for recursive types if TYPE_CHECKING or HAVE_PROTOCOL: @@ -30,9 +72,12 @@ class _DTypeDict(_DTypeDictBase, total=False): itemsize: int aligned: bool + _DType_co = TypeVar("_DType_co", covariant=True, bound=np.dtype) + # A protocol for anything with the dtype attribute - class _SupportsDType(Protocol): - dtype: _DTypeLikeNested + class _SupportsDType(Protocol[_DType_co]): + @property + def dtype(self) -> _DType_co: ... else: _DTypeDict = Any @@ -61,13 +106,13 @@ class _SupportsDType(Protocol): # Anything that can be coerced into numpy.dtype. # Reference: https://docs.scipy.org/doc/numpy/reference/arrays.dtypes.html DTypeLike = Union[ - dtype, + np.dtype, # default data type (float64) None, # array-scalar types and generic types type, # TODO: enumerate these when we add type hints for numpy scalars # anything with a dtype attribute - _SupportsDType, + "_SupportsDType[np.dtype[Any]]", # character codes, type strings or comma-separated fields, e.g., 'float64' str, _VoidDTypeLike, @@ -79,3 +124,115 @@ class _SupportsDType(Protocol): # therefore not included in the Union defining `DTypeLike`. # # See https://github.com/numpy/numpy/issues/16891 for more details. + +# Aliases for commonly used dtype-like objects. +# Note that the precision of `np.number` subclasses is ignored herein. +_DTypeLikeBool = Union[ + Type[bool], + Type[np.bool_], + "np.dtype[np.bool_]", + "_SupportsDType[np.dtype[np.bool_]]", + _BoolCodes, +] +_DTypeLikeUInt = Union[ + Type[np.unsignedinteger], + "np.dtype[np.unsignedinteger]", + "_SupportsDType[np.dtype[np.unsignedinteger]]", + _UInt8Codes, + _UInt16Codes, + _UInt32Codes, + _UInt64Codes, + _UByteCodes, + _UShortCodes, + _UIntCCodes, + _UIntPCodes, + _UIntCodes, + _ULongLongCodes, +] +_DTypeLikeInt = Union[ + Type[int], + Type[np.signedinteger], + "np.dtype[np.signedinteger]", + "_SupportsDType[np.dtype[np.signedinteger]]", + _Int8Codes, + _Int16Codes, + _Int32Codes, + _Int64Codes, + _ByteCodes, + _ShortCodes, + _IntCCodes, + _IntPCodes, + _IntCodes, + _LongLongCodes, +] +_DTypeLikeFloat = Union[ + Type[float], + Type[np.floating], + "np.dtype[np.floating]", + "_SupportsDType[np.dtype[np.floating]]", + _Float16Codes, + _Float32Codes, + _Float64Codes, + _HalfCodes, + _SingleCodes, + _DoubleCodes, + _LongDoubleCodes, +] +_DTypeLikeComplex = Union[ + Type[complex], + Type[np.complexfloating], + "np.dtype[np.complexfloating]", + "_SupportsDType[np.dtype[np.complexfloating]]", + _Complex64Codes, + _Complex128Codes, + _CSingleCodes, + _CDoubleCodes, + _CLongDoubleCodes, +] +_DTypeLikeDT64 = Union[ + Type[np.timedelta64], + "np.dtype[np.timedelta64]", + "_SupportsDType[np.dtype[np.timedelta64]]", + _TD64Codes, +] +_DTypeLikeTD64 = Union[ + Type[np.datetime64], + "np.dtype[np.datetime64]", + "_SupportsDType[np.dtype[np.datetime64]]", + _DT64Codes, +] +_DTypeLikeStr = Union[ + Type[str], + Type[np.str_], + "np.dtype[np.str_]", + "_SupportsDType[np.dtype[np.str_]]", + _StrCodes, +] +_DTypeLikeBytes = Union[ + Type[bytes], + Type[np.bytes_], + "np.dtype[np.bytes_]", + "_SupportsDType[np.dtype[np.bytes_]]", + _BytesCodes, +] +_DTypeLikeVoid = Union[ + Type[np.void], + "np.dtype[np.void]", + "_SupportsDType[np.dtype[np.void]]", + _VoidCodes, + _VoidDTypeLike, +] +_DTypeLikeObject = Union[ + type, + "np.dtype[np.object_]", + "_SupportsDType[np.dtype[np.object_]]", + _ObjectCodes, +] + +_DTypeLikeComplex_co = Union[ + _DTypeLikeBool, + _DTypeLikeUInt, + _DTypeLikeInt, + _DTypeLikeFloat, + _DTypeLikeComplex, +] diff --git a/numpy/typing/_extended_precision.py b/numpy/typing/_extended_precision.py new file mode 100644 index 000000000000..3f1ce2038282 --- /dev/null +++ b/numpy/typing/_extended_precision.py @@ -0,0 +1,42 @@ +"""A module with platform-specific extended precision `numpy.number` subclasses. + +The subclasses are defined here (instead of ``__init__.pyi``) such +that they can be imported conditionally via the numpy's mypy plugin. +""" + +from typing import TYPE_CHECKING + +import numpy as np +from . import ( + _80Bit, + _96Bit, + _128Bit, + _256Bit, +) + +if TYPE_CHECKING: + uint128 = np.unsignedinteger[_128Bit] + uint256 = np.unsignedinteger[_256Bit] + int128 = np.signedinteger[_128Bit] + int256 = np.signedinteger[_256Bit] + float80 = np.floating[_80Bit] + float96 = np.floating[_96Bit] + float128 = np.floating[_128Bit] + float256 = np.floating[_256Bit] + complex160 = np.complexfloating[_80Bit, _80Bit] + complex192 = np.complexfloating[_96Bit, _96Bit] + complex256 = np.complexfloating[_128Bit, _128Bit] + complex512 = np.complexfloating[_256Bit, _256Bit] +else: + uint128 = NotImplemented + uint256 = NotImplemented + int128 = NotImplemented + int256 = NotImplemented + float80 = NotImplemented + float96 = NotImplemented + float128 = NotImplemented + float256 = NotImplemented + complex160 = NotImplemented + complex192 = NotImplemented + complex256 = NotImplemented + complex512 = NotImplemented diff --git a/numpy/typing/_generic_alias.py b/numpy/typing/_generic_alias.py new file mode 100644 index 000000000000..68523827a71a --- /dev/null +++ b/numpy/typing/_generic_alias.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import sys +import types +from typing import ( + Any, + ClassVar, + FrozenSet, + Generator, + Iterable, + Iterator, + List, + NoReturn, + Tuple, + Type, + TypeVar, + TYPE_CHECKING, +) + +import numpy as np + +__all__ = ["_GenericAlias", "NDArray"] + +_T = TypeVar("_T", bound="_GenericAlias") + + +def _to_str(obj: object) -> str: + """Helper function for `_GenericAlias.__repr__`.""" + if obj is Ellipsis: + return '...' + elif isinstance(obj, type) and not isinstance(obj, _GENERIC_ALIAS_TYPE): + if obj.__module__ == 'builtins': + return obj.__qualname__ + else: + return f'{obj.__module__}.{obj.__qualname__}' + else: + return repr(obj) + + +def _parse_parameters(args: Iterable[Any]) -> Generator[TypeVar, None, None]: + """Search for all typevars and typevar-containing objects in `args`. + + Helper function for `_GenericAlias.__init__`. + + """ + for i in args: + if hasattr(i, "__parameters__"): + yield from i.__parameters__ + elif isinstance(i, TypeVar): + yield i + + +def _reconstruct_alias(alias: _T, parameters: Iterator[TypeVar]) -> _T: + """Recursivelly replace all typevars with those from `parameters`. + + Helper function for `_GenericAlias.__getitem__`. + + """ + args = [] + for i in alias.__args__: + if isinstance(i, TypeVar): + value: Any = next(parameters) + elif isinstance(i, _GenericAlias): + value = _reconstruct_alias(i, parameters) + elif hasattr(i, "__parameters__"): + prm_tup = tuple(next(parameters) for _ in i.__parameters__) + value = i[prm_tup] + else: + value = i + args.append(value) + + cls = type(alias) + return cls(alias.__origin__, tuple(args)) + + +class _GenericAlias: + """A python-based backport of the `types.GenericAlias` class. + + E.g. for ``t = list[int]``, ``t.__origin__`` is ``list`` and + ``t.__args__`` is ``(int,)``. + + See Also + -------- + :pep:`585` + The PEP responsible for introducing `types.GenericAlias`. + + """ + + __slots__ = ("__weakref__", "_origin", "_args", "_parameters", "_hash") + + @property + def __origin__(self) -> type: + return super().__getattribute__("_origin") + + @property + def __args__(self) -> Tuple[Any, ...]: + return super().__getattribute__("_args") + + @property + def __parameters__(self) -> Tuple[TypeVar, ...]: + """Type variables in the ``GenericAlias``.""" + return super().__getattribute__("_parameters") + + def __init__(self, origin: type, args: Any) -> None: + self._origin = origin + self._args = args if isinstance(args, tuple) else (args,) + self._parameters = tuple(_parse_parameters(args)) + + @property + def __call__(self) -> type: + return self.__origin__ + + def __reduce__(self: _T) -> Tuple[Type[_T], Tuple[type, Tuple[Any, ...]]]: + cls = type(self) + return cls, (self.__origin__, self.__args__) + + def __mro_entries__(self, bases: Iterable[object]) -> Tuple[type]: + return (self.__origin__,) + + def __dir__(self) -> List[str]: + """Implement ``dir(self)``.""" + cls = type(self) + dir_origin = set(dir(self.__origin__)) + return sorted(cls._ATTR_EXCEPTIONS | dir_origin) + + def __hash__(self) -> int: + """Return ``hash(self)``.""" + # Attempt to use the cached hash + try: + return super().__getattribute__("_hash") + except AttributeError: + self._hash: int = hash(self.__origin__) ^ hash(self.__args__) + return super().__getattribute__("_hash") + + def __instancecheck__(self, obj: object) -> NoReturn: + """Check if an `obj` is an instance.""" + raise TypeError("isinstance() argument 2 cannot be a " + "parameterized generic") + + def __subclasscheck__(self, cls: type) -> NoReturn: + """Check if a `cls` is a subclass.""" + raise TypeError("issubclass() argument 2 cannot be a " + "parameterized generic") + + def __repr__(self) -> str: + """Return ``repr(self)``.""" + args = ", ".join(_to_str(i) for i in self.__args__) + origin = _to_str(self.__origin__) + return f"{origin}[{args}]" + + def __getitem__(self: _T, key: Any) -> _T: + """Return ``self[key]``.""" + key_tup = key if isinstance(key, tuple) else (key,) + + if len(self.__parameters__) == 0: + raise TypeError(f"There are no type variables left in {self}") + elif len(key_tup) > len(self.__parameters__): + raise TypeError(f"Too many arguments for {self}") + elif len(key_tup) < len(self.__parameters__): + raise TypeError(f"Too few arguments for {self}") + + key_iter = iter(key_tup) + return _reconstruct_alias(self, key_iter) + + def __eq__(self, value: object) -> bool: + """Return ``self == value``.""" + if not isinstance(value, _GENERIC_ALIAS_TYPE): + return NotImplemented + return ( + self.__origin__ == value.__origin__ and + self.__args__ == value.__args__ + ) + + _ATTR_EXCEPTIONS: ClassVar[FrozenSet[str]] = frozenset({ + "__origin__", + "__args__", + "__parameters__", + "__mro_entries__", + "__reduce__", + "__reduce_ex__", + }) + + def __getattribute__(self, name: str) -> Any: + """Return ``getattr(self, name)``.""" + # Pull the attribute from `__origin__` unless its + # name is in `_ATTR_EXCEPTIONS` + cls = type(self) + if name in cls._ATTR_EXCEPTIONS: + return super().__getattribute__(name) + return getattr(self.__origin__, name) + + +# See `_GenericAlias.__eq__` +if sys.version_info >= (3, 9): + _GENERIC_ALIAS_TYPE = (_GenericAlias, types.GenericAlias) +else: + _GENERIC_ALIAS_TYPE = (_GenericAlias,) + +ScalarType = TypeVar("ScalarType", bound=np.generic, covariant=True) + +if TYPE_CHECKING: + NDArray = np.ndarray[Any, np.dtype[ScalarType]] +elif sys.version_info >= (3, 9): + _DType = types.GenericAlias(np.dtype, (ScalarType,)) + NDArray = types.GenericAlias(np.ndarray, (Any, _DType)) +else: + _DType = _GenericAlias(np.dtype, (ScalarType,)) + NDArray = _GenericAlias(np.ndarray, (Any, _DType)) diff --git a/numpy/typing/_nbit.py b/numpy/typing/_nbit.py new file mode 100644 index 000000000000..b8d35db4f594 --- /dev/null +++ b/numpy/typing/_nbit.py @@ -0,0 +1,16 @@ +"""A module with the precisions of platform-specific `~numpy.number`s.""" + +from typing import Any + +# To-be replaced with a `npt.NBitBase` subclass by numpy's mypy plugin +_NBitByte = Any +_NBitShort = Any +_NBitIntC = Any +_NBitIntP = Any +_NBitInt = Any +_NBitLongLong = Any + +_NBitHalf = Any +_NBitSingle = Any +_NBitDouble = Any +_NBitLongDouble = Any diff --git a/numpy/typing/_scalars.py b/numpy/typing/_scalars.py index e4fc28b074ac..516b996dc007 100644 --- a/numpy/typing/_scalars.py +++ b/numpy/typing/_scalars.py @@ -2,18 +2,22 @@ import numpy as np -# NOTE: `_StrLike` and `_BytesLike` are pointless, as `np.str_` and `np.bytes_` -# are already subclasses of their builtin counterpart +# NOTE: `_StrLike_co` and `_BytesLike_co` are pointless, as `np.str_` and +# `np.bytes_` are already subclasses of their builtin counterpart -_CharLike = Union[str, bytes] +_CharLike_co = Union[str, bytes] -_BoolLike = Union[bool, np.bool_] -_IntLike = Union[int, np.integer] -_FloatLike = Union[_IntLike, float, np.floating] -_ComplexLike = Union[_FloatLike, complex, np.complexfloating] -_NumberLike = Union[int, float, complex, np.number, np.bool_] +# The 6 `Like_co` type-aliases below represent all scalars that can be +# coerced into `` (with the casting rule `same_kind`) +_BoolLike_co = Union[bool, np.bool_] +_UIntLike_co = Union[_BoolLike_co, np.unsignedinteger] +_IntLike_co = Union[_BoolLike_co, int, np.integer] +_FloatLike_co = Union[_IntLike_co, float, np.floating] +_ComplexLike_co = Union[_FloatLike_co, complex, np.complexfloating] +_TD64Like_co = Union[_IntLike_co, np.timedelta64] -_ScalarLike = Union[ +_NumberLike_co = Union[int, float, complex, np.number, np.bool_] +_ScalarLike_co = Union[ int, float, complex, @@ -22,5 +26,5 @@ np.generic, ] -# `_VoidLike` is technically not a scalar, but it's close enough -_VoidLike = Union[Tuple[Any, ...], np.void] +# `_VoidLike_co` is technically not a scalar, but it's close enough +_VoidLike_co = Union[Tuple[Any, ...], np.void] diff --git a/numpy/typing/_shape.py b/numpy/typing/_shape.py index 4629046ea3eb..b720c3ffc192 100644 --- a/numpy/typing/_shape.py +++ b/numpy/typing/_shape.py @@ -1,6 +1,15 @@ +import sys from typing import Sequence, Tuple, Union +if sys.version_info >= (3, 8): + from typing import SupportsIndex +else: + try: + from typing_extensions import SupportsIndex + except ImportError: + SupportsIndex = NotImplemented + _Shape = Tuple[int, ...] # Anything that can be coerced to a shape tuple -_ShapeLike = Union[int, Sequence[int]] +_ShapeLike = Union[SupportsIndex, Sequence[SupportsIndex]] diff --git a/numpy/typing/_ufunc.pyi b/numpy/typing/_ufunc.pyi new file mode 100644 index 000000000000..b3b9fa95ed39 --- /dev/null +++ b/numpy/typing/_ufunc.pyi @@ -0,0 +1,405 @@ +"""A module with private type-check-only `numpy.ufunc` subclasses. + +The signatures of the ufuncs are too varied to reasonably type +with a single class. So instead, `ufunc` has been expanded into +four private subclasses, one for each combination of +`~ufunc.nin` and `~ufunc.nout`. + +""" + +from typing import ( + Any, + Generic, + List, + Optional, + overload, + Tuple, + TypeVar, + Union, +) + +from numpy import ufunc, _Casting, _OrderKACF +from numpy.typing import NDArray + +from ._shape import _ShapeLike +from ._scalars import _ScalarLike_co +from ._array_like import ArrayLike, _ArrayLikeBool_co, _ArrayLikeInt_co +from ._dtype_like import DTypeLike + +from typing_extensions import Literal, SupportsIndex + +_T = TypeVar("_T") +_2Tuple = Tuple[_T, _T] +_3Tuple = Tuple[_T, _T, _T] +_4Tuple = Tuple[_T, _T, _T, _T] + +_NTypes = TypeVar("_NTypes", bound=int) +_IDType = TypeVar("_IDType", bound=Any) +_NameType = TypeVar("_NameType", bound=str) + +# NOTE: In reality `extobj` should be a length of list 3 containing an +# int, an int, and a callable, but there's no way to properly express +# non-homogenous lists. +# Use `Any` over `Union` to avoid issues related to lists invariance. + +# NOTE: `reduce`, `accumulate`, `reduceat` and `outer` raise a ValueError for +# ufuncs that don't accept two input arguments and return one output argument. +# In such cases the respective methods are simply typed as `None`. + +# NOTE: Similarly, `at` won't be defined for ufuncs that return +# multiple outputs; in such cases `at` is typed as `None` + +# NOTE: If 2 output types are returned then `out` must be a +# 2-tuple of arrays. Otherwise `None` or a plain array are also acceptable + +class _UFunc_Nin1_Nout1(ufunc, Generic[_NameType, _NTypes, _IDType]): + @property + def __name__(self) -> _NameType: ... + @property + def ntypes(self) -> _NTypes: ... + @property + def identity(self) -> _IDType: ... + @property + def nin(self) -> Literal[1]: ... + @property + def nout(self) -> Literal[1]: ... + @property + def nargs(self) -> Literal[2]: ... + @property + def signature(self) -> None: ... + @property + def reduce(self) -> None: ... + @property + def accumulate(self) -> None: ... + @property + def reduceat(self) -> None: ... + @property + def outer(self) -> None: ... + + @overload + def __call__( + self, + __x1: _ScalarLike_co, + out: None = ..., + *, + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _2Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> Any: ... + @overload + def __call__( + self, + __x1: ArrayLike, + out: Union[None, NDArray[Any], Tuple[NDArray[Any]]] = ..., + *, + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _2Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> NDArray[Any]: ... + + def at( + self, + __a: NDArray[Any], + __indices: _ArrayLikeInt_co, + ) -> None: ... + +class _UFunc_Nin2_Nout1(ufunc, Generic[_NameType, _NTypes, _IDType]): + @property + def __name__(self) -> _NameType: ... + @property + def ntypes(self) -> _NTypes: ... + @property + def identity(self) -> _IDType: ... + @property + def nin(self) -> Literal[2]: ... + @property + def nout(self) -> Literal[1]: ... + @property + def nargs(self) -> Literal[3]: ... + @property + def signature(self) -> None: ... + + @overload + def __call__( + self, + __x1: _ScalarLike_co, + __x2: _ScalarLike_co, + out: None = ..., + *, + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> Any: ... + @overload + def __call__( + self, + __x1: ArrayLike, + __x2: ArrayLike, + out: Union[None, NDArray[Any], Tuple[NDArray[Any]]] = ..., + *, + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> NDArray[Any]: ... + + def at( + self, + __a: NDArray[Any], + __indices: _ArrayLikeInt_co, + __b: ArrayLike, + ) -> None: ... + + def reduce( + self, + array: ArrayLike, + axis: Optional[_ShapeLike] = ..., + dtype: DTypeLike = ..., + out: Optional[NDArray[Any]] = ..., + keepdims: bool = ..., + initial: Any = ..., + where: _ArrayLikeBool_co = ..., + ) -> Any: ... + + def accumulate( + self, + array: ArrayLike, + axis: SupportsIndex = ..., + dtype: DTypeLike = ..., + out: Optional[NDArray[Any]] = ..., + ) -> NDArray[Any]: ... + + def reduceat( + self, + array: ArrayLike, + indices: _ArrayLikeInt_co, + axis: SupportsIndex = ..., + dtype: DTypeLike = ..., + out: Optional[NDArray[Any]] = ..., + ) -> NDArray[Any]: ... + + # Expand `**kwargs` into explicit keyword-only arguments + @overload + def outer( + self, + __A: _ScalarLike_co, + __B: _ScalarLike_co, + *, + out: None = ..., + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> Any: ... + @overload + def outer( # type: ignore[misc] + self, + __A: ArrayLike, + __B: ArrayLike, + *, + out: Union[None, NDArray[Any], Tuple[NDArray[Any]]] = ..., + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> NDArray[Any]: ... + +class _UFunc_Nin1_Nout2(ufunc, Generic[_NameType, _NTypes, _IDType]): + @property + def __name__(self) -> _NameType: ... + @property + def ntypes(self) -> _NTypes: ... + @property + def identity(self) -> _IDType: ... + @property + def nin(self) -> Literal[1]: ... + @property + def nout(self) -> Literal[2]: ... + @property + def nargs(self) -> Literal[3]: ... + @property + def signature(self) -> None: ... + @property + def at(self) -> None: ... + @property + def reduce(self) -> None: ... + @property + def accumulate(self) -> None: ... + @property + def reduceat(self) -> None: ... + @property + def outer(self) -> None: ... + + @overload + def __call__( + self, + __x1: _ScalarLike_co, + __out1: None = ..., + __out2: None = ..., + *, + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> _2Tuple[Any]: ... + @overload + def __call__( + self, + __x1: ArrayLike, + __out1: Optional[NDArray[Any]] = ..., + __out2: Optional[NDArray[Any]] = ..., + *, + out: _2Tuple[NDArray[Any]] = ..., + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> _2Tuple[NDArray[Any]]: ... + +class _UFunc_Nin2_Nout2(ufunc, Generic[_NameType, _NTypes, _IDType]): + @property + def __name__(self) -> _NameType: ... + @property + def ntypes(self) -> _NTypes: ... + @property + def identity(self) -> _IDType: ... + @property + def nin(self) -> Literal[2]: ... + @property + def nout(self) -> Literal[2]: ... + @property + def nargs(self) -> Literal[4]: ... + @property + def signature(self) -> None: ... + @property + def at(self) -> None: ... + @property + def reduce(self) -> None: ... + @property + def accumulate(self) -> None: ... + @property + def reduceat(self) -> None: ... + @property + def outer(self) -> None: ... + + @overload + def __call__( + self, + __x1: _ScalarLike_co, + __x2: _ScalarLike_co, + __out1: None = ..., + __out2: None = ..., + *, + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _4Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> _2Tuple[Any]: ... + @overload + def __call__( + self, + __x1: ArrayLike, + __x2: ArrayLike, + __out1: Optional[NDArray[Any]] = ..., + __out2: Optional[NDArray[Any]] = ..., + *, + out: _2Tuple[NDArray[Any]] = ..., + where: Optional[_ArrayLikeBool_co] = ..., + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _4Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + ) -> _2Tuple[NDArray[Any]]: ... + +class _GUFunc_Nin2_Nout1(ufunc, Generic[_NameType, _NTypes, _IDType]): + @property + def __name__(self) -> _NameType: ... + @property + def ntypes(self) -> _NTypes: ... + @property + def identity(self) -> _IDType: ... + @property + def nin(self) -> Literal[2]: ... + @property + def nout(self) -> Literal[1]: ... + @property + def nargs(self) -> Literal[3]: ... + + # NOTE: In practice the only gufunc in the main name is `matmul`, + # so we can use its signature here + @property + def signature(self) -> Literal["(n?,k),(k,m?)->(n?,m?)"]: ... + @property + def reduce(self) -> None: ... + @property + def accumulate(self) -> None: ... + @property + def reduceat(self) -> None: ... + @property + def outer(self) -> None: ... + @property + def at(self) -> None: ... + + # Scalar for 1D array-likes; ndarray otherwise + @overload + def __call__( + self, + __x1: ArrayLike, + __x2: ArrayLike, + out: None = ..., + *, + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + axes: List[_2Tuple[SupportsIndex]] = ..., + ) -> Any: ... + @overload + def __call__( + self, + __x1: ArrayLike, + __x2: ArrayLike, + out: Union[NDArray[Any], Tuple[NDArray[Any]]], + *, + casting: _Casting = ..., + order: _OrderKACF = ..., + dtype: DTypeLike = ..., + subok: bool = ..., + signature: Union[str, _3Tuple[Optional[str]]] = ..., + extobj: List[Any] = ..., + axes: List[_2Tuple[SupportsIndex]] = ..., + ) -> NDArray[Any]: ... diff --git a/numpy/typing/mypy_plugin.py b/numpy/typing/mypy_plugin.py new file mode 100644 index 000000000000..901bf4fb121e --- /dev/null +++ b/numpy/typing/mypy_plugin.py @@ -0,0 +1,131 @@ +"""A module containing `numpy`-specific plugins for mypy.""" + +from __future__ import annotations + +import typing as t + +import numpy as np + +try: + import mypy.types + from mypy.types import Type + from mypy.plugin import Plugin, AnalyzeTypeContext + from mypy.nodes import MypyFile, ImportFrom, Statement + from mypy.build import PRI_MED + + _HookFunc = t.Callable[[AnalyzeTypeContext], Type] + MYPY_EX: t.Optional[ModuleNotFoundError] = None +except ModuleNotFoundError as ex: + MYPY_EX = ex + +__all__: t.List[str] = [] + + +def _get_precision_dict() -> t.Dict[str, str]: + names = [ + ("_NBitByte", np.byte), + ("_NBitShort", np.short), + ("_NBitIntC", np.intc), + ("_NBitIntP", np.intp), + ("_NBitInt", np.int_), + ("_NBitLongLong", np.longlong), + + ("_NBitHalf", np.half), + ("_NBitSingle", np.single), + ("_NBitDouble", np.double), + ("_NBitLongDouble", np.longdouble), + ] + ret = {} + for name, typ in names: + n: int = 8 * typ().dtype.itemsize + ret[f'numpy.typing._nbit.{name}'] = f"numpy._{n}Bit" + return ret + + +def _get_extended_precision_list() -> t.List[str]: + extended_types = [np.ulonglong, np.longlong, np.longdouble, np.clongdouble] + extended_names = { + "uint128", + "uint256", + "int128", + "int256", + "float80", + "float96", + "float128", + "float256", + "complex160", + "complex192", + "complex256", + "complex512", + } + return [i.__name__ for i in extended_types if i.__name__ in extended_names] + + +#: A dictionary mapping type-aliases in `numpy.typing._nbit` to +#: concrete `numpy.typing.NBitBase` subclasses. +_PRECISION_DICT: t.Final = _get_precision_dict() + +#: A list with the names of all extended precision `np.number` subclasses. +_EXTENDED_PRECISION_LIST: t.Final = _get_extended_precision_list() + + +def _hook(ctx: AnalyzeTypeContext) -> Type: + """Replace a type-alias with a concrete ``NBitBase`` subclass.""" + typ, _, api = ctx + name = typ.name.split(".")[-1] + name_new = _PRECISION_DICT[f"numpy.typing._nbit.{name}"] + return api.named_type(name_new) + + +if t.TYPE_CHECKING or MYPY_EX is None: + def _index(iterable: t.Iterable[Statement], id: str) -> int: + """Identify the first ``ImportFrom`` instance the specified `id`.""" + for i, value in enumerate(iterable): + if getattr(value, "id", None) == id: + return i + else: + raise ValueError("Failed to identify a `ImportFrom` instance " + f"with the following id: {id!r}") + + class _NumpyPlugin(Plugin): + """A plugin for assigning platform-specific `numpy.number` precisions.""" + + def get_type_analyze_hook(self, fullname: str) -> t.Optional[_HookFunc]: + """Set the precision of platform-specific `numpy.number` subclasses. + + For example: `numpy.int_`, `numpy.longlong` and `numpy.longdouble`. + """ + if fullname in _PRECISION_DICT: + return _hook + return None + + def get_additional_deps(self, file: MypyFile) -> t.List[t.Tuple[int, str, int]]: + """Import platform-specific extended-precision `numpy.number` subclasses. + + For example: `numpy.float96`, `numpy.float128` and `numpy.complex256`. + """ + ret = [(PRI_MED, file.fullname, -1)] + if file.fullname == "numpy": + # Import ONLY the extended precision types available to the + # platform in question + imports = ImportFrom( + "numpy.typing._extended_precision", 0, + names=[(v, v) for v in _EXTENDED_PRECISION_LIST], + ) + imports.is_top_level = True + + # Replace the much broader extended-precision import + # (defined in `numpy/__init__.pyi`) with a more specific one + for lst in [file.defs, file.imports]: # type: t.List[Statement] + i = _index(lst, "numpy.typing._extended_precision") + lst[i] = imports + return ret + + def plugin(version: str) -> t.Type[_NumpyPlugin]: + """An entry-point for mypy.""" + return _NumpyPlugin + +else: + def plugin(version: str) -> t.Type[_NumpyPlugin]: + """An entry-point for mypy.""" + raise MYPY_EX diff --git a/numpy/typing/tests/data/fail/arithmetic.py b/numpy/typing/tests/data/fail/arithmetic.py index f32eddc4bc8d..02bbffa53ba3 100644 --- a/numpy/typing/tests/data/fail/arithmetic.py +++ b/numpy/typing/tests/data/fail/arithmetic.py @@ -1,9 +1,113 @@ +from typing import List, Any import numpy as np b_ = np.bool_() dt = np.datetime64(0, "D") td = np.timedelta64(0, "D") +AR_b: np.ndarray[Any, np.dtype[np.bool_]] +AR_u: np.ndarray[Any, np.dtype[np.uint32]] +AR_i: np.ndarray[Any, np.dtype[np.int64]] +AR_f: np.ndarray[Any, np.dtype[np.float64]] +AR_c: np.ndarray[Any, np.dtype[np.complex128]] +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] + +ANY: Any + +AR_LIKE_b: List[bool] +AR_LIKE_u: List[np.uint32] +AR_LIKE_i: List[int] +AR_LIKE_f: List[float] +AR_LIKE_c: List[complex] +AR_LIKE_m: List[np.timedelta64] +AR_LIKE_M: List[np.datetime64] + +# Array subtraction + +# NOTE: mypys `NoReturn` errors are, unfortunately, not that great +_1 = AR_b - AR_LIKE_b # E: Need type annotation +_2 = AR_LIKE_b - AR_b # E: Need type annotation + +AR_f - AR_LIKE_m # E: Unsupported operand types +AR_f - AR_LIKE_M # E: Unsupported operand types +AR_c - AR_LIKE_m # E: Unsupported operand types +AR_c - AR_LIKE_M # E: Unsupported operand types + +AR_m - AR_LIKE_f # E: Unsupported operand types +AR_M - AR_LIKE_f # E: Unsupported operand types +AR_m - AR_LIKE_c # E: Unsupported operand types +AR_M - AR_LIKE_c # E: Unsupported operand types + +AR_m - AR_LIKE_M # E: Unsupported operand types +AR_LIKE_m - AR_M # E: Unsupported operand types + +# array floor division + +AR_M // AR_LIKE_b # E: Unsupported operand types +AR_M // AR_LIKE_u # E: Unsupported operand types +AR_M // AR_LIKE_i # E: Unsupported operand types +AR_M // AR_LIKE_f # E: Unsupported operand types +AR_M // AR_LIKE_c # E: Unsupported operand types +AR_M // AR_LIKE_m # E: Unsupported operand types +AR_M // AR_LIKE_M # E: Unsupported operand types + +AR_b // AR_LIKE_M # E: Unsupported operand types +AR_u // AR_LIKE_M # E: Unsupported operand types +AR_i // AR_LIKE_M # E: Unsupported operand types +AR_f // AR_LIKE_M # E: Unsupported operand types +AR_c // AR_LIKE_M # E: Unsupported operand types +AR_m // AR_LIKE_M # E: Unsupported operand types +AR_M // AR_LIKE_M # E: Unsupported operand types + +_3 = AR_m // AR_LIKE_b # E: Need type annotation +AR_m // AR_LIKE_c # E: Unsupported operand types + +AR_b // AR_LIKE_m # E: Unsupported operand types +AR_u // AR_LIKE_m # E: Unsupported operand types +AR_i // AR_LIKE_m # E: Unsupported operand types +AR_f // AR_LIKE_m # E: Unsupported operand types +AR_c // AR_LIKE_m # E: Unsupported operand types + +# Array multiplication + +AR_b *= AR_LIKE_u # E: incompatible type +AR_b *= AR_LIKE_i # E: incompatible type +AR_b *= AR_LIKE_f # E: incompatible type +AR_b *= AR_LIKE_c # E: incompatible type +AR_b *= AR_LIKE_m # E: incompatible type + +AR_u *= AR_LIKE_i # E: incompatible type +AR_u *= AR_LIKE_f # E: incompatible type +AR_u *= AR_LIKE_c # E: incompatible type +AR_u *= AR_LIKE_m # E: incompatible type + +AR_i *= AR_LIKE_f # E: incompatible type +AR_i *= AR_LIKE_c # E: incompatible type +AR_i *= AR_LIKE_m # E: incompatible type + +AR_f *= AR_LIKE_c # E: incompatible type +AR_f *= AR_LIKE_m # E: incompatible type + +# Array power + +AR_b **= AR_LIKE_b # E: incompatible type +AR_b **= AR_LIKE_u # E: incompatible type +AR_b **= AR_LIKE_i # E: incompatible type +AR_b **= AR_LIKE_f # E: incompatible type +AR_b **= AR_LIKE_c # E: incompatible type + +AR_u **= AR_LIKE_i # E: incompatible type +AR_u **= AR_LIKE_f # E: incompatible type +AR_u **= AR_LIKE_c # E: incompatible type + +AR_i **= AR_LIKE_f # E: incompatible type +AR_i **= AR_LIKE_c # E: incompatible type + +AR_f **= AR_LIKE_c # E: incompatible type + +# Scalars + b_ - b_ # E: No overload variant dt + dt # E: Unsupported operand types diff --git a/numpy/typing/tests/data/fail/array_constructors.py b/numpy/typing/tests/data/fail/array_constructors.py index 9cb59fe5f72e..f13fdacb26c9 100644 --- a/numpy/typing/tests/data/fail/array_constructors.py +++ b/numpy/typing/tests/data/fail/array_constructors.py @@ -7,10 +7,10 @@ np.require(a, requirements="TEST") # E: incompatible type np.zeros("test") # E: incompatible type -np.zeros() # E: Too few arguments +np.zeros() # E: Missing positional argument np.ones("test") # E: incompatible type -np.ones() # E: Too few arguments +np.ones() # E: Missing positional argument np.array(0, float, True) # E: Too many positional diff --git a/numpy/typing/tests/data/fail/array_like.py b/numpy/typing/tests/data/fail/array_like.py index a97e72dc73a9..3bbd2906150f 100644 --- a/numpy/typing/tests/data/fail/array_like.py +++ b/numpy/typing/tests/data/fail/array_like.py @@ -11,6 +11,6 @@ class A: x3: ArrayLike = {1: "foo", 2: "bar"} # E: Incompatible types in assignment scalar = np.int64(1) -scalar.__array__(dtype=np.float64) # E: Unexpected keyword argument +scalar.__array__(dtype=np.float64) # E: No overload variant array = np.array([1]) -array.__array__(dtype=np.float64) # E: Unexpected keyword argument +array.__array__(dtype=np.float64) # E: No overload variant diff --git a/numpy/typing/tests/data/fail/arrayprint.py b/numpy/typing/tests/data/fail/arrayprint.py new file mode 100644 index 000000000000..86297a0b24a4 --- /dev/null +++ b/numpy/typing/tests/data/fail/arrayprint.py @@ -0,0 +1,13 @@ +from typing import Callable, Any +import numpy as np + +AR: np.ndarray +func1: Callable[[Any], str] +func2: Callable[[np.integer[Any]], str] + +np.array2string(AR, style=None) # E: Unexpected keyword argument +np.array2string(AR, legacy="1.14") # E: incompatible type +np.array2string(AR, sign="*") # E: incompatible type +np.array2string(AR, floatmode="default") # E: incompatible type +np.array2string(AR, formatter={"A": func1}) # E: incompatible type +np.array2string(AR, formatter={"float": func2}) # E: Incompatible types diff --git a/numpy/typing/tests/data/fail/arrayterator.py b/numpy/typing/tests/data/fail/arrayterator.py new file mode 100644 index 000000000000..c50fb2ec4e52 --- /dev/null +++ b/numpy/typing/tests/data/fail/arrayterator.py @@ -0,0 +1,14 @@ +from typing import Any +import numpy as np + +AR_i8: np.ndarray[Any, np.dtype[np.int64]] +ar_iter = np.lib.Arrayterator(AR_i8) + +np.lib.Arrayterator(np.int64()) # E: incompatible type +ar_iter.shape = (10, 5) # E: is read-only +ar_iter[None] # E: Invalid index type +ar_iter[None, 1] # E: Invalid index type +ar_iter[np.intp()] # E: Invalid index type +ar_iter[np.intp(), ...] # E: Invalid index type +ar_iter[AR_i8] # E: Invalid index type +ar_iter[AR_i8, :] # E: Invalid index type diff --git a/numpy/typing/tests/data/fail/comparisons.py b/numpy/typing/tests/data/fail/comparisons.py new file mode 100644 index 000000000000..cad1c6555de3 --- /dev/null +++ b/numpy/typing/tests/data/fail/comparisons.py @@ -0,0 +1,28 @@ +from typing import Any +import numpy as np + +AR_i: np.ndarray[Any, np.dtype[np.int64]] +AR_f: np.ndarray[Any, np.dtype[np.float64]] +AR_c: np.ndarray[Any, np.dtype[np.complex128]] +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] + +AR_f > AR_m # E: Unsupported operand types +AR_c > AR_m # E: Unsupported operand types + +AR_m > AR_f # E: Unsupported operand types +AR_m > AR_c # E: Unsupported operand types + +AR_i > AR_M # E: Unsupported operand types +AR_f > AR_M # E: Unsupported operand types +AR_m > AR_M # E: Unsupported operand types + +AR_M > AR_i # E: Unsupported operand types +AR_M > AR_f # E: Unsupported operand types +AR_M > AR_m # E: Unsupported operand types + +# Unfortunately `NoReturn` errors are not the most descriptive +_1 = AR_i > str() # E: Need type annotation +_2 = AR_i > bytes() # E: Need type annotation +_3 = str() > AR_M # E: Need type annotation +_4 = bytes() > AR_M # E: Need type annotation diff --git a/numpy/typing/tests/data/fail/datasource.py b/numpy/typing/tests/data/fail/datasource.py new file mode 100644 index 000000000000..345277d45370 --- /dev/null +++ b/numpy/typing/tests/data/fail/datasource.py @@ -0,0 +1,15 @@ +from pathlib import Path +import numpy as np + +path: Path +d1: np.DataSource + +d1.abspath(path) # E: incompatible type +d1.abspath(b"...") # E: incompatible type + +d1.exists(path) # E: incompatible type +d1.exists(b"...") # E: incompatible type + +d1.open(path, "r") # E: incompatible type +d1.open(b"...", encoding="utf8") # E: incompatible type +d1.open(None, newline="/n") # E: incompatible type diff --git a/numpy/typing/tests/data/fail/dtype.py b/numpy/typing/tests/data/fail/dtype.py index 7d4783d8f651..7d419a1d1e5f 100644 --- a/numpy/typing/tests/data/fail/dtype.py +++ b/numpy/typing/tests/data/fail/dtype.py @@ -1,10 +1,16 @@ import numpy as np -class Test: - not_dtype = float +class Test1: + not_dtype = np.dtype(float) -np.dtype(Test()) # E: No overload variant of "dtype" matches + +class Test2: + dtype = float + + +np.dtype(Test1()) # E: No overload variant of "dtype" matches +np.dtype(Test2()) # E: incompatible type np.dtype( # E: No overload variant of "dtype" matches { diff --git a/numpy/typing/tests/data/fail/einsumfunc.py b/numpy/typing/tests/data/fail/einsumfunc.py new file mode 100644 index 000000000000..33722f861199 --- /dev/null +++ b/numpy/typing/tests/data/fail/einsumfunc.py @@ -0,0 +1,15 @@ +from typing import List, Any +import numpy as np + +AR_i: np.ndarray[Any, np.dtype[np.int64]] +AR_f: np.ndarray[Any, np.dtype[np.float64]] +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] +AR_O: np.ndarray[Any, np.dtype[np.object_]] +AR_U: np.ndarray[Any, np.dtype[np.str_]] + +np.einsum("i,i->i", AR_i, AR_m) # E: incompatible type +np.einsum("i,i->i", AR_O, AR_O) # E: incompatible type +np.einsum("i,i->i", AR_f, AR_f, dtype=np.int32) # E: incompatible type +np.einsum("i,i->i", AR_i, AR_i, dtype=np.timedelta64, casting="unsafe") # E: No overload variant +np.einsum("i,i->i", AR_i, AR_i, out=AR_U) # E: Value of type variable "_ArrayType" of "einsum" cannot be +np.einsum("i,i->i", AR_i, AR_i, out=AR_U, casting="unsafe") # E: No overload variant diff --git a/numpy/typing/tests/data/fail/fromnumeric.py b/numpy/typing/tests/data/fail/fromnumeric.py index c9156895dd51..8fafed1b7705 100644 --- a/numpy/typing/tests/data/fail/fromnumeric.py +++ b/numpy/typing/tests/data/fail/fromnumeric.py @@ -7,17 +7,17 @@ a = np.bool_(True) -np.take(a, None) # E: No overload variant of "take" matches argument type -np.take(a, axis=1.0) # E: No overload variant of "take" matches argument type -np.take(A, out=1) # E: No overload variant of "take" matches argument type -np.take(A, mode="bob") # E: No overload variant of "take" matches argument type +np.take(a, None) # E: incompatible type +np.take(a, axis=1.0) # E: incompatible type +np.take(A, out=1) # E: incompatible type +np.take(A, mode="bob") # E: incompatible type np.reshape(a, None) # E: Argument 2 to "reshape" has incompatible type np.reshape(A, 1, order="bob") # E: Argument "order" to "reshape" has incompatible type -np.choose(a, None) # E: No overload variant of "choose" matches argument type -np.choose(a, out=1.0) # E: No overload variant of "choose" matches argument type -np.choose(A, mode="bob") # E: No overload variant of "choose" matches argument type +np.choose(a, None) # E: incompatible type +np.choose(a, out=1.0) # E: incompatible type +np.choose(A, mode="bob") # E: incompatible type np.repeat(a, None) # E: Argument 2 to "repeat" has incompatible type np.repeat(A, 1, axis=1.0) # E: Argument "axis" to "repeat" has incompatible type @@ -38,14 +38,14 @@ A, 0, order=range(5) # E: Argument "order" to "partition" has incompatible type ) -np.argpartition( # E: No overload variant of "argpartition" matches argument type - a, None +np.argpartition( + a, None # E: incompatible type ) -np.argpartition( # E: No overload variant of "argpartition" matches argument type - a, 0, axis="bob" +np.argpartition( + a, 0, axis="bob" # E: incompatible type ) -np.argpartition( # E: No overload variant of "argpartition" matches argument type - A, 0, kind="bob" +np.argpartition( + A, 0, kind="bob" # E: incompatible type ) np.argpartition( A, 0, order=range(5) # E: Argument "order" to "argpartition" has incompatible type @@ -93,62 +93,62 @@ np.clip(a, 1, 2, out=1) # E: No overload variant of "clip" matches argument type np.clip(1, None, None) # E: No overload variant of "clip" matches argument type -np.sum(a, axis=1.0) # E: No overload variant of "sum" matches argument type -np.sum(a, keepdims=1.0) # E: No overload variant of "sum" matches argument type -np.sum(a, initial=[1]) # E: No overload variant of "sum" matches argument type +np.sum(a, axis=1.0) # E: incompatible type +np.sum(a, keepdims=1.0) # E: incompatible type +np.sum(a, initial=[1]) # E: incompatible type -np.all(a, axis=1.0) # E: No overload variant of "all" matches argument type -np.all(a, keepdims=1.0) # E: No overload variant of "all" matches argument type -np.all(a, out=1.0) # E: No overload variant of "all" matches argument type +np.all(a, axis=1.0) # E: No overload variant +np.all(a, keepdims=1.0) # E: No overload variant +np.all(a, out=1.0) # E: No overload variant -np.any(a, axis=1.0) # E: No overload variant of "any" matches argument type -np.any(a, keepdims=1.0) # E: No overload variant of "any" matches argument type -np.any(a, out=1.0) # E: No overload variant of "any" matches argument type +np.any(a, axis=1.0) # E: No overload variant +np.any(a, keepdims=1.0) # E: No overload variant +np.any(a, out=1.0) # E: No overload variant -np.cumsum(a, axis=1.0) # E: Argument "axis" to "cumsum" has incompatible type -np.cumsum(a, dtype=1.0) # E: Argument "dtype" to "cumsum" has incompatible type -np.cumsum(a, out=1.0) # E: Argument "out" to "cumsum" has incompatible type +np.cumsum(a, axis=1.0) # E: incompatible type +np.cumsum(a, dtype=1.0) # E: incompatible type +np.cumsum(a, out=1.0) # E: incompatible type -np.ptp(a, axis=1.0) # E: No overload variant of "ptp" matches argument type -np.ptp(a, keepdims=1.0) # E: No overload variant of "ptp" matches argument type -np.ptp(a, out=1.0) # E: No overload variant of "ptp" matches argument type +np.ptp(a, axis=1.0) # E: incompatible type +np.ptp(a, keepdims=1.0) # E: incompatible type +np.ptp(a, out=1.0) # E: incompatible type -np.amax(a, axis=1.0) # E: No overload variant of "amax" matches argument type -np.amax(a, keepdims=1.0) # E: No overload variant of "amax" matches argument type -np.amax(a, out=1.0) # E: No overload variant of "amax" matches argument type -np.amax(a, initial=[1.0]) # E: No overload variant of "amax" matches argument type -np.amax(a, where=[1.0]) # E: List item 0 has incompatible type +np.amax(a, axis=1.0) # E: incompatible type +np.amax(a, keepdims=1.0) # E: incompatible type +np.amax(a, out=1.0) # E: incompatible type +np.amax(a, initial=[1.0]) # E: incompatible type +np.amax(a, where=[1.0]) # E: incompatible type -np.amin(a, axis=1.0) # E: No overload variant of "amin" matches argument type -np.amin(a, keepdims=1.0) # E: No overload variant of "amin" matches argument type -np.amin(a, out=1.0) # E: No overload variant of "amin" matches argument type -np.amin(a, initial=[1.0]) # E: No overload variant of "amin" matches argument type -np.amin(a, where=[1.0]) # E: List item 0 has incompatible type +np.amin(a, axis=1.0) # E: incompatible type +np.amin(a, keepdims=1.0) # E: incompatible type +np.amin(a, out=1.0) # E: incompatible type +np.amin(a, initial=[1.0]) # E: incompatible type +np.amin(a, where=[1.0]) # E: incompatible type -np.prod(a, axis=1.0) # E: No overload variant of "prod" matches argument type -np.prod(a, out=False) # E: No overload variant of "prod" matches argument type -np.prod(a, keepdims=1.0) # E: No overload variant of "prod" matches argument type -np.prod(a, initial=int) # E: No overload variant of "prod" matches argument type -np.prod(a, where=1.0) # E: No overload variant of "prod" matches argument type +np.prod(a, axis=1.0) # E: incompatible type +np.prod(a, out=False) # E: incompatible type +np.prod(a, keepdims=1.0) # E: incompatible type +np.prod(a, initial=int) # E: incompatible type +np.prod(a, where=1.0) # E: incompatible type np.cumprod(a, axis=1.0) # E: Argument "axis" to "cumprod" has incompatible type np.cumprod(a, out=False) # E: Argument "out" to "cumprod" has incompatible type np.size(a, axis=1.0) # E: Argument "axis" to "size" has incompatible type -np.around(a, decimals=1.0) # E: No overload variant of "around" matches argument type -np.around(a, out=type) # E: No overload variant of "around" matches argument type +np.around(a, decimals=1.0) # E: incompatible type +np.around(a, out=type) # E: incompatible type -np.mean(a, axis=1.0) # E: No overload variant of "mean" matches argument type -np.mean(a, out=False) # E: No overload variant of "mean" matches argument type -np.mean(a, keepdims=1.0) # E: No overload variant of "mean" matches argument type +np.mean(a, axis=1.0) # E: incompatible type +np.mean(a, out=False) # E: incompatible type +np.mean(a, keepdims=1.0) # E: incompatible type -np.std(a, axis=1.0) # E: No overload variant of "std" matches argument type -np.std(a, out=False) # E: No overload variant of "std" matches argument type -np.std(a, ddof='test') # E: No overload variant of "std" matches argument type -np.std(a, keepdims=1.0) # E: No overload variant of "std" matches argument type +np.std(a, axis=1.0) # E: incompatible type +np.std(a, out=False) # E: incompatible type +np.std(a, ddof='test') # E: incompatible type +np.std(a, keepdims=1.0) # E: incompatible type -np.var(a, axis=1.0) # E: No overload variant of "var" matches argument type -np.var(a, out=False) # E: No overload variant of "var" matches argument type -np.var(a, ddof='test') # E: No overload variant of "var" matches argument type -np.var(a, keepdims=1.0) # E: No overload variant of "var" matches argument type +np.var(a, axis=1.0) # E: incompatible type +np.var(a, out=False) # E: incompatible type +np.var(a, ddof='test') # E: incompatible type +np.var(a, keepdims=1.0) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/index_tricks.py b/numpy/typing/tests/data/fail/index_tricks.py new file mode 100644 index 000000000000..c508bf3aeae6 --- /dev/null +++ b/numpy/typing/tests/data/fail/index_tricks.py @@ -0,0 +1,14 @@ +from typing import List +import numpy as np + +AR_LIKE_i: List[int] +AR_LIKE_f: List[float] + +np.unravel_index(AR_LIKE_f, (1, 2, 3)) # E: incompatible type +np.ravel_multi_index(AR_LIKE_i, (1, 2, 3), mode="bob") # E: No overload variant +np.mgrid[1] # E: Invalid index type +np.mgrid[...] # E: Invalid index type +np.ogrid[1] # E: Invalid index type +np.ogrid[...] # E: Invalid index type +np.fill_diagonal(AR_LIKE_f, 2) # E: incompatible type +np.diag_indices(1.0) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/lib_utils.py b/numpy/typing/tests/data/fail/lib_utils.py new file mode 100644 index 000000000000..e16c926aa645 --- /dev/null +++ b/numpy/typing/tests/data/fail/lib_utils.py @@ -0,0 +1,13 @@ +import numpy as np + +np.deprecate(1) # E: No overload variant + +np.deprecate_with_doc(1) # E: incompatible type + +np.byte_bounds(1) # E: incompatible type + +np.who(1) # E: incompatible type + +np.lookfor(None) # E: incompatible type + +np.safe_eval(None) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/lib_version.py b/numpy/typing/tests/data/fail/lib_version.py new file mode 100644 index 000000000000..2758cfe40438 --- /dev/null +++ b/numpy/typing/tests/data/fail/lib_version.py @@ -0,0 +1,6 @@ +from numpy.lib import NumpyVersion + +version: NumpyVersion + +NumpyVersion(b"1.8.0") # E: incompatible type +version >= b"1.8.0" # E: Unsupported operand types diff --git a/numpy/typing/tests/data/fail/modules.py b/numpy/typing/tests/data/fail/modules.py index 5e2d820abc85..7b9309329ac8 100644 --- a/numpy/typing/tests/data/fail/modules.py +++ b/numpy/typing/tests/data/fail/modules.py @@ -8,3 +8,12 @@ np.sys # E: Module has no attribute np.os # E: Module has no attribute np.math # E: Module has no attribute + +# Public sub-modules that are not imported to their parent module by default; +# e.g. one must first execute `import numpy.lib.recfunctions` +np.lib.recfunctions # E: Module has no attribute +np.ma.mrecords # E: Module has no attribute + +np.__NUMPY_SETUP__ # E: Module has no attribute +np.__deprecated_attrs__ # E: Module has no attribute +np.__expired_functions__ # E: Module has no attribute diff --git a/numpy/typing/tests/data/fail/ndarray_misc.py b/numpy/typing/tests/data/fail/ndarray_misc.py index 1e1496bfecca..cf3fedc45f2c 100644 --- a/numpy/typing/tests/data/fail/ndarray_misc.py +++ b/numpy/typing/tests/data/fail/ndarray_misc.py @@ -6,9 +6,20 @@ """ +from typing import Any import numpy as np f8: np.float64 +AR_f8: np.ndarray[Any, np.dtype[np.float64]] +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] +AR_b: np.ndarray[Any, np.dtype[np.bool_]] + +ctypes_obj = AR_f8.ctypes + +reveal_type(ctypes_obj.get_data()) # E: has no attribute +reveal_type(ctypes_obj.get_shape()) # E: has no attribute +reveal_type(ctypes_obj.get_strides()) # E: has no attribute +reveal_type(ctypes_obj.get_as_parameter()) # E: has no attribute f8.argpartition(0) # E: has no attribute f8.diagonal() # E: has no attribute @@ -19,3 +30,8 @@ f8.setfield(2, np.float64) # E: has no attribute f8.sort() # E: has no attribute f8.trace() # E: has no attribute + +AR_M.__int__() # E: Invalid self argument +AR_M.__float__() # E: Invalid self argument +AR_M.__complex__() # E: Invalid self argument +AR_b.__index__() # E: Invalid self argument diff --git a/numpy/typing/tests/data/fail/random.py b/numpy/typing/tests/data/fail/random.py new file mode 100644 index 000000000000..c4d1e3e3e802 --- /dev/null +++ b/numpy/typing/tests/data/fail/random.py @@ -0,0 +1,61 @@ +import numpy as np +from typing import Any, List + +SEED_FLOAT: float = 457.3 +SEED_ARR_FLOAT: np.ndarray[Any, np.dtype[np.float64]] = np.array([1.0, 2, 3, 4]) +SEED_ARRLIKE_FLOAT: List[float] = [1.0, 2.0, 3.0, 4.0] +SEED_SEED_SEQ: np.random.SeedSequence = np.random.SeedSequence(0) +SEED_STR: str = "String seeding not allowed" +# default rng +np.random.default_rng(SEED_FLOAT) # E: incompatible type +np.random.default_rng(SEED_ARR_FLOAT) # E: incompatible type +np.random.default_rng(SEED_ARRLIKE_FLOAT) # E: incompatible type +np.random.default_rng(SEED_STR) # E: incompatible type + +# Seed Sequence +np.random.SeedSequence(SEED_FLOAT) # E: incompatible type +np.random.SeedSequence(SEED_ARR_FLOAT) # E: incompatible type +np.random.SeedSequence(SEED_ARRLIKE_FLOAT) # E: incompatible type +np.random.SeedSequence(SEED_SEED_SEQ) # E: incompatible type +np.random.SeedSequence(SEED_STR) # E: incompatible type + +seed_seq: np.random.bit_generator.SeedSequence = np.random.SeedSequence() +seed_seq.spawn(11.5) # E: incompatible type +seed_seq.generate_state(3.14) # E: incompatible type +seed_seq.generate_state(3, np.uint8) # E: incompatible type +seed_seq.generate_state(3, "uint8") # E: incompatible type +seed_seq.generate_state(3, "u1") # E: incompatible type +seed_seq.generate_state(3, np.uint16) # E: incompatible type +seed_seq.generate_state(3, "uint16") # E: incompatible type +seed_seq.generate_state(3, "u2") # E: incompatible type +seed_seq.generate_state(3, np.int32) # E: incompatible type +seed_seq.generate_state(3, "int32") # E: incompatible type +seed_seq.generate_state(3, "i4") # E: incompatible type + +# Bit Generators +np.random.MT19937(SEED_FLOAT) # E: incompatible type +np.random.MT19937(SEED_ARR_FLOAT) # E: incompatible type +np.random.MT19937(SEED_ARRLIKE_FLOAT) # E: incompatible type +np.random.MT19937(SEED_STR) # E: incompatible type + +np.random.PCG64(SEED_FLOAT) # E: incompatible type +np.random.PCG64(SEED_ARR_FLOAT) # E: incompatible type +np.random.PCG64(SEED_ARRLIKE_FLOAT) # E: incompatible type +np.random.PCG64(SEED_STR) # E: incompatible type + +np.random.Philox(SEED_FLOAT) # E: incompatible type +np.random.Philox(SEED_ARR_FLOAT) # E: incompatible type +np.random.Philox(SEED_ARRLIKE_FLOAT) # E: incompatible type +np.random.Philox(SEED_STR) # E: incompatible type + +np.random.SFC64(SEED_FLOAT) # E: incompatible type +np.random.SFC64(SEED_ARR_FLOAT) # E: incompatible type +np.random.SFC64(SEED_ARRLIKE_FLOAT) # E: incompatible type +np.random.SFC64(SEED_STR) # E: incompatible type + +# Generator +np.random.Generator(None) # E: incompatible type +np.random.Generator(12333283902830213) # E: incompatible type +np.random.Generator("OxFEEDF00D") # E: incompatible type +np.random.Generator([123, 234]) # E: incompatible type +np.random.Generator(np.array([123, 234], dtype="u4")) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/scalars.py b/numpy/typing/tests/data/fail/scalars.py index f097408757f6..0aeff398fc87 100644 --- a/numpy/typing/tests/data/fail/scalars.py +++ b/numpy/typing/tests/data/fail/scalars.py @@ -1,5 +1,6 @@ import numpy as np +f2: np.float16 f8: np.float64 # Construction @@ -74,3 +75,8 @@ def __float__(self): f8.squeeze(axis=1) # E: incompatible type f8.squeeze(axis=(0, 1)) # E: incompatible type f8.transpose(1) # E: incompatible type + +def func(a: np.float32) -> None: ... + +func(f2) # E: incompatible type +func(f8) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/ufunclike.py b/numpy/typing/tests/data/fail/ufunclike.py new file mode 100644 index 000000000000..82a5f3a1d091 --- /dev/null +++ b/numpy/typing/tests/data/fail/ufunclike.py @@ -0,0 +1,21 @@ +from typing import List, Any +import numpy as np + +AR_c: np.ndarray[Any, np.dtype[np.complex128]] +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] +AR_O: np.ndarray[Any, np.dtype[np.object_]] + +np.fix(AR_c) # E: incompatible type +np.fix(AR_m) # E: incompatible type +np.fix(AR_M) # E: incompatible type + +np.isposinf(AR_c) # E: incompatible type +np.isposinf(AR_m) # E: incompatible type +np.isposinf(AR_M) # E: incompatible type +np.isposinf(AR_O) # E: incompatible type + +np.isneginf(AR_c) # E: incompatible type +np.isneginf(AR_m) # E: incompatible type +np.isneginf(AR_M) # E: incompatible type +np.isneginf(AR_O) # E: incompatible type diff --git a/numpy/typing/tests/data/fail/ufuncs.py b/numpy/typing/tests/data/fail/ufuncs.py index 4da9d08baf4c..e827267c6072 100644 --- a/numpy/typing/tests/data/fail/ufuncs.py +++ b/numpy/typing/tests/data/fail/ufuncs.py @@ -1,7 +1,41 @@ import numpy as np +import numpy.typing as npt + +AR_f8: npt.NDArray[np.float64] np.sin.nin + "foo" # E: Unsupported operand types -np.sin(1, foo="bar") # E: Unexpected keyword argument -np.sin(1, extobj=["foo", "foo", "foo"]) # E: incompatible type +np.sin(1, foo="bar") # E: No overload variant + +np.abs(None) # E: No overload variant + +np.add(1, 1, 1) # E: No overload variant +np.add(1, 1, axis=0) # E: No overload variant + +np.matmul(AR_f8, AR_f8, where=True) # E: No overload variant + +np.frexp(AR_f8, out=None) # E: No overload variant +np.frexp(AR_f8, out=AR_f8) # E: No overload variant + +np.absolute.outer() # E: "None" not callable +np.frexp.outer() # E: "None" not callable +np.divmod.outer() # E: "None" not callable +np.matmul.outer() # E: "None" not callable + +np.absolute.reduceat() # E: "None" not callable +np.frexp.reduceat() # E: "None" not callable +np.divmod.reduceat() # E: "None" not callable +np.matmul.reduceat() # E: "None" not callable + +np.absolute.reduce() # E: "None" not callable +np.frexp.reduce() # E: "None" not callable +np.divmod.reduce() # E: "None" not callable +np.matmul.reduce() # E: "None" not callable + +np.absolute.accumulate() # E: "None" not callable +np.frexp.accumulate() # E: "None" not callable +np.divmod.accumulate() # E: "None" not callable +np.matmul.accumulate() # E: "None" not callable -np.abs(None) # E: incompatible type +np.frexp.at() # E: "None" not callable +np.divmod.at() # E: "None" not callable +np.matmul.at() # E: "None" not callable diff --git a/numpy/typing/tests/data/misc/extended_precision.py b/numpy/typing/tests/data/misc/extended_precision.py new file mode 100644 index 000000000000..1e495e4f3cc4 --- /dev/null +++ b/numpy/typing/tests/data/misc/extended_precision.py @@ -0,0 +1,17 @@ +import numpy as np + +reveal_type(np.uint128()) +reveal_type(np.uint256()) + +reveal_type(np.int128()) +reveal_type(np.int256()) + +reveal_type(np.float80()) +reveal_type(np.float96()) +reveal_type(np.float128()) +reveal_type(np.float256()) + +reveal_type(np.complex160()) +reveal_type(np.complex192()) +reveal_type(np.complex256()) +reveal_type(np.complex512()) diff --git a/numpy/typing/tests/data/mypy.ini b/numpy/typing/tests/data/mypy.ini index 91d93588af15..548f762612fc 100644 --- a/numpy/typing/tests/data/mypy.ini +++ b/numpy/typing/tests/data/mypy.ini @@ -1,5 +1,6 @@ [mypy] -mypy_path = ../../.. +plugins = numpy.typing.mypy_plugin +show_absolute_path = True [mypy-numpy] ignore_errors = True diff --git a/numpy/typing/tests/data/pass/arithmetic.py b/numpy/typing/tests/data/pass/arithmetic.py index ffbaf2975332..62bd79004a11 100644 --- a/numpy/typing/tests/data/pass/arithmetic.py +++ b/numpy/typing/tests/data/pass/arithmetic.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from typing import Any import numpy as np c16 = np.complex128(1) @@ -20,8 +23,322 @@ f = float(1) i = int(1) -AR = np.ones(1, dtype=np.float64) -AR.setflags(write=False) + +class Object: + def __array__(self) -> np.ndarray[Any, np.dtype[np.object_]]: + ret = np.empty((), dtype=object) + ret[()] = self + return ret + + def __sub__(self, value: Any) -> Object: + return self + + def __rsub__(self, value: Any) -> Object: + return self + + def __floordiv__(self, value: Any) -> Object: + return self + + def __rfloordiv__(self, value: Any) -> Object: + return self + + def __mul__(self, value: Any) -> Object: + return self + + def __rmul__(self, value: Any) -> Object: + return self + + def __pow__(self, value: Any) -> Object: + return self + + def __rpow__(self, value: Any) -> Object: + return self + + +AR_b: np.ndarray[Any, np.dtype[np.bool_]] = np.array([True]) +AR_u: np.ndarray[Any, np.dtype[np.uint32]] = np.array([1], dtype=np.uint32) +AR_i: np.ndarray[Any, np.dtype[np.int64]] = np.array([1]) +AR_f: np.ndarray[Any, np.dtype[np.float64]] = np.array([1.0]) +AR_c: np.ndarray[Any, np.dtype[np.complex128]] = np.array([1j]) +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] = np.array([np.timedelta64(1, "D")]) +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] = np.array([np.datetime64(1, "D")]) +AR_O: np.ndarray[Any, np.dtype[np.object_]] = np.array([Object()]) + +AR_LIKE_b = [True] +AR_LIKE_u = [np.uint32(1)] +AR_LIKE_i = [1] +AR_LIKE_f = [1.0] +AR_LIKE_c = [1j] +AR_LIKE_m = [np.timedelta64(1, "D")] +AR_LIKE_M = [np.datetime64(1, "D")] +AR_LIKE_O = [Object()] + +# Array subtractions + +AR_b - AR_LIKE_u +AR_b - AR_LIKE_i +AR_b - AR_LIKE_f +AR_b - AR_LIKE_c +AR_b - AR_LIKE_m +AR_b - AR_LIKE_O + +AR_LIKE_u - AR_b +AR_LIKE_i - AR_b +AR_LIKE_f - AR_b +AR_LIKE_c - AR_b +AR_LIKE_m - AR_b +AR_LIKE_M - AR_b +AR_LIKE_O - AR_b + +AR_u - AR_LIKE_b +AR_u - AR_LIKE_u +AR_u - AR_LIKE_i +AR_u - AR_LIKE_f +AR_u - AR_LIKE_c +AR_u - AR_LIKE_m +AR_u - AR_LIKE_O + +AR_LIKE_b - AR_u +AR_LIKE_u - AR_u +AR_LIKE_i - AR_u +AR_LIKE_f - AR_u +AR_LIKE_c - AR_u +AR_LIKE_m - AR_u +AR_LIKE_M - AR_u +AR_LIKE_O - AR_u + +AR_i - AR_LIKE_b +AR_i - AR_LIKE_u +AR_i - AR_LIKE_i +AR_i - AR_LIKE_f +AR_i - AR_LIKE_c +AR_i - AR_LIKE_m +AR_i - AR_LIKE_O + +AR_LIKE_b - AR_i +AR_LIKE_u - AR_i +AR_LIKE_i - AR_i +AR_LIKE_f - AR_i +AR_LIKE_c - AR_i +AR_LIKE_m - AR_i +AR_LIKE_M - AR_i +AR_LIKE_O - AR_i + +AR_f - AR_LIKE_b +AR_f - AR_LIKE_u +AR_f - AR_LIKE_i +AR_f - AR_LIKE_f +AR_f - AR_LIKE_c +AR_f - AR_LIKE_O + +AR_LIKE_b - AR_f +AR_LIKE_u - AR_f +AR_LIKE_i - AR_f +AR_LIKE_f - AR_f +AR_LIKE_c - AR_f +AR_LIKE_O - AR_f + +AR_c - AR_LIKE_b +AR_c - AR_LIKE_u +AR_c - AR_LIKE_i +AR_c - AR_LIKE_f +AR_c - AR_LIKE_c +AR_c - AR_LIKE_O + +AR_LIKE_b - AR_c +AR_LIKE_u - AR_c +AR_LIKE_i - AR_c +AR_LIKE_f - AR_c +AR_LIKE_c - AR_c +AR_LIKE_O - AR_c + +AR_m - AR_LIKE_b +AR_m - AR_LIKE_u +AR_m - AR_LIKE_i +AR_m - AR_LIKE_m + +AR_LIKE_b - AR_m +AR_LIKE_u - AR_m +AR_LIKE_i - AR_m +AR_LIKE_m - AR_m +AR_LIKE_M - AR_m + +AR_M - AR_LIKE_b +AR_M - AR_LIKE_u +AR_M - AR_LIKE_i +AR_M - AR_LIKE_m +AR_M - AR_LIKE_M + +AR_LIKE_M - AR_M + +AR_O - AR_LIKE_b +AR_O - AR_LIKE_u +AR_O - AR_LIKE_i +AR_O - AR_LIKE_f +AR_O - AR_LIKE_c +AR_O - AR_LIKE_O + +AR_LIKE_b - AR_O +AR_LIKE_u - AR_O +AR_LIKE_i - AR_O +AR_LIKE_f - AR_O +AR_LIKE_c - AR_O +AR_LIKE_O - AR_O + +# Array floor division + +AR_b // AR_LIKE_b +AR_b // AR_LIKE_u +AR_b // AR_LIKE_i +AR_b // AR_LIKE_f +AR_b // AR_LIKE_c +AR_b // AR_LIKE_O + +AR_LIKE_b // AR_b +AR_LIKE_u // AR_b +AR_LIKE_i // AR_b +AR_LIKE_f // AR_b +AR_LIKE_c // AR_b +AR_LIKE_O // AR_b + +AR_u // AR_LIKE_b +AR_u // AR_LIKE_u +AR_u // AR_LIKE_i +AR_u // AR_LIKE_f +AR_u // AR_LIKE_c +AR_u // AR_LIKE_O + +AR_LIKE_b // AR_u +AR_LIKE_u // AR_u +AR_LIKE_i // AR_u +AR_LIKE_f // AR_u +AR_LIKE_c // AR_u +AR_LIKE_m // AR_u +AR_LIKE_O // AR_u + +AR_i // AR_LIKE_b +AR_i // AR_LIKE_u +AR_i // AR_LIKE_i +AR_i // AR_LIKE_f +AR_i // AR_LIKE_c +AR_i // AR_LIKE_O + +AR_LIKE_b // AR_i +AR_LIKE_u // AR_i +AR_LIKE_i // AR_i +AR_LIKE_f // AR_i +AR_LIKE_c // AR_i +AR_LIKE_m // AR_i +AR_LIKE_O // AR_i + +AR_f // AR_LIKE_b +AR_f // AR_LIKE_u +AR_f // AR_LIKE_i +AR_f // AR_LIKE_f +AR_f // AR_LIKE_c +AR_f // AR_LIKE_O + +AR_LIKE_b // AR_f +AR_LIKE_u // AR_f +AR_LIKE_i // AR_f +AR_LIKE_f // AR_f +AR_LIKE_c // AR_f +AR_LIKE_m // AR_f +AR_LIKE_O // AR_f + +AR_c // AR_LIKE_b +AR_c // AR_LIKE_u +AR_c // AR_LIKE_i +AR_c // AR_LIKE_f +AR_c // AR_LIKE_c + +AR_LIKE_b // AR_c +AR_LIKE_u // AR_c +AR_LIKE_i // AR_c +AR_LIKE_f // AR_c +AR_LIKE_c // AR_c +AR_LIKE_O // AR_c + +AR_m // AR_LIKE_u +AR_m // AR_LIKE_i +AR_m // AR_LIKE_f +AR_m // AR_LIKE_m + +AR_LIKE_m // AR_m + +AR_O // AR_LIKE_b +AR_O // AR_LIKE_u +AR_O // AR_LIKE_i +AR_O // AR_LIKE_f +AR_O // AR_LIKE_c +AR_O // AR_LIKE_O + +AR_LIKE_b // AR_O +AR_LIKE_u // AR_O +AR_LIKE_i // AR_O +AR_LIKE_f // AR_O +AR_LIKE_O // AR_O + +# Inplace multiplication + +AR_b *= AR_LIKE_b + +AR_u *= AR_LIKE_b +AR_u *= AR_LIKE_u + +AR_i *= AR_LIKE_b +AR_i *= AR_LIKE_u +AR_i *= AR_LIKE_i + +AR_f *= AR_LIKE_b +AR_f *= AR_LIKE_u +AR_f *= AR_LIKE_i +AR_f *= AR_LIKE_f + +AR_c *= AR_LIKE_b +AR_c *= AR_LIKE_u +AR_c *= AR_LIKE_i +AR_c *= AR_LIKE_f +AR_c *= AR_LIKE_c + +AR_m *= AR_LIKE_b +AR_m *= AR_LIKE_u +AR_m *= AR_LIKE_i +AR_m *= AR_LIKE_f + +AR_O *= AR_LIKE_b +AR_O *= AR_LIKE_u +AR_O *= AR_LIKE_i +AR_O *= AR_LIKE_f +AR_O *= AR_LIKE_c +AR_O *= AR_LIKE_O + +# Inplace power + +AR_u **= AR_LIKE_b +AR_u **= AR_LIKE_u + +AR_i **= AR_LIKE_b +AR_i **= AR_LIKE_u +AR_i **= AR_LIKE_i + +AR_f **= AR_LIKE_b +AR_f **= AR_LIKE_u +AR_f **= AR_LIKE_i +AR_f **= AR_LIKE_f + +AR_c **= AR_LIKE_b +AR_c **= AR_LIKE_u +AR_c **= AR_LIKE_i +AR_c **= AR_LIKE_f +AR_c **= AR_LIKE_c + +AR_O **= AR_LIKE_b +AR_O **= AR_LIKE_u +AR_O **= AR_LIKE_i +AR_O **= AR_LIKE_f +AR_O **= AR_LIKE_c +AR_O **= AR_LIKE_O # unary ops @@ -34,7 +351,7 @@ -u8 -u4 -td --AR +-AR_f +c16 +c8 @@ -45,7 +362,7 @@ +u8 +u4 +td -+AR ++AR_f abs(c16) abs(c8) @@ -57,7 +374,7 @@ abs(u4) abs(td) abs(b_) -abs(AR) +abs(AR_f) # Time structures @@ -129,7 +446,7 @@ c16 + c c16 + f c16 + i -c16 + AR +c16 + AR_f c16 + c16 f8 + c16 @@ -142,7 +459,7 @@ c + c16 f + c16 i + c16 -AR + c16 +AR_f + c16 c8 + c16 c8 + f8 @@ -155,7 +472,7 @@ c8 + c c8 + f c8 + i -c8 + AR +c8 + AR_f c16 + c8 f8 + c8 @@ -168,7 +485,7 @@ c + c8 f + c8 i + c8 -AR + c8 +AR_f + c8 # Float @@ -181,7 +498,7 @@ f8 + c f8 + f f8 + i -f8 + AR +f8 + AR_f f8 + f8 i8 + f8 @@ -192,7 +509,7 @@ c + f8 f + f8 i + f8 -AR + f8 +AR_f + f8 f4 + f8 f4 + i8 @@ -203,7 +520,7 @@ f4 + c f4 + f f4 + i -f4 + AR +f4 + AR_f f8 + f4 i8 + f4 @@ -214,7 +531,7 @@ c + f4 f + f4 i + f4 -AR + f4 +AR_f + f4 # Int @@ -227,7 +544,7 @@ i8 + c i8 + f i8 + i -i8 + AR +i8 + AR_f u8 + u8 u8 + i4 @@ -237,7 +554,7 @@ u8 + c u8 + f u8 + i -u8 + AR +u8 + AR_f i8 + i8 u8 + i8 @@ -248,7 +565,7 @@ c + i8 f + i8 i + i8 -AR + i8 +AR_f + i8 u8 + u8 i4 + u8 @@ -258,14 +575,14 @@ c + u8 f + u8 i + u8 -AR + u8 +AR_f + u8 i4 + i8 i4 + i4 i4 + i i4 + b_ i4 + b -i4 + AR +i4 + AR_f u4 + i8 u4 + i4 @@ -274,14 +591,14 @@ u4 + i u4 + b_ u4 + b -u4 + AR +u4 + AR_f i8 + i4 i4 + i4 i + i4 b_ + i4 b + i4 -AR + i4 +AR_f + i4 i8 + u4 i4 + u4 @@ -290,4 +607,4 @@ b_ + u4 b + u4 i + u4 -AR + u4 +AR_f + u4 diff --git a/numpy/typing/tests/data/pass/array_constructors.py b/numpy/typing/tests/data/pass/array_constructors.py index 63208f139c39..206f70a15bb1 100644 --- a/numpy/typing/tests/data/pass/array_constructors.py +++ b/numpy/typing/tests/data/pass/array_constructors.py @@ -1,11 +1,20 @@ +import sys from typing import List, Any import numpy as np + class Index: def __index__(self) -> int: return 0 -class SubClass(np.ndarray): ... + +class SubClass(np.ndarray): + pass + + +def func(i: int, j: int, **kwargs: Any) -> SubClass: + return B + i8 = np.int64(1) @@ -14,8 +23,9 @@ class SubClass(np.ndarray): ... B_stack = np.array([[1], [1]]).view(SubClass) C = [1] -def func(i: int, j: int, **kwargs: Any) -> SubClass: - return B +if sys.version_info >= (3, 8): + np.ndarray(Index()) + np.ndarray([Index()]) np.array(1, dtype=float) np.array(1, copy=False) diff --git a/numpy/typing/tests/data/pass/array_like.py b/numpy/typing/tests/data/pass/array_like.py index f85724267448..e16d196b6024 100644 --- a/numpy/typing/tests/data/pass/array_like.py +++ b/numpy/typing/tests/data/pass/array_like.py @@ -1,7 +1,7 @@ from typing import Any, List, Optional import numpy as np -from numpy.typing import ArrayLike, DTypeLike, _SupportsArray +from numpy.typing import ArrayLike, _SupportsArray x1: ArrayLike = True x2: ArrayLike = 5 @@ -18,20 +18,20 @@ class A: - def __array__(self, dtype: DTypeLike = None) -> np.ndarray: + def __array__(self, dtype: Optional[np.dtype] = None) -> np.ndarray: return np.array([1, 2, 3]) x13: ArrayLike = A() scalar: _SupportsArray = np.int64(1) -scalar.__array__(np.float64) +scalar.__array__() array: _SupportsArray = np.array(1) -array.__array__(np.float64) +array.__array__() a: _SupportsArray = A() -a.__array__(np.int64) -a.__array__(dtype=np.int64) +a.__array__() +a.__array__() # Escape hatch for when you mean to make something like an object # array. diff --git a/numpy/typing/tests/data/pass/arrayprint.py b/numpy/typing/tests/data/pass/arrayprint.py new file mode 100644 index 000000000000..6c704c755570 --- /dev/null +++ b/numpy/typing/tests/data/pass/arrayprint.py @@ -0,0 +1,37 @@ +import numpy as np + +AR = np.arange(10) +AR.setflags(write=False) + +with np.printoptions(): + np.set_printoptions( + precision=1, + threshold=2, + edgeitems=3, + linewidth=4, + suppress=False, + nanstr="Bob", + infstr="Bill", + formatter={}, + sign="+", + floatmode="unique", + ) + np.get_printoptions() + str(AR) + + np.array2string( + AR, + max_line_width=5, + precision=2, + suppress_small=True, + separator=";", + prefix="test", + threshold=5, + floatmode="fixed", + suffix="?", + legacy="1.13", + ) + np.format_float_scientific(1, precision=5) + np.format_float_positional(1, trim="k") + np.array_repr(AR) + np.array_str(AR) diff --git a/numpy/typing/tests/data/pass/arrayterator.py b/numpy/typing/tests/data/pass/arrayterator.py new file mode 100644 index 000000000000..572be5e2fe29 --- /dev/null +++ b/numpy/typing/tests/data/pass/arrayterator.py @@ -0,0 +1,27 @@ + +from __future__ import annotations + +from typing import Any +import numpy as np + +AR_i8: np.ndarray[Any, np.dtype[np.int_]] = np.arange(10) +ar_iter = np.lib.Arrayterator(AR_i8) + +ar_iter.var +ar_iter.buf_size +ar_iter.start +ar_iter.stop +ar_iter.step +ar_iter.shape +ar_iter.flat + +ar_iter.__array__() + +for i in ar_iter: + pass + +ar_iter[0] +ar_iter[...] +ar_iter[:] +ar_iter[0, 0, 0] +ar_iter[..., 0, :] diff --git a/numpy/typing/tests/data/pass/comparisons.py b/numpy/typing/tests/data/pass/comparisons.py new file mode 100644 index 000000000000..ce41de43596e --- /dev/null +++ b/numpy/typing/tests/data/pass/comparisons.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +from typing import Any +import numpy as np + +c16 = np.complex128() +f8 = np.float64() +i8 = np.int64() +u8 = np.uint64() + +c8 = np.complex64() +f4 = np.float32() +i4 = np.int32() +u4 = np.uint32() + +dt = np.datetime64(0, "D") +td = np.timedelta64(0, "D") + +b_ = np.bool_() + +b = bool() +c = complex() +f = float() +i = int() + +SEQ = (0, 1, 2, 3, 4) + +AR_b: np.ndarray[Any, np.dtype[np.bool_]] = np.array([True]) +AR_u: np.ndarray[Any, np.dtype[np.uint32]] = np.array([1], dtype=np.uint32) +AR_i: np.ndarray[Any, np.dtype[np.int_]] = np.array([1]) +AR_f: np.ndarray[Any, np.dtype[np.float_]] = np.array([1.0]) +AR_c: np.ndarray[Any, np.dtype[np.complex_]] = np.array([1.0j]) +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] = np.array([np.timedelta64("1")]) +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] = np.array([np.datetime64("1")]) +AR_O: np.ndarray[Any, np.dtype[np.object_]] = np.array([1], dtype=object) + +# Arrays + +AR_b > AR_b +AR_b > AR_u +AR_b > AR_i +AR_b > AR_f +AR_b > AR_c + +AR_u > AR_b +AR_u > AR_u +AR_u > AR_i +AR_u > AR_f +AR_u > AR_c + +AR_i > AR_b +AR_i > AR_u +AR_i > AR_i +AR_i > AR_f +AR_i > AR_c + +AR_f > AR_b +AR_f > AR_u +AR_f > AR_i +AR_f > AR_f +AR_f > AR_c + +AR_c > AR_b +AR_c > AR_u +AR_c > AR_i +AR_c > AR_f +AR_c > AR_c + +AR_m > AR_b +AR_m > AR_u +AR_m > AR_i +AR_b > AR_m +AR_u > AR_m +AR_i > AR_m + +AR_M > AR_M + +AR_O > AR_O +1 > AR_O +AR_O > 1 + +# Time structures + +dt > dt + +td > td +td > i +td > i4 +td > i8 +td > AR_i +td > SEQ + +# boolean + +b_ > b +b_ > b_ +b_ > i +b_ > i8 +b_ > i4 +b_ > u8 +b_ > u4 +b_ > f +b_ > f8 +b_ > f4 +b_ > c +b_ > c16 +b_ > c8 +b_ > AR_i +b_ > SEQ + +# Complex + +c16 > c16 +c16 > f8 +c16 > i8 +c16 > c8 +c16 > f4 +c16 > i4 +c16 > b_ +c16 > b +c16 > c +c16 > f +c16 > i +c16 > AR_i +c16 > SEQ + +c16 > c16 +f8 > c16 +i8 > c16 +c8 > c16 +f4 > c16 +i4 > c16 +b_ > c16 +b > c16 +c > c16 +f > c16 +i > c16 +AR_i > c16 +SEQ > c16 + +c8 > c16 +c8 > f8 +c8 > i8 +c8 > c8 +c8 > f4 +c8 > i4 +c8 > b_ +c8 > b +c8 > c +c8 > f +c8 > i +c8 > AR_i +c8 > SEQ + +c16 > c8 +f8 > c8 +i8 > c8 +c8 > c8 +f4 > c8 +i4 > c8 +b_ > c8 +b > c8 +c > c8 +f > c8 +i > c8 +AR_i > c8 +SEQ > c8 + +# Float + +f8 > f8 +f8 > i8 +f8 > f4 +f8 > i4 +f8 > b_ +f8 > b +f8 > c +f8 > f +f8 > i +f8 > AR_i +f8 > SEQ + +f8 > f8 +i8 > f8 +f4 > f8 +i4 > f8 +b_ > f8 +b > f8 +c > f8 +f > f8 +i > f8 +AR_i > f8 +SEQ > f8 + +f4 > f8 +f4 > i8 +f4 > f4 +f4 > i4 +f4 > b_ +f4 > b +f4 > c +f4 > f +f4 > i +f4 > AR_i +f4 > SEQ + +f8 > f4 +i8 > f4 +f4 > f4 +i4 > f4 +b_ > f4 +b > f4 +c > f4 +f > f4 +i > f4 +AR_i > f4 +SEQ > f4 + +# Int + +i8 > i8 +i8 > u8 +i8 > i4 +i8 > u4 +i8 > b_ +i8 > b +i8 > c +i8 > f +i8 > i +i8 > AR_i +i8 > SEQ + +u8 > u8 +u8 > i4 +u8 > u4 +u8 > b_ +u8 > b +u8 > c +u8 > f +u8 > i +u8 > AR_i +u8 > SEQ + +i8 > i8 +u8 > i8 +i4 > i8 +u4 > i8 +b_ > i8 +b > i8 +c > i8 +f > i8 +i > i8 +AR_i > i8 +SEQ > i8 + +u8 > u8 +i4 > u8 +u4 > u8 +b_ > u8 +b > u8 +c > u8 +f > u8 +i > u8 +AR_i > u8 +SEQ > u8 + +i4 > i8 +i4 > i4 +i4 > i +i4 > b_ +i4 > b +i4 > AR_i +i4 > SEQ + +u4 > i8 +u4 > i4 +u4 > u8 +u4 > u4 +u4 > i +u4 > b_ +u4 > b +u4 > AR_i +u4 > SEQ + +i8 > i4 +i4 > i4 +i > i4 +b_ > i4 +b > i4 +AR_i > i4 +SEQ > i4 + +i8 > u4 +i4 > u4 +u8 > u4 +u4 > u4 +b_ > u4 +b > u4 +i > u4 +AR_i > u4 +SEQ > u4 diff --git a/numpy/typing/tests/data/pass/dtype.py b/numpy/typing/tests/data/pass/dtype.py index cbae8c078551..e849cfdd4e11 100644 --- a/numpy/typing/tests/data/pass/dtype.py +++ b/numpy/typing/tests/data/pass/dtype.py @@ -1,5 +1,8 @@ import numpy as np +dtype_obj = np.dtype(np.str_) +void_dtype_obj = np.dtype([("f0", np.float64), ("f1", np.float32)]) + np.dtype(dtype=np.int64) np.dtype(int) np.dtype("int") @@ -29,7 +32,26 @@ class Test: - dtype = float + dtype = np.dtype(float) np.dtype(Test()) + +# Methods and attributes +dtype_obj.base +dtype_obj.subdtype +dtype_obj.newbyteorder() +dtype_obj.type +dtype_obj.name +dtype_obj.names + +dtype_obj * 0 +dtype_obj * 2 + +0 * dtype_obj +2 * dtype_obj + +void_dtype_obj["f0"] +void_dtype_obj[0] +void_dtype_obj[["f0", "f1"]] +void_dtype_obj[["f0"]] diff --git a/numpy/typing/tests/data/pass/einsumfunc.py b/numpy/typing/tests/data/pass/einsumfunc.py new file mode 100644 index 000000000000..a2a39fb1c4c1 --- /dev/null +++ b/numpy/typing/tests/data/pass/einsumfunc.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import List, Any + +import numpy as np + +AR_LIKE_b = [True, True, True] +AR_LIKE_u = [np.uint32(1), np.uint32(2), np.uint32(3)] +AR_LIKE_i = [1, 2, 3] +AR_LIKE_f = [1.0, 2.0, 3.0] +AR_LIKE_c = [1j, 2j, 3j] +AR_LIKE_U = ["1", "2", "3"] + +OUT_f: np.ndarray[Any, np.dtype[np.float64]] = np.empty(3, dtype=np.float64) +OUT_c: np.ndarray[Any, np.dtype[np.complex128]] = np.empty(3, dtype=np.complex128) + +np.einsum("i,i->i", AR_LIKE_b, AR_LIKE_b) +np.einsum("i,i->i", AR_LIKE_u, AR_LIKE_u) +np.einsum("i,i->i", AR_LIKE_i, AR_LIKE_i) +np.einsum("i,i->i", AR_LIKE_f, AR_LIKE_f) +np.einsum("i,i->i", AR_LIKE_c, AR_LIKE_c) +np.einsum("i,i->i", AR_LIKE_b, AR_LIKE_i) +np.einsum("i,i,i,i->i", AR_LIKE_b, AR_LIKE_u, AR_LIKE_i, AR_LIKE_c) + +np.einsum("i,i->i", AR_LIKE_f, AR_LIKE_f, dtype="c16") +np.einsum("i,i->i", AR_LIKE_U, AR_LIKE_U, dtype=bool, casting="unsafe") +np.einsum("i,i->i", AR_LIKE_f, AR_LIKE_f, out=OUT_c) +np.einsum("i,i->i", AR_LIKE_U, AR_LIKE_U, dtype=int, casting="unsafe", out=OUT_f) + +np.einsum_path("i,i->i", AR_LIKE_b, AR_LIKE_b) +np.einsum_path("i,i->i", AR_LIKE_u, AR_LIKE_u) +np.einsum_path("i,i->i", AR_LIKE_i, AR_LIKE_i) +np.einsum_path("i,i->i", AR_LIKE_f, AR_LIKE_f) +np.einsum_path("i,i->i", AR_LIKE_c, AR_LIKE_c) +np.einsum_path("i,i->i", AR_LIKE_b, AR_LIKE_i) +np.einsum_path("i,i,i,i->i", AR_LIKE_b, AR_LIKE_u, AR_LIKE_i, AR_LIKE_c) diff --git a/numpy/typing/tests/data/pass/flatiter.py b/numpy/typing/tests/data/pass/flatiter.py index c0219eb2beda..63c839af4b23 100644 --- a/numpy/typing/tests/data/pass/flatiter.py +++ b/numpy/typing/tests/data/pass/flatiter.py @@ -12,3 +12,5 @@ a[[0, 1, 2]] a[...] a[:] +a.__array__() +a.__array__(np.dtype(np.float64)) diff --git a/numpy/typing/tests/data/pass/index_tricks.py b/numpy/typing/tests/data/pass/index_tricks.py new file mode 100644 index 000000000000..4c4c1195990a --- /dev/null +++ b/numpy/typing/tests/data/pass/index_tricks.py @@ -0,0 +1,64 @@ +from __future__ import annotations +from typing import Any +import numpy as np + +AR_LIKE_b = [[True, True], [True, True]] +AR_LIKE_i = [[1, 2], [3, 4]] +AR_LIKE_f = [[1.0, 2.0], [3.0, 4.0]] +AR_LIKE_U = [["1", "2"], ["3", "4"]] + +AR_i8: np.ndarray[Any, np.dtype[np.int64]] = np.array(AR_LIKE_i, dtype=np.int64) + +np.ndenumerate(AR_i8) +np.ndenumerate(AR_LIKE_f) +np.ndenumerate(AR_LIKE_U) + +np.ndenumerate(AR_i8).iter +np.ndenumerate(AR_LIKE_f).iter +np.ndenumerate(AR_LIKE_U).iter + +next(np.ndenumerate(AR_i8)) +next(np.ndenumerate(AR_LIKE_f)) +next(np.ndenumerate(AR_LIKE_U)) + +iter(np.ndenumerate(AR_i8)) +iter(np.ndenumerate(AR_LIKE_f)) +iter(np.ndenumerate(AR_LIKE_U)) + +iter(np.ndindex(1, 2, 3)) +next(np.ndindex(1, 2, 3)) + +np.unravel_index([22, 41, 37], (7, 6)) +np.unravel_index([31, 41, 13], (7, 6), order='F') +np.unravel_index(1621, (6, 7, 8, 9)) + +np.ravel_multi_index(AR_LIKE_i, (7, 6)) +np.ravel_multi_index(AR_LIKE_i, (7, 6), order='F') +np.ravel_multi_index(AR_LIKE_i, (4, 6), mode='clip') +np.ravel_multi_index(AR_LIKE_i, (4, 4), mode=('clip', 'wrap')) +np.ravel_multi_index((3, 1, 4, 1), (6, 7, 8, 9)) + +np.mgrid[1:1:2] +np.mgrid[1:1:2, None:10] + +np.ogrid[1:1:2] +np.ogrid[1:1:2, None:10] + +np.index_exp[0:1] +np.index_exp[0:1, None:3] +np.index_exp[0, 0:1, ..., [0, 1, 3]] + +np.s_[0:1] +np.s_[0:1, None:3] +np.s_[0, 0:1, ..., [0, 1, 3]] + +np.ix_(AR_LIKE_b[0]) +np.ix_(AR_LIKE_i[0], AR_LIKE_f[0]) +np.ix_(AR_i8[0]) + +np.fill_diagonal(AR_i8, 5) + +np.diag_indices(4) +np.diag_indices(2, 3) + +np.diag_indices_from(AR_i8) diff --git a/numpy/typing/tests/data/pass/lib_utils.py b/numpy/typing/tests/data/pass/lib_utils.py new file mode 100644 index 000000000000..c602923d9104 --- /dev/null +++ b/numpy/typing/tests/data/pass/lib_utils.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from io import StringIO +from typing import Any + +import numpy as np + +FILE = StringIO() +AR: np.ndarray[Any, np.dtype[np.float64]] = np.arange(10).astype(np.float64) + +def func(a: int) -> bool: ... + +np.deprecate(func) +np.deprecate() + +np.deprecate_with_doc("test") +np.deprecate_with_doc(None) + +np.byte_bounds(AR) +np.byte_bounds(np.float64()) + +np.info(1, output=FILE) + +np.source(np.interp, output=FILE) + +np.lookfor("binary representation", output=FILE) diff --git a/numpy/typing/tests/data/pass/lib_version.py b/numpy/typing/tests/data/pass/lib_version.py new file mode 100644 index 000000000000..f3825eca5247 --- /dev/null +++ b/numpy/typing/tests/data/pass/lib_version.py @@ -0,0 +1,18 @@ +from numpy.lib import NumpyVersion + +version = NumpyVersion("1.8.0") + +version.vstring +version.version +version.major +version.minor +version.bugfix +version.pre_release +version.is_devversion + +version == version +version != version +version < "1.8.0" +version <= version +version > version +version >= "1.8.0" diff --git a/numpy/typing/tests/data/pass/modules.py b/numpy/typing/tests/data/pass/modules.py new file mode 100644 index 000000000000..9261874d565a --- /dev/null +++ b/numpy/typing/tests/data/pass/modules.py @@ -0,0 +1,43 @@ +import numpy as np +from numpy import f2py + +np.char +np.ctypeslib +np.emath +np.fft +np.lib +np.linalg +np.ma +np.matrixlib +np.polynomial +np.random +np.rec +np.testing +np.version + +np.lib.format +np.lib.mixins +np.lib.scimath +np.lib.stride_tricks +np.ma.extras +np.polynomial.chebyshev +np.polynomial.hermite +np.polynomial.hermite_e +np.polynomial.laguerre +np.polynomial.legendre +np.polynomial.polynomial + +np.__path__ +np.__version__ +np.__git_version__ + +np.__all__ +np.char.__all__ +np.ctypeslib.__all__ +np.emath.__all__ +np.lib.__all__ +np.ma.__all__ +np.random.__all__ +np.rec.__all__ +np.testing.__all__ +f2py.__all__ diff --git a/numpy/typing/tests/data/pass/multiarray.py b/numpy/typing/tests/data/pass/multiarray.py new file mode 100644 index 000000000000..e2b5d16a04a2 --- /dev/null +++ b/numpy/typing/tests/data/pass/multiarray.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Any +import numpy as np + +AR_f8: np.ndarray[Any, np.dtype[np.float64]] = np.array([1.0]) +AR_i8: np.ndarray[Any, np.dtype[np.int_]] = np.array([1]) + +b_f8 = np.broadcast(AR_f8) +b_i8_f8_f8 = np.broadcast(AR_i8, AR_f8, AR_f8) + +next(b_f8) +next(b_i8_f8_f8) + +b_f8.reset() +b_i8_f8_f8.reset() + +b_f8.index +b_i8_f8_f8.index + +b_f8.iters +b_i8_f8_f8.iters + +b_f8.nd +b_i8_f8_f8.nd + +b_f8.ndim +b_i8_f8_f8.ndim + +b_f8.numiter +b_i8_f8_f8.numiter + +b_f8.shape +b_i8_f8_f8.shape + +b_f8.size +b_i8_f8_f8.size diff --git a/numpy/typing/tests/data/pass/ndarray_misc.py b/numpy/typing/tests/data/pass/ndarray_misc.py index 6c6f5d50b986..62024603c949 100644 --- a/numpy/typing/tests/data/pass/ndarray_misc.py +++ b/numpy/typing/tests/data/pass/ndarray_misc.py @@ -6,17 +6,21 @@ """ -from typing import cast +from __future__ import annotations + +import operator +from typing import cast, Any + import numpy as np class SubClass(np.ndarray): ... i4 = np.int32(1) -A = np.array([[1]], dtype=np.int32) +A: np.ndarray[Any, np.dtype[np.int32]] = np.array([[1]], dtype=np.int32) B0 = np.empty((), dtype=np.int32).view(SubClass) B1 = np.empty((1,), dtype=np.int32).view(SubClass) B2 = np.empty((1, 1), dtype=np.int32).view(SubClass) -C = np.array([0, 1, 2], dtype=np.int32) +C: np.ndarray[Any, np.dtype[np.int32]] = np.array([0, 1, 2], dtype=np.int32) D = np.empty(3).view(SubClass) i4.all() @@ -157,3 +161,25 @@ class SubClass(np.ndarray): ... void = cast(np.void, np.array(1, dtype=[("f", np.float64)]).take(0)) void.setfield(10, np.float64) + +A.item(0) +C.item(0) + +A.ravel() +C.ravel() + +A.flatten() +C.flatten() + +A.reshape(1) +C.reshape(3) + +int(np.array(1.0, dtype=np.float64)) +int(np.array("1", dtype=np.str_)) + +float(np.array(1.0, dtype=np.float64)) +float(np.array("1", dtype=np.str_)) + +complex(np.array(1.0, dtype=np.float64)) + +operator.index(np.array(1, dtype=np.int64)) diff --git a/numpy/typing/tests/data/pass/numerictypes.py b/numpy/typing/tests/data/pass/numerictypes.py index 4f205cabcb0d..5af0d171ca04 100644 --- a/numpy/typing/tests/data/pass/numerictypes.py +++ b/numpy/typing/tests/data/pass/numerictypes.py @@ -27,3 +27,21 @@ np.find_common_type((), (np.int64, np.float32, complex)) np.find_common_type([np.int64, np.float32], []) np.find_common_type([np.float32], [np.int64, np.float64]) + +np.cast[int] +np.cast["i8"] +np.cast[np.int64] + +np.nbytes[int] +np.nbytes["i8"] +np.nbytes[np.int64] + +np.ScalarType +np.ScalarType[0] +np.ScalarType[4] +np.ScalarType[9] +np.ScalarType[11] + +np.typecodes["Character"] +np.typecodes["Complex"] +np.typecodes["All"] diff --git a/numpy/typing/tests/data/pass/random.py b/numpy/typing/tests/data/pass/random.py new file mode 100644 index 000000000000..05bd62112ff2 --- /dev/null +++ b/numpy/typing/tests/data/pass/random.py @@ -0,0 +1,1497 @@ +from __future__ import annotations + +from typing import Any, List, Dict + +import numpy as np + +SEED_NONE = None +SEED_INT = 4579435749574957634658964293569 +SEED_ARR: np.ndarray[Any, np.dtype[np.int64]] = np.array([1, 2, 3, 4], dtype=np.int64) +SEED_ARRLIKE: List[int] = [1, 2, 3, 4] +SEED_SEED_SEQ: np.random.SeedSequence = np.random.SeedSequence(0) +SEED_MT19937: np.random.MT19937 = np.random.MT19937(0) +SEED_PCG64: np.random.PCG64 = np.random.PCG64(0) +SEED_PHILOX: np.random.Philox = np.random.Philox(0) +SEED_SFC64: np.random.SFC64 = np.random.SFC64(0) + +# default rng +np.random.default_rng() +np.random.default_rng(SEED_NONE) +np.random.default_rng(SEED_INT) +np.random.default_rng(SEED_ARR) +np.random.default_rng(SEED_ARRLIKE) +np.random.default_rng(SEED_SEED_SEQ) +np.random.default_rng(SEED_MT19937) +np.random.default_rng(SEED_PCG64) +np.random.default_rng(SEED_PHILOX) +np.random.default_rng(SEED_SFC64) + +# Seed Sequence +np.random.SeedSequence(SEED_NONE) +np.random.SeedSequence(SEED_INT) +np.random.SeedSequence(SEED_ARR) +np.random.SeedSequence(SEED_ARRLIKE) + +# Bit Generators +np.random.MT19937(SEED_NONE) +np.random.MT19937(SEED_INT) +np.random.MT19937(SEED_ARR) +np.random.MT19937(SEED_ARRLIKE) +np.random.MT19937(SEED_SEED_SEQ) + +np.random.PCG64(SEED_NONE) +np.random.PCG64(SEED_INT) +np.random.PCG64(SEED_ARR) +np.random.PCG64(SEED_ARRLIKE) +np.random.PCG64(SEED_SEED_SEQ) + +np.random.Philox(SEED_NONE) +np.random.Philox(SEED_INT) +np.random.Philox(SEED_ARR) +np.random.Philox(SEED_ARRLIKE) +np.random.Philox(SEED_SEED_SEQ) + +np.random.SFC64(SEED_NONE) +np.random.SFC64(SEED_INT) +np.random.SFC64(SEED_ARR) +np.random.SFC64(SEED_ARRLIKE) +np.random.SFC64(SEED_SEED_SEQ) + +seed_seq: np.random.bit_generator.SeedSequence = np.random.SeedSequence(SEED_NONE) +seed_seq.spawn(10) +seed_seq.generate_state(3) +seed_seq.generate_state(3, "u4") +seed_seq.generate_state(3, "uint32") +seed_seq.generate_state(3, "u8") +seed_seq.generate_state(3, "uint64") +seed_seq.generate_state(3, np.uint32) +seed_seq.generate_state(3, np.uint64) + + +def_gen: np.random.Generator = np.random.default_rng() + +D_arr_0p1: np.ndarray[Any, np.dtype[np.float64]] = np.array([0.1]) +D_arr_0p5: np.ndarray[Any, np.dtype[np.float64]] = np.array([0.5]) +D_arr_0p9: np.ndarray[Any, np.dtype[np.float64]] = np.array([0.9]) +D_arr_1p5: np.ndarray[Any, np.dtype[np.float64]] = np.array([1.5]) +I_arr_10: np.ndarray[Any, np.dtype[np.int_]] = np.array([10], dtype=np.int_) +I_arr_20: np.ndarray[Any, np.dtype[np.int_]] = np.array([20], dtype=np.int_) +D_arr_like_0p1: List[float] = [0.1] +D_arr_like_0p5: List[float] = [0.5] +D_arr_like_0p9: List[float] = [0.9] +D_arr_like_1p5: List[float] = [1.5] +I_arr_like_10: List[int] = [10] +I_arr_like_20: List[int] = [20] +D_2D_like: List[List[float]] = [[1, 2], [2, 3], [3, 4], [4, 5.1]] +D_2D: np.ndarray[Any, np.dtype[np.float64]] = np.array(D_2D_like) + +S_out: np.ndarray[Any, np.dtype[np.float32]] = np.empty(1, dtype=np.float32) +D_out: np.ndarray[Any, np.dtype[np.float64]] = np.empty(1) + +def_gen.standard_normal() +def_gen.standard_normal(dtype=np.float32) +def_gen.standard_normal(dtype="float32") +def_gen.standard_normal(dtype="double") +def_gen.standard_normal(dtype=np.float64) +def_gen.standard_normal(size=None) +def_gen.standard_normal(size=1) +def_gen.standard_normal(size=1, dtype=np.float32) +def_gen.standard_normal(size=1, dtype="f4") +def_gen.standard_normal(size=1, dtype="float32", out=S_out) +def_gen.standard_normal(dtype=np.float32, out=S_out) +def_gen.standard_normal(size=1, dtype=np.float64) +def_gen.standard_normal(size=1, dtype="float64") +def_gen.standard_normal(size=1, dtype="f8") +def_gen.standard_normal(out=D_out) +def_gen.standard_normal(size=1, dtype="float64") +def_gen.standard_normal(size=1, dtype="float64", out=D_out) + +def_gen.random() +def_gen.random(dtype=np.float32) +def_gen.random(dtype="float32") +def_gen.random(dtype="double") +def_gen.random(dtype=np.float64) +def_gen.random(size=None) +def_gen.random(size=1) +def_gen.random(size=1, dtype=np.float32) +def_gen.random(size=1, dtype="f4") +def_gen.random(size=1, dtype="float32", out=S_out) +def_gen.random(dtype=np.float32, out=S_out) +def_gen.random(size=1, dtype=np.float64) +def_gen.random(size=1, dtype="float64") +def_gen.random(size=1, dtype="f8") +def_gen.random(out=D_out) +def_gen.random(size=1, dtype="float64") +def_gen.random(size=1, dtype="float64", out=D_out) + +def_gen.standard_cauchy() +def_gen.standard_cauchy(size=None) +def_gen.standard_cauchy(size=1) + +def_gen.standard_exponential() +def_gen.standard_exponential(method="inv") +def_gen.standard_exponential(dtype=np.float32) +def_gen.standard_exponential(dtype="float32") +def_gen.standard_exponential(dtype="double") +def_gen.standard_exponential(dtype=np.float64) +def_gen.standard_exponential(size=None) +def_gen.standard_exponential(size=None, method="inv") +def_gen.standard_exponential(size=1, method="inv") +def_gen.standard_exponential(size=1, dtype=np.float32) +def_gen.standard_exponential(size=1, dtype="f4", method="inv") +def_gen.standard_exponential(size=1, dtype="float32", out=S_out) +def_gen.standard_exponential(dtype=np.float32, out=S_out) +def_gen.standard_exponential(size=1, dtype=np.float64, method="inv") +def_gen.standard_exponential(size=1, dtype="float64") +def_gen.standard_exponential(size=1, dtype="f8") +def_gen.standard_exponential(out=D_out) +def_gen.standard_exponential(size=1, dtype="float64") +def_gen.standard_exponential(size=1, dtype="float64", out=D_out) + +def_gen.zipf(1.5) +def_gen.zipf(1.5, size=None) +def_gen.zipf(1.5, size=1) +def_gen.zipf(D_arr_1p5) +def_gen.zipf(D_arr_1p5, size=1) +def_gen.zipf(D_arr_like_1p5) +def_gen.zipf(D_arr_like_1p5, size=1) + +def_gen.weibull(0.5) +def_gen.weibull(0.5, size=None) +def_gen.weibull(0.5, size=1) +def_gen.weibull(D_arr_0p5) +def_gen.weibull(D_arr_0p5, size=1) +def_gen.weibull(D_arr_like_0p5) +def_gen.weibull(D_arr_like_0p5, size=1) + +def_gen.standard_t(0.5) +def_gen.standard_t(0.5, size=None) +def_gen.standard_t(0.5, size=1) +def_gen.standard_t(D_arr_0p5) +def_gen.standard_t(D_arr_0p5, size=1) +def_gen.standard_t(D_arr_like_0p5) +def_gen.standard_t(D_arr_like_0p5, size=1) + +def_gen.poisson(0.5) +def_gen.poisson(0.5, size=None) +def_gen.poisson(0.5, size=1) +def_gen.poisson(D_arr_0p5) +def_gen.poisson(D_arr_0p5, size=1) +def_gen.poisson(D_arr_like_0p5) +def_gen.poisson(D_arr_like_0p5, size=1) + +def_gen.power(0.5) +def_gen.power(0.5, size=None) +def_gen.power(0.5, size=1) +def_gen.power(D_arr_0p5) +def_gen.power(D_arr_0p5, size=1) +def_gen.power(D_arr_like_0p5) +def_gen.power(D_arr_like_0p5, size=1) + +def_gen.pareto(0.5) +def_gen.pareto(0.5, size=None) +def_gen.pareto(0.5, size=1) +def_gen.pareto(D_arr_0p5) +def_gen.pareto(D_arr_0p5, size=1) +def_gen.pareto(D_arr_like_0p5) +def_gen.pareto(D_arr_like_0p5, size=1) + +def_gen.chisquare(0.5) +def_gen.chisquare(0.5, size=None) +def_gen.chisquare(0.5, size=1) +def_gen.chisquare(D_arr_0p5) +def_gen.chisquare(D_arr_0p5, size=1) +def_gen.chisquare(D_arr_like_0p5) +def_gen.chisquare(D_arr_like_0p5, size=1) + +def_gen.exponential(0.5) +def_gen.exponential(0.5, size=None) +def_gen.exponential(0.5, size=1) +def_gen.exponential(D_arr_0p5) +def_gen.exponential(D_arr_0p5, size=1) +def_gen.exponential(D_arr_like_0p5) +def_gen.exponential(D_arr_like_0p5, size=1) + +def_gen.geometric(0.5) +def_gen.geometric(0.5, size=None) +def_gen.geometric(0.5, size=1) +def_gen.geometric(D_arr_0p5) +def_gen.geometric(D_arr_0p5, size=1) +def_gen.geometric(D_arr_like_0p5) +def_gen.geometric(D_arr_like_0p5, size=1) + +def_gen.logseries(0.5) +def_gen.logseries(0.5, size=None) +def_gen.logseries(0.5, size=1) +def_gen.logseries(D_arr_0p5) +def_gen.logseries(D_arr_0p5, size=1) +def_gen.logseries(D_arr_like_0p5) +def_gen.logseries(D_arr_like_0p5, size=1) + +def_gen.rayleigh(0.5) +def_gen.rayleigh(0.5, size=None) +def_gen.rayleigh(0.5, size=1) +def_gen.rayleigh(D_arr_0p5) +def_gen.rayleigh(D_arr_0p5, size=1) +def_gen.rayleigh(D_arr_like_0p5) +def_gen.rayleigh(D_arr_like_0p5, size=1) + +def_gen.standard_gamma(0.5) +def_gen.standard_gamma(0.5, size=None) +def_gen.standard_gamma(0.5, dtype="float32") +def_gen.standard_gamma(0.5, size=None, dtype="float32") +def_gen.standard_gamma(0.5, size=1) +def_gen.standard_gamma(D_arr_0p5) +def_gen.standard_gamma(D_arr_0p5, dtype="f4") +def_gen.standard_gamma(0.5, size=1, dtype="float32", out=S_out) +def_gen.standard_gamma(D_arr_0p5, dtype=np.float32, out=S_out) +def_gen.standard_gamma(D_arr_0p5, size=1) +def_gen.standard_gamma(D_arr_like_0p5) +def_gen.standard_gamma(D_arr_like_0p5, size=1) +def_gen.standard_gamma(0.5, out=D_out) +def_gen.standard_gamma(D_arr_like_0p5, out=D_out) +def_gen.standard_gamma(D_arr_like_0p5, size=1) +def_gen.standard_gamma(D_arr_like_0p5, size=1, out=D_out, dtype=np.float64) + +def_gen.vonmises(0.5, 0.5) +def_gen.vonmises(0.5, 0.5, size=None) +def_gen.vonmises(0.5, 0.5, size=1) +def_gen.vonmises(D_arr_0p5, 0.5) +def_gen.vonmises(0.5, D_arr_0p5) +def_gen.vonmises(D_arr_0p5, 0.5, size=1) +def_gen.vonmises(0.5, D_arr_0p5, size=1) +def_gen.vonmises(D_arr_like_0p5, 0.5) +def_gen.vonmises(0.5, D_arr_like_0p5) +def_gen.vonmises(D_arr_0p5, D_arr_0p5) +def_gen.vonmises(D_arr_like_0p5, D_arr_like_0p5) +def_gen.vonmises(D_arr_0p5, D_arr_0p5, size=1) +def_gen.vonmises(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.wald(0.5, 0.5) +def_gen.wald(0.5, 0.5, size=None) +def_gen.wald(0.5, 0.5, size=1) +def_gen.wald(D_arr_0p5, 0.5) +def_gen.wald(0.5, D_arr_0p5) +def_gen.wald(D_arr_0p5, 0.5, size=1) +def_gen.wald(0.5, D_arr_0p5, size=1) +def_gen.wald(D_arr_like_0p5, 0.5) +def_gen.wald(0.5, D_arr_like_0p5) +def_gen.wald(D_arr_0p5, D_arr_0p5) +def_gen.wald(D_arr_like_0p5, D_arr_like_0p5) +def_gen.wald(D_arr_0p5, D_arr_0p5, size=1) +def_gen.wald(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.uniform(0.5, 0.5) +def_gen.uniform(0.5, 0.5, size=None) +def_gen.uniform(0.5, 0.5, size=1) +def_gen.uniform(D_arr_0p5, 0.5) +def_gen.uniform(0.5, D_arr_0p5) +def_gen.uniform(D_arr_0p5, 0.5, size=1) +def_gen.uniform(0.5, D_arr_0p5, size=1) +def_gen.uniform(D_arr_like_0p5, 0.5) +def_gen.uniform(0.5, D_arr_like_0p5) +def_gen.uniform(D_arr_0p5, D_arr_0p5) +def_gen.uniform(D_arr_like_0p5, D_arr_like_0p5) +def_gen.uniform(D_arr_0p5, D_arr_0p5, size=1) +def_gen.uniform(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.beta(0.5, 0.5) +def_gen.beta(0.5, 0.5, size=None) +def_gen.beta(0.5, 0.5, size=1) +def_gen.beta(D_arr_0p5, 0.5) +def_gen.beta(0.5, D_arr_0p5) +def_gen.beta(D_arr_0p5, 0.5, size=1) +def_gen.beta(0.5, D_arr_0p5, size=1) +def_gen.beta(D_arr_like_0p5, 0.5) +def_gen.beta(0.5, D_arr_like_0p5) +def_gen.beta(D_arr_0p5, D_arr_0p5) +def_gen.beta(D_arr_like_0p5, D_arr_like_0p5) +def_gen.beta(D_arr_0p5, D_arr_0p5, size=1) +def_gen.beta(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.f(0.5, 0.5) +def_gen.f(0.5, 0.5, size=None) +def_gen.f(0.5, 0.5, size=1) +def_gen.f(D_arr_0p5, 0.5) +def_gen.f(0.5, D_arr_0p5) +def_gen.f(D_arr_0p5, 0.5, size=1) +def_gen.f(0.5, D_arr_0p5, size=1) +def_gen.f(D_arr_like_0p5, 0.5) +def_gen.f(0.5, D_arr_like_0p5) +def_gen.f(D_arr_0p5, D_arr_0p5) +def_gen.f(D_arr_like_0p5, D_arr_like_0p5) +def_gen.f(D_arr_0p5, D_arr_0p5, size=1) +def_gen.f(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.gamma(0.5, 0.5) +def_gen.gamma(0.5, 0.5, size=None) +def_gen.gamma(0.5, 0.5, size=1) +def_gen.gamma(D_arr_0p5, 0.5) +def_gen.gamma(0.5, D_arr_0p5) +def_gen.gamma(D_arr_0p5, 0.5, size=1) +def_gen.gamma(0.5, D_arr_0p5, size=1) +def_gen.gamma(D_arr_like_0p5, 0.5) +def_gen.gamma(0.5, D_arr_like_0p5) +def_gen.gamma(D_arr_0p5, D_arr_0p5) +def_gen.gamma(D_arr_like_0p5, D_arr_like_0p5) +def_gen.gamma(D_arr_0p5, D_arr_0p5, size=1) +def_gen.gamma(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.gumbel(0.5, 0.5) +def_gen.gumbel(0.5, 0.5, size=None) +def_gen.gumbel(0.5, 0.5, size=1) +def_gen.gumbel(D_arr_0p5, 0.5) +def_gen.gumbel(0.5, D_arr_0p5) +def_gen.gumbel(D_arr_0p5, 0.5, size=1) +def_gen.gumbel(0.5, D_arr_0p5, size=1) +def_gen.gumbel(D_arr_like_0p5, 0.5) +def_gen.gumbel(0.5, D_arr_like_0p5) +def_gen.gumbel(D_arr_0p5, D_arr_0p5) +def_gen.gumbel(D_arr_like_0p5, D_arr_like_0p5) +def_gen.gumbel(D_arr_0p5, D_arr_0p5, size=1) +def_gen.gumbel(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.laplace(0.5, 0.5) +def_gen.laplace(0.5, 0.5, size=None) +def_gen.laplace(0.5, 0.5, size=1) +def_gen.laplace(D_arr_0p5, 0.5) +def_gen.laplace(0.5, D_arr_0p5) +def_gen.laplace(D_arr_0p5, 0.5, size=1) +def_gen.laplace(0.5, D_arr_0p5, size=1) +def_gen.laplace(D_arr_like_0p5, 0.5) +def_gen.laplace(0.5, D_arr_like_0p5) +def_gen.laplace(D_arr_0p5, D_arr_0p5) +def_gen.laplace(D_arr_like_0p5, D_arr_like_0p5) +def_gen.laplace(D_arr_0p5, D_arr_0p5, size=1) +def_gen.laplace(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.logistic(0.5, 0.5) +def_gen.logistic(0.5, 0.5, size=None) +def_gen.logistic(0.5, 0.5, size=1) +def_gen.logistic(D_arr_0p5, 0.5) +def_gen.logistic(0.5, D_arr_0p5) +def_gen.logistic(D_arr_0p5, 0.5, size=1) +def_gen.logistic(0.5, D_arr_0p5, size=1) +def_gen.logistic(D_arr_like_0p5, 0.5) +def_gen.logistic(0.5, D_arr_like_0p5) +def_gen.logistic(D_arr_0p5, D_arr_0p5) +def_gen.logistic(D_arr_like_0p5, D_arr_like_0p5) +def_gen.logistic(D_arr_0p5, D_arr_0p5, size=1) +def_gen.logistic(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.lognormal(0.5, 0.5) +def_gen.lognormal(0.5, 0.5, size=None) +def_gen.lognormal(0.5, 0.5, size=1) +def_gen.lognormal(D_arr_0p5, 0.5) +def_gen.lognormal(0.5, D_arr_0p5) +def_gen.lognormal(D_arr_0p5, 0.5, size=1) +def_gen.lognormal(0.5, D_arr_0p5, size=1) +def_gen.lognormal(D_arr_like_0p5, 0.5) +def_gen.lognormal(0.5, D_arr_like_0p5) +def_gen.lognormal(D_arr_0p5, D_arr_0p5) +def_gen.lognormal(D_arr_like_0p5, D_arr_like_0p5) +def_gen.lognormal(D_arr_0p5, D_arr_0p5, size=1) +def_gen.lognormal(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.noncentral_chisquare(0.5, 0.5) +def_gen.noncentral_chisquare(0.5, 0.5, size=None) +def_gen.noncentral_chisquare(0.5, 0.5, size=1) +def_gen.noncentral_chisquare(D_arr_0p5, 0.5) +def_gen.noncentral_chisquare(0.5, D_arr_0p5) +def_gen.noncentral_chisquare(D_arr_0p5, 0.5, size=1) +def_gen.noncentral_chisquare(0.5, D_arr_0p5, size=1) +def_gen.noncentral_chisquare(D_arr_like_0p5, 0.5) +def_gen.noncentral_chisquare(0.5, D_arr_like_0p5) +def_gen.noncentral_chisquare(D_arr_0p5, D_arr_0p5) +def_gen.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5) +def_gen.noncentral_chisquare(D_arr_0p5, D_arr_0p5, size=1) +def_gen.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.normal(0.5, 0.5) +def_gen.normal(0.5, 0.5, size=None) +def_gen.normal(0.5, 0.5, size=1) +def_gen.normal(D_arr_0p5, 0.5) +def_gen.normal(0.5, D_arr_0p5) +def_gen.normal(D_arr_0p5, 0.5, size=1) +def_gen.normal(0.5, D_arr_0p5, size=1) +def_gen.normal(D_arr_like_0p5, 0.5) +def_gen.normal(0.5, D_arr_like_0p5) +def_gen.normal(D_arr_0p5, D_arr_0p5) +def_gen.normal(D_arr_like_0p5, D_arr_like_0p5) +def_gen.normal(D_arr_0p5, D_arr_0p5, size=1) +def_gen.normal(D_arr_like_0p5, D_arr_like_0p5, size=1) + +def_gen.triangular(0.1, 0.5, 0.9) +def_gen.triangular(0.1, 0.5, 0.9, size=None) +def_gen.triangular(0.1, 0.5, 0.9, size=1) +def_gen.triangular(D_arr_0p1, 0.5, 0.9) +def_gen.triangular(0.1, D_arr_0p5, 0.9) +def_gen.triangular(D_arr_0p1, 0.5, D_arr_like_0p9, size=1) +def_gen.triangular(0.1, D_arr_0p5, 0.9, size=1) +def_gen.triangular(D_arr_like_0p1, 0.5, D_arr_0p9) +def_gen.triangular(0.5, D_arr_like_0p5, 0.9) +def_gen.triangular(D_arr_0p1, D_arr_0p5, 0.9) +def_gen.triangular(D_arr_like_0p1, D_arr_like_0p5, 0.9) +def_gen.triangular(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1) +def_gen.triangular(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1) + +def_gen.noncentral_f(0.1, 0.5, 0.9) +def_gen.noncentral_f(0.1, 0.5, 0.9, size=None) +def_gen.noncentral_f(0.1, 0.5, 0.9, size=1) +def_gen.noncentral_f(D_arr_0p1, 0.5, 0.9) +def_gen.noncentral_f(0.1, D_arr_0p5, 0.9) +def_gen.noncentral_f(D_arr_0p1, 0.5, D_arr_like_0p9, size=1) +def_gen.noncentral_f(0.1, D_arr_0p5, 0.9, size=1) +def_gen.noncentral_f(D_arr_like_0p1, 0.5, D_arr_0p9) +def_gen.noncentral_f(0.5, D_arr_like_0p5, 0.9) +def_gen.noncentral_f(D_arr_0p1, D_arr_0p5, 0.9) +def_gen.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, 0.9) +def_gen.noncentral_f(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1) +def_gen.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1) + +def_gen.binomial(10, 0.5) +def_gen.binomial(10, 0.5, size=None) +def_gen.binomial(10, 0.5, size=1) +def_gen.binomial(I_arr_10, 0.5) +def_gen.binomial(10, D_arr_0p5) +def_gen.binomial(I_arr_10, 0.5, size=1) +def_gen.binomial(10, D_arr_0p5, size=1) +def_gen.binomial(I_arr_like_10, 0.5) +def_gen.binomial(10, D_arr_like_0p5) +def_gen.binomial(I_arr_10, D_arr_0p5) +def_gen.binomial(I_arr_like_10, D_arr_like_0p5) +def_gen.binomial(I_arr_10, D_arr_0p5, size=1) +def_gen.binomial(I_arr_like_10, D_arr_like_0p5, size=1) + +def_gen.negative_binomial(10, 0.5) +def_gen.negative_binomial(10, 0.5, size=None) +def_gen.negative_binomial(10, 0.5, size=1) +def_gen.negative_binomial(I_arr_10, 0.5) +def_gen.negative_binomial(10, D_arr_0p5) +def_gen.negative_binomial(I_arr_10, 0.5, size=1) +def_gen.negative_binomial(10, D_arr_0p5, size=1) +def_gen.negative_binomial(I_arr_like_10, 0.5) +def_gen.negative_binomial(10, D_arr_like_0p5) +def_gen.negative_binomial(I_arr_10, D_arr_0p5) +def_gen.negative_binomial(I_arr_like_10, D_arr_like_0p5) +def_gen.negative_binomial(I_arr_10, D_arr_0p5, size=1) +def_gen.negative_binomial(I_arr_like_10, D_arr_like_0p5, size=1) + +def_gen.hypergeometric(20, 20, 10) +def_gen.hypergeometric(20, 20, 10, size=None) +def_gen.hypergeometric(20, 20, 10, size=1) +def_gen.hypergeometric(I_arr_20, 20, 10) +def_gen.hypergeometric(20, I_arr_20, 10) +def_gen.hypergeometric(I_arr_20, 20, I_arr_like_10, size=1) +def_gen.hypergeometric(20, I_arr_20, 10, size=1) +def_gen.hypergeometric(I_arr_like_20, 20, I_arr_10) +def_gen.hypergeometric(20, I_arr_like_20, 10) +def_gen.hypergeometric(I_arr_20, I_arr_20, 10) +def_gen.hypergeometric(I_arr_like_20, I_arr_like_20, 10) +def_gen.hypergeometric(I_arr_20, I_arr_20, I_arr_10, size=1) +def_gen.hypergeometric(I_arr_like_20, I_arr_like_20, I_arr_like_10, size=1) + +I_int64_100: np.ndarray[Any, np.dtype[np.int64]] = np.array([100], dtype=np.int64) + +def_gen.integers(0, 100) +def_gen.integers(100) +def_gen.integers([100]) +def_gen.integers(0, [100]) + +I_bool_low: np.ndarray[Any, np.dtype[np.bool_]] = np.array([0], dtype=np.bool_) +I_bool_low_like: List[int] = [0] +I_bool_high_open: np.ndarray[Any, np.dtype[np.bool_]] = np.array([1], dtype=np.bool_) +I_bool_high_closed: np.ndarray[Any, np.dtype[np.bool_]] = np.array([1], dtype=np.bool_) + +def_gen.integers(2, dtype=bool) +def_gen.integers(0, 2, dtype=bool) +def_gen.integers(1, dtype=bool, endpoint=True) +def_gen.integers(0, 1, dtype=bool, endpoint=True) +def_gen.integers(I_bool_low_like, 1, dtype=bool, endpoint=True) +def_gen.integers(I_bool_high_open, dtype=bool) +def_gen.integers(I_bool_low, I_bool_high_open, dtype=bool) +def_gen.integers(0, I_bool_high_open, dtype=bool) +def_gen.integers(I_bool_high_closed, dtype=bool, endpoint=True) +def_gen.integers(I_bool_low, I_bool_high_closed, dtype=bool, endpoint=True) +def_gen.integers(0, I_bool_high_closed, dtype=bool, endpoint=True) + +def_gen.integers(2, dtype=np.bool_) +def_gen.integers(0, 2, dtype=np.bool_) +def_gen.integers(1, dtype=np.bool_, endpoint=True) +def_gen.integers(0, 1, dtype=np.bool_, endpoint=True) +def_gen.integers(I_bool_low_like, 1, dtype=np.bool_, endpoint=True) +def_gen.integers(I_bool_high_open, dtype=np.bool_) +def_gen.integers(I_bool_low, I_bool_high_open, dtype=np.bool_) +def_gen.integers(0, I_bool_high_open, dtype=np.bool_) +def_gen.integers(I_bool_high_closed, dtype=np.bool_, endpoint=True) +def_gen.integers(I_bool_low, I_bool_high_closed, dtype=np.bool_, endpoint=True) +def_gen.integers(0, I_bool_high_closed, dtype=np.bool_, endpoint=True) + +I_u1_low: np.ndarray[Any, np.dtype[np.uint8]] = np.array([0], dtype=np.uint8) +I_u1_low_like: List[int] = [0] +I_u1_high_open: np.ndarray[Any, np.dtype[np.uint8]] = np.array([255], dtype=np.uint8) +I_u1_high_closed: np.ndarray[Any, np.dtype[np.uint8]] = np.array([255], dtype=np.uint8) + +def_gen.integers(256, dtype="u1") +def_gen.integers(0, 256, dtype="u1") +def_gen.integers(255, dtype="u1", endpoint=True) +def_gen.integers(0, 255, dtype="u1", endpoint=True) +def_gen.integers(I_u1_low_like, 255, dtype="u1", endpoint=True) +def_gen.integers(I_u1_high_open, dtype="u1") +def_gen.integers(I_u1_low, I_u1_high_open, dtype="u1") +def_gen.integers(0, I_u1_high_open, dtype="u1") +def_gen.integers(I_u1_high_closed, dtype="u1", endpoint=True) +def_gen.integers(I_u1_low, I_u1_high_closed, dtype="u1", endpoint=True) +def_gen.integers(0, I_u1_high_closed, dtype="u1", endpoint=True) + +def_gen.integers(256, dtype="uint8") +def_gen.integers(0, 256, dtype="uint8") +def_gen.integers(255, dtype="uint8", endpoint=True) +def_gen.integers(0, 255, dtype="uint8", endpoint=True) +def_gen.integers(I_u1_low_like, 255, dtype="uint8", endpoint=True) +def_gen.integers(I_u1_high_open, dtype="uint8") +def_gen.integers(I_u1_low, I_u1_high_open, dtype="uint8") +def_gen.integers(0, I_u1_high_open, dtype="uint8") +def_gen.integers(I_u1_high_closed, dtype="uint8", endpoint=True) +def_gen.integers(I_u1_low, I_u1_high_closed, dtype="uint8", endpoint=True) +def_gen.integers(0, I_u1_high_closed, dtype="uint8", endpoint=True) + +def_gen.integers(256, dtype=np.uint8) +def_gen.integers(0, 256, dtype=np.uint8) +def_gen.integers(255, dtype=np.uint8, endpoint=True) +def_gen.integers(0, 255, dtype=np.uint8, endpoint=True) +def_gen.integers(I_u1_low_like, 255, dtype=np.uint8, endpoint=True) +def_gen.integers(I_u1_high_open, dtype=np.uint8) +def_gen.integers(I_u1_low, I_u1_high_open, dtype=np.uint8) +def_gen.integers(0, I_u1_high_open, dtype=np.uint8) +def_gen.integers(I_u1_high_closed, dtype=np.uint8, endpoint=True) +def_gen.integers(I_u1_low, I_u1_high_closed, dtype=np.uint8, endpoint=True) +def_gen.integers(0, I_u1_high_closed, dtype=np.uint8, endpoint=True) + +I_u2_low: np.ndarray[Any, np.dtype[np.uint16]] = np.array([0], dtype=np.uint16) +I_u2_low_like: List[int] = [0] +I_u2_high_open: np.ndarray[Any, np.dtype[np.uint16]] = np.array([65535], dtype=np.uint16) +I_u2_high_closed: np.ndarray[Any, np.dtype[np.uint16]] = np.array([65535], dtype=np.uint16) + +def_gen.integers(65536, dtype="u2") +def_gen.integers(0, 65536, dtype="u2") +def_gen.integers(65535, dtype="u2", endpoint=True) +def_gen.integers(0, 65535, dtype="u2", endpoint=True) +def_gen.integers(I_u2_low_like, 65535, dtype="u2", endpoint=True) +def_gen.integers(I_u2_high_open, dtype="u2") +def_gen.integers(I_u2_low, I_u2_high_open, dtype="u2") +def_gen.integers(0, I_u2_high_open, dtype="u2") +def_gen.integers(I_u2_high_closed, dtype="u2", endpoint=True) +def_gen.integers(I_u2_low, I_u2_high_closed, dtype="u2", endpoint=True) +def_gen.integers(0, I_u2_high_closed, dtype="u2", endpoint=True) + +def_gen.integers(65536, dtype="uint16") +def_gen.integers(0, 65536, dtype="uint16") +def_gen.integers(65535, dtype="uint16", endpoint=True) +def_gen.integers(0, 65535, dtype="uint16", endpoint=True) +def_gen.integers(I_u2_low_like, 65535, dtype="uint16", endpoint=True) +def_gen.integers(I_u2_high_open, dtype="uint16") +def_gen.integers(I_u2_low, I_u2_high_open, dtype="uint16") +def_gen.integers(0, I_u2_high_open, dtype="uint16") +def_gen.integers(I_u2_high_closed, dtype="uint16", endpoint=True) +def_gen.integers(I_u2_low, I_u2_high_closed, dtype="uint16", endpoint=True) +def_gen.integers(0, I_u2_high_closed, dtype="uint16", endpoint=True) + +def_gen.integers(65536, dtype=np.uint16) +def_gen.integers(0, 65536, dtype=np.uint16) +def_gen.integers(65535, dtype=np.uint16, endpoint=True) +def_gen.integers(0, 65535, dtype=np.uint16, endpoint=True) +def_gen.integers(I_u2_low_like, 65535, dtype=np.uint16, endpoint=True) +def_gen.integers(I_u2_high_open, dtype=np.uint16) +def_gen.integers(I_u2_low, I_u2_high_open, dtype=np.uint16) +def_gen.integers(0, I_u2_high_open, dtype=np.uint16) +def_gen.integers(I_u2_high_closed, dtype=np.uint16, endpoint=True) +def_gen.integers(I_u2_low, I_u2_high_closed, dtype=np.uint16, endpoint=True) +def_gen.integers(0, I_u2_high_closed, dtype=np.uint16, endpoint=True) + +I_u4_low: np.ndarray[Any, np.dtype[np.uint32]] = np.array([0], dtype=np.uint32) +I_u4_low_like: List[int] = [0] +I_u4_high_open: np.ndarray[Any, np.dtype[np.uint32]] = np.array([4294967295], dtype=np.uint32) +I_u4_high_closed: np.ndarray[Any, np.dtype[np.uint32]] = np.array([4294967295], dtype=np.uint32) + +def_gen.integers(4294967296, dtype="u4") +def_gen.integers(0, 4294967296, dtype="u4") +def_gen.integers(4294967295, dtype="u4", endpoint=True) +def_gen.integers(0, 4294967295, dtype="u4", endpoint=True) +def_gen.integers(I_u4_low_like, 4294967295, dtype="u4", endpoint=True) +def_gen.integers(I_u4_high_open, dtype="u4") +def_gen.integers(I_u4_low, I_u4_high_open, dtype="u4") +def_gen.integers(0, I_u4_high_open, dtype="u4") +def_gen.integers(I_u4_high_closed, dtype="u4", endpoint=True) +def_gen.integers(I_u4_low, I_u4_high_closed, dtype="u4", endpoint=True) +def_gen.integers(0, I_u4_high_closed, dtype="u4", endpoint=True) + +def_gen.integers(4294967296, dtype="uint32") +def_gen.integers(0, 4294967296, dtype="uint32") +def_gen.integers(4294967295, dtype="uint32", endpoint=True) +def_gen.integers(0, 4294967295, dtype="uint32", endpoint=True) +def_gen.integers(I_u4_low_like, 4294967295, dtype="uint32", endpoint=True) +def_gen.integers(I_u4_high_open, dtype="uint32") +def_gen.integers(I_u4_low, I_u4_high_open, dtype="uint32") +def_gen.integers(0, I_u4_high_open, dtype="uint32") +def_gen.integers(I_u4_high_closed, dtype="uint32", endpoint=True) +def_gen.integers(I_u4_low, I_u4_high_closed, dtype="uint32", endpoint=True) +def_gen.integers(0, I_u4_high_closed, dtype="uint32", endpoint=True) + +def_gen.integers(4294967296, dtype=np.uint32) +def_gen.integers(0, 4294967296, dtype=np.uint32) +def_gen.integers(4294967295, dtype=np.uint32, endpoint=True) +def_gen.integers(0, 4294967295, dtype=np.uint32, endpoint=True) +def_gen.integers(I_u4_low_like, 4294967295, dtype=np.uint32, endpoint=True) +def_gen.integers(I_u4_high_open, dtype=np.uint32) +def_gen.integers(I_u4_low, I_u4_high_open, dtype=np.uint32) +def_gen.integers(0, I_u4_high_open, dtype=np.uint32) +def_gen.integers(I_u4_high_closed, dtype=np.uint32, endpoint=True) +def_gen.integers(I_u4_low, I_u4_high_closed, dtype=np.uint32, endpoint=True) +def_gen.integers(0, I_u4_high_closed, dtype=np.uint32, endpoint=True) + +I_u8_low: np.ndarray[Any, np.dtype[np.uint64]] = np.array([0], dtype=np.uint64) +I_u8_low_like: List[int] = [0] +I_u8_high_open: np.ndarray[Any, np.dtype[np.uint64]] = np.array([18446744073709551615], dtype=np.uint64) +I_u8_high_closed: np.ndarray[Any, np.dtype[np.uint64]] = np.array([18446744073709551615], dtype=np.uint64) + +def_gen.integers(18446744073709551616, dtype="u8") +def_gen.integers(0, 18446744073709551616, dtype="u8") +def_gen.integers(18446744073709551615, dtype="u8", endpoint=True) +def_gen.integers(0, 18446744073709551615, dtype="u8", endpoint=True) +def_gen.integers(I_u8_low_like, 18446744073709551615, dtype="u8", endpoint=True) +def_gen.integers(I_u8_high_open, dtype="u8") +def_gen.integers(I_u8_low, I_u8_high_open, dtype="u8") +def_gen.integers(0, I_u8_high_open, dtype="u8") +def_gen.integers(I_u8_high_closed, dtype="u8", endpoint=True) +def_gen.integers(I_u8_low, I_u8_high_closed, dtype="u8", endpoint=True) +def_gen.integers(0, I_u8_high_closed, dtype="u8", endpoint=True) + +def_gen.integers(18446744073709551616, dtype="uint64") +def_gen.integers(0, 18446744073709551616, dtype="uint64") +def_gen.integers(18446744073709551615, dtype="uint64", endpoint=True) +def_gen.integers(0, 18446744073709551615, dtype="uint64", endpoint=True) +def_gen.integers(I_u8_low_like, 18446744073709551615, dtype="uint64", endpoint=True) +def_gen.integers(I_u8_high_open, dtype="uint64") +def_gen.integers(I_u8_low, I_u8_high_open, dtype="uint64") +def_gen.integers(0, I_u8_high_open, dtype="uint64") +def_gen.integers(I_u8_high_closed, dtype="uint64", endpoint=True) +def_gen.integers(I_u8_low, I_u8_high_closed, dtype="uint64", endpoint=True) +def_gen.integers(0, I_u8_high_closed, dtype="uint64", endpoint=True) + +def_gen.integers(18446744073709551616, dtype=np.uint64) +def_gen.integers(0, 18446744073709551616, dtype=np.uint64) +def_gen.integers(18446744073709551615, dtype=np.uint64, endpoint=True) +def_gen.integers(0, 18446744073709551615, dtype=np.uint64, endpoint=True) +def_gen.integers(I_u8_low_like, 18446744073709551615, dtype=np.uint64, endpoint=True) +def_gen.integers(I_u8_high_open, dtype=np.uint64) +def_gen.integers(I_u8_low, I_u8_high_open, dtype=np.uint64) +def_gen.integers(0, I_u8_high_open, dtype=np.uint64) +def_gen.integers(I_u8_high_closed, dtype=np.uint64, endpoint=True) +def_gen.integers(I_u8_low, I_u8_high_closed, dtype=np.uint64, endpoint=True) +def_gen.integers(0, I_u8_high_closed, dtype=np.uint64, endpoint=True) + +I_i1_low: np.ndarray[Any, np.dtype[np.int8]] = np.array([-128], dtype=np.int8) +I_i1_low_like: List[int] = [-128] +I_i1_high_open: np.ndarray[Any, np.dtype[np.int8]] = np.array([127], dtype=np.int8) +I_i1_high_closed: np.ndarray[Any, np.dtype[np.int8]] = np.array([127], dtype=np.int8) + +def_gen.integers(128, dtype="i1") +def_gen.integers(-128, 128, dtype="i1") +def_gen.integers(127, dtype="i1", endpoint=True) +def_gen.integers(-128, 127, dtype="i1", endpoint=True) +def_gen.integers(I_i1_low_like, 127, dtype="i1", endpoint=True) +def_gen.integers(I_i1_high_open, dtype="i1") +def_gen.integers(I_i1_low, I_i1_high_open, dtype="i1") +def_gen.integers(-128, I_i1_high_open, dtype="i1") +def_gen.integers(I_i1_high_closed, dtype="i1", endpoint=True) +def_gen.integers(I_i1_low, I_i1_high_closed, dtype="i1", endpoint=True) +def_gen.integers(-128, I_i1_high_closed, dtype="i1", endpoint=True) + +def_gen.integers(128, dtype="int8") +def_gen.integers(-128, 128, dtype="int8") +def_gen.integers(127, dtype="int8", endpoint=True) +def_gen.integers(-128, 127, dtype="int8", endpoint=True) +def_gen.integers(I_i1_low_like, 127, dtype="int8", endpoint=True) +def_gen.integers(I_i1_high_open, dtype="int8") +def_gen.integers(I_i1_low, I_i1_high_open, dtype="int8") +def_gen.integers(-128, I_i1_high_open, dtype="int8") +def_gen.integers(I_i1_high_closed, dtype="int8", endpoint=True) +def_gen.integers(I_i1_low, I_i1_high_closed, dtype="int8", endpoint=True) +def_gen.integers(-128, I_i1_high_closed, dtype="int8", endpoint=True) + +def_gen.integers(128, dtype=np.int8) +def_gen.integers(-128, 128, dtype=np.int8) +def_gen.integers(127, dtype=np.int8, endpoint=True) +def_gen.integers(-128, 127, dtype=np.int8, endpoint=True) +def_gen.integers(I_i1_low_like, 127, dtype=np.int8, endpoint=True) +def_gen.integers(I_i1_high_open, dtype=np.int8) +def_gen.integers(I_i1_low, I_i1_high_open, dtype=np.int8) +def_gen.integers(-128, I_i1_high_open, dtype=np.int8) +def_gen.integers(I_i1_high_closed, dtype=np.int8, endpoint=True) +def_gen.integers(I_i1_low, I_i1_high_closed, dtype=np.int8, endpoint=True) +def_gen.integers(-128, I_i1_high_closed, dtype=np.int8, endpoint=True) + +I_i2_low: np.ndarray[Any, np.dtype[np.int16]] = np.array([-32768], dtype=np.int16) +I_i2_low_like: List[int] = [-32768] +I_i2_high_open: np.ndarray[Any, np.dtype[np.int16]] = np.array([32767], dtype=np.int16) +I_i2_high_closed: np.ndarray[Any, np.dtype[np.int16]] = np.array([32767], dtype=np.int16) + +def_gen.integers(32768, dtype="i2") +def_gen.integers(-32768, 32768, dtype="i2") +def_gen.integers(32767, dtype="i2", endpoint=True) +def_gen.integers(-32768, 32767, dtype="i2", endpoint=True) +def_gen.integers(I_i2_low_like, 32767, dtype="i2", endpoint=True) +def_gen.integers(I_i2_high_open, dtype="i2") +def_gen.integers(I_i2_low, I_i2_high_open, dtype="i2") +def_gen.integers(-32768, I_i2_high_open, dtype="i2") +def_gen.integers(I_i2_high_closed, dtype="i2", endpoint=True) +def_gen.integers(I_i2_low, I_i2_high_closed, dtype="i2", endpoint=True) +def_gen.integers(-32768, I_i2_high_closed, dtype="i2", endpoint=True) + +def_gen.integers(32768, dtype="int16") +def_gen.integers(-32768, 32768, dtype="int16") +def_gen.integers(32767, dtype="int16", endpoint=True) +def_gen.integers(-32768, 32767, dtype="int16", endpoint=True) +def_gen.integers(I_i2_low_like, 32767, dtype="int16", endpoint=True) +def_gen.integers(I_i2_high_open, dtype="int16") +def_gen.integers(I_i2_low, I_i2_high_open, dtype="int16") +def_gen.integers(-32768, I_i2_high_open, dtype="int16") +def_gen.integers(I_i2_high_closed, dtype="int16", endpoint=True) +def_gen.integers(I_i2_low, I_i2_high_closed, dtype="int16", endpoint=True) +def_gen.integers(-32768, I_i2_high_closed, dtype="int16", endpoint=True) + +def_gen.integers(32768, dtype=np.int16) +def_gen.integers(-32768, 32768, dtype=np.int16) +def_gen.integers(32767, dtype=np.int16, endpoint=True) +def_gen.integers(-32768, 32767, dtype=np.int16, endpoint=True) +def_gen.integers(I_i2_low_like, 32767, dtype=np.int16, endpoint=True) +def_gen.integers(I_i2_high_open, dtype=np.int16) +def_gen.integers(I_i2_low, I_i2_high_open, dtype=np.int16) +def_gen.integers(-32768, I_i2_high_open, dtype=np.int16) +def_gen.integers(I_i2_high_closed, dtype=np.int16, endpoint=True) +def_gen.integers(I_i2_low, I_i2_high_closed, dtype=np.int16, endpoint=True) +def_gen.integers(-32768, I_i2_high_closed, dtype=np.int16, endpoint=True) + +I_i4_low: np.ndarray[Any, np.dtype[np.int32]] = np.array([-2147483648], dtype=np.int32) +I_i4_low_like: List[int] = [-2147483648] +I_i4_high_open: np.ndarray[Any, np.dtype[np.int32]] = np.array([2147483647], dtype=np.int32) +I_i4_high_closed: np.ndarray[Any, np.dtype[np.int32]] = np.array([2147483647], dtype=np.int32) + +def_gen.integers(2147483648, dtype="i4") +def_gen.integers(-2147483648, 2147483648, dtype="i4") +def_gen.integers(2147483647, dtype="i4", endpoint=True) +def_gen.integers(-2147483648, 2147483647, dtype="i4", endpoint=True) +def_gen.integers(I_i4_low_like, 2147483647, dtype="i4", endpoint=True) +def_gen.integers(I_i4_high_open, dtype="i4") +def_gen.integers(I_i4_low, I_i4_high_open, dtype="i4") +def_gen.integers(-2147483648, I_i4_high_open, dtype="i4") +def_gen.integers(I_i4_high_closed, dtype="i4", endpoint=True) +def_gen.integers(I_i4_low, I_i4_high_closed, dtype="i4", endpoint=True) +def_gen.integers(-2147483648, I_i4_high_closed, dtype="i4", endpoint=True) + +def_gen.integers(2147483648, dtype="int32") +def_gen.integers(-2147483648, 2147483648, dtype="int32") +def_gen.integers(2147483647, dtype="int32", endpoint=True) +def_gen.integers(-2147483648, 2147483647, dtype="int32", endpoint=True) +def_gen.integers(I_i4_low_like, 2147483647, dtype="int32", endpoint=True) +def_gen.integers(I_i4_high_open, dtype="int32") +def_gen.integers(I_i4_low, I_i4_high_open, dtype="int32") +def_gen.integers(-2147483648, I_i4_high_open, dtype="int32") +def_gen.integers(I_i4_high_closed, dtype="int32", endpoint=True) +def_gen.integers(I_i4_low, I_i4_high_closed, dtype="int32", endpoint=True) +def_gen.integers(-2147483648, I_i4_high_closed, dtype="int32", endpoint=True) + +def_gen.integers(2147483648, dtype=np.int32) +def_gen.integers(-2147483648, 2147483648, dtype=np.int32) +def_gen.integers(2147483647, dtype=np.int32, endpoint=True) +def_gen.integers(-2147483648, 2147483647, dtype=np.int32, endpoint=True) +def_gen.integers(I_i4_low_like, 2147483647, dtype=np.int32, endpoint=True) +def_gen.integers(I_i4_high_open, dtype=np.int32) +def_gen.integers(I_i4_low, I_i4_high_open, dtype=np.int32) +def_gen.integers(-2147483648, I_i4_high_open, dtype=np.int32) +def_gen.integers(I_i4_high_closed, dtype=np.int32, endpoint=True) +def_gen.integers(I_i4_low, I_i4_high_closed, dtype=np.int32, endpoint=True) +def_gen.integers(-2147483648, I_i4_high_closed, dtype=np.int32, endpoint=True) + +I_i8_low: np.ndarray[Any, np.dtype[np.int64]] = np.array([-9223372036854775808], dtype=np.int64) +I_i8_low_like: List[int] = [-9223372036854775808] +I_i8_high_open: np.ndarray[Any, np.dtype[np.int64]] = np.array([9223372036854775807], dtype=np.int64) +I_i8_high_closed: np.ndarray[Any, np.dtype[np.int64]] = np.array([9223372036854775807], dtype=np.int64) + +def_gen.integers(9223372036854775808, dtype="i8") +def_gen.integers(-9223372036854775808, 9223372036854775808, dtype="i8") +def_gen.integers(9223372036854775807, dtype="i8", endpoint=True) +def_gen.integers(-9223372036854775808, 9223372036854775807, dtype="i8", endpoint=True) +def_gen.integers(I_i8_low_like, 9223372036854775807, dtype="i8", endpoint=True) +def_gen.integers(I_i8_high_open, dtype="i8") +def_gen.integers(I_i8_low, I_i8_high_open, dtype="i8") +def_gen.integers(-9223372036854775808, I_i8_high_open, dtype="i8") +def_gen.integers(I_i8_high_closed, dtype="i8", endpoint=True) +def_gen.integers(I_i8_low, I_i8_high_closed, dtype="i8", endpoint=True) +def_gen.integers(-9223372036854775808, I_i8_high_closed, dtype="i8", endpoint=True) + +def_gen.integers(9223372036854775808, dtype="int64") +def_gen.integers(-9223372036854775808, 9223372036854775808, dtype="int64") +def_gen.integers(9223372036854775807, dtype="int64", endpoint=True) +def_gen.integers(-9223372036854775808, 9223372036854775807, dtype="int64", endpoint=True) +def_gen.integers(I_i8_low_like, 9223372036854775807, dtype="int64", endpoint=True) +def_gen.integers(I_i8_high_open, dtype="int64") +def_gen.integers(I_i8_low, I_i8_high_open, dtype="int64") +def_gen.integers(-9223372036854775808, I_i8_high_open, dtype="int64") +def_gen.integers(I_i8_high_closed, dtype="int64", endpoint=True) +def_gen.integers(I_i8_low, I_i8_high_closed, dtype="int64", endpoint=True) +def_gen.integers(-9223372036854775808, I_i8_high_closed, dtype="int64", endpoint=True) + +def_gen.integers(9223372036854775808, dtype=np.int64) +def_gen.integers(-9223372036854775808, 9223372036854775808, dtype=np.int64) +def_gen.integers(9223372036854775807, dtype=np.int64, endpoint=True) +def_gen.integers(-9223372036854775808, 9223372036854775807, dtype=np.int64, endpoint=True) +def_gen.integers(I_i8_low_like, 9223372036854775807, dtype=np.int64, endpoint=True) +def_gen.integers(I_i8_high_open, dtype=np.int64) +def_gen.integers(I_i8_low, I_i8_high_open, dtype=np.int64) +def_gen.integers(-9223372036854775808, I_i8_high_open, dtype=np.int64) +def_gen.integers(I_i8_high_closed, dtype=np.int64, endpoint=True) +def_gen.integers(I_i8_low, I_i8_high_closed, dtype=np.int64, endpoint=True) +def_gen.integers(-9223372036854775808, I_i8_high_closed, dtype=np.int64, endpoint=True) + + +def_gen.bit_generator + +def_gen.bytes(2) + +def_gen.choice(5) +def_gen.choice(5, 3) +def_gen.choice(5, 3, replace=True) +def_gen.choice(5, 3, p=[1 / 5] * 5) +def_gen.choice(5, 3, p=[1 / 5] * 5, replace=False) + +def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"]) +def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3) +def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, p=[1 / 4] * 4) +def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=True) +def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=False, p=np.array([1 / 8, 1 / 8, 1 / 2, 1 / 4])) + +def_gen.dirichlet([0.5, 0.5]) +def_gen.dirichlet(np.array([0.5, 0.5])) +def_gen.dirichlet(np.array([0.5, 0.5]), size=3) + +def_gen.multinomial(20, [1 / 6.0] * 6) +def_gen.multinomial(20, np.array([0.5, 0.5])) +def_gen.multinomial(20, [1 / 6.0] * 6, size=2) +def_gen.multinomial([[10], [20]], [1 / 6.0] * 6, size=(2, 2)) +def_gen.multinomial(np.array([[10], [20]]), np.array([0.5, 0.5]), size=(2, 2)) + +def_gen.multivariate_hypergeometric([3, 5, 7], 2) +def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2) +def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2, size=4) +def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2, size=(4, 7)) +def_gen.multivariate_hypergeometric([3, 5, 7], 2, method="count") +def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2, method="marginals") + +def_gen.multivariate_normal([0.0], [[1.0]]) +def_gen.multivariate_normal([0.0], np.array([[1.0]])) +def_gen.multivariate_normal(np.array([0.0]), [[1.0]]) +def_gen.multivariate_normal([0.0], np.array([[1.0]])) + +def_gen.permutation(10) +def_gen.permutation([1, 2, 3, 4]) +def_gen.permutation(np.array([1, 2, 3, 4])) +def_gen.permutation(D_2D, axis=1) +def_gen.permuted(D_2D) +def_gen.permuted(D_2D_like) +def_gen.permuted(D_2D, axis=1) +def_gen.permuted(D_2D, out=D_2D) +def_gen.permuted(D_2D_like, out=D_2D) +def_gen.permuted(D_2D_like, out=D_2D) +def_gen.permuted(D_2D, axis=1, out=D_2D) + +def_gen.shuffle(np.arange(10)) +def_gen.shuffle([1, 2, 3, 4, 5]) +def_gen.shuffle(D_2D, axis=1) + +def_gen.__str__() +def_gen.__repr__() +def_gen_state: Dict[str, Any] +def_gen_state = def_gen.__getstate__() +def_gen.__setstate__(def_gen_state) + +# RandomState +random_st: np.random.RandomState = np.random.RandomState() + +random_st.standard_normal() +random_st.standard_normal(size=None) +random_st.standard_normal(size=1) + +random_st.random() +random_st.random(size=None) +random_st.random(size=1) + +random_st.standard_cauchy() +random_st.standard_cauchy(size=None) +random_st.standard_cauchy(size=1) + +random_st.standard_exponential() +random_st.standard_exponential(size=None) +random_st.standard_exponential(size=1) + +random_st.zipf(1.5) +random_st.zipf(1.5, size=None) +random_st.zipf(1.5, size=1) +random_st.zipf(D_arr_1p5) +random_st.zipf(D_arr_1p5, size=1) +random_st.zipf(D_arr_like_1p5) +random_st.zipf(D_arr_like_1p5, size=1) + +random_st.weibull(0.5) +random_st.weibull(0.5, size=None) +random_st.weibull(0.5, size=1) +random_st.weibull(D_arr_0p5) +random_st.weibull(D_arr_0p5, size=1) +random_st.weibull(D_arr_like_0p5) +random_st.weibull(D_arr_like_0p5, size=1) + +random_st.standard_t(0.5) +random_st.standard_t(0.5, size=None) +random_st.standard_t(0.5, size=1) +random_st.standard_t(D_arr_0p5) +random_st.standard_t(D_arr_0p5, size=1) +random_st.standard_t(D_arr_like_0p5) +random_st.standard_t(D_arr_like_0p5, size=1) + +random_st.poisson(0.5) +random_st.poisson(0.5, size=None) +random_st.poisson(0.5, size=1) +random_st.poisson(D_arr_0p5) +random_st.poisson(D_arr_0p5, size=1) +random_st.poisson(D_arr_like_0p5) +random_st.poisson(D_arr_like_0p5, size=1) + +random_st.power(0.5) +random_st.power(0.5, size=None) +random_st.power(0.5, size=1) +random_st.power(D_arr_0p5) +random_st.power(D_arr_0p5, size=1) +random_st.power(D_arr_like_0p5) +random_st.power(D_arr_like_0p5, size=1) + +random_st.pareto(0.5) +random_st.pareto(0.5, size=None) +random_st.pareto(0.5, size=1) +random_st.pareto(D_arr_0p5) +random_st.pareto(D_arr_0p5, size=1) +random_st.pareto(D_arr_like_0p5) +random_st.pareto(D_arr_like_0p5, size=1) + +random_st.chisquare(0.5) +random_st.chisquare(0.5, size=None) +random_st.chisquare(0.5, size=1) +random_st.chisquare(D_arr_0p5) +random_st.chisquare(D_arr_0p5, size=1) +random_st.chisquare(D_arr_like_0p5) +random_st.chisquare(D_arr_like_0p5, size=1) + +random_st.exponential(0.5) +random_st.exponential(0.5, size=None) +random_st.exponential(0.5, size=1) +random_st.exponential(D_arr_0p5) +random_st.exponential(D_arr_0p5, size=1) +random_st.exponential(D_arr_like_0p5) +random_st.exponential(D_arr_like_0p5, size=1) + +random_st.geometric(0.5) +random_st.geometric(0.5, size=None) +random_st.geometric(0.5, size=1) +random_st.geometric(D_arr_0p5) +random_st.geometric(D_arr_0p5, size=1) +random_st.geometric(D_arr_like_0p5) +random_st.geometric(D_arr_like_0p5, size=1) + +random_st.logseries(0.5) +random_st.logseries(0.5, size=None) +random_st.logseries(0.5, size=1) +random_st.logseries(D_arr_0p5) +random_st.logseries(D_arr_0p5, size=1) +random_st.logseries(D_arr_like_0p5) +random_st.logseries(D_arr_like_0p5, size=1) + +random_st.rayleigh(0.5) +random_st.rayleigh(0.5, size=None) +random_st.rayleigh(0.5, size=1) +random_st.rayleigh(D_arr_0p5) +random_st.rayleigh(D_arr_0p5, size=1) +random_st.rayleigh(D_arr_like_0p5) +random_st.rayleigh(D_arr_like_0p5, size=1) + +random_st.standard_gamma(0.5) +random_st.standard_gamma(0.5, size=None) +random_st.standard_gamma(0.5, size=1) +random_st.standard_gamma(D_arr_0p5) +random_st.standard_gamma(D_arr_0p5, size=1) +random_st.standard_gamma(D_arr_like_0p5) +random_st.standard_gamma(D_arr_like_0p5, size=1) +random_st.standard_gamma(D_arr_like_0p5, size=1) + +random_st.vonmises(0.5, 0.5) +random_st.vonmises(0.5, 0.5, size=None) +random_st.vonmises(0.5, 0.5, size=1) +random_st.vonmises(D_arr_0p5, 0.5) +random_st.vonmises(0.5, D_arr_0p5) +random_st.vonmises(D_arr_0p5, 0.5, size=1) +random_st.vonmises(0.5, D_arr_0p5, size=1) +random_st.vonmises(D_arr_like_0p5, 0.5) +random_st.vonmises(0.5, D_arr_like_0p5) +random_st.vonmises(D_arr_0p5, D_arr_0p5) +random_st.vonmises(D_arr_like_0p5, D_arr_like_0p5) +random_st.vonmises(D_arr_0p5, D_arr_0p5, size=1) +random_st.vonmises(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.wald(0.5, 0.5) +random_st.wald(0.5, 0.5, size=None) +random_st.wald(0.5, 0.5, size=1) +random_st.wald(D_arr_0p5, 0.5) +random_st.wald(0.5, D_arr_0p5) +random_st.wald(D_arr_0p5, 0.5, size=1) +random_st.wald(0.5, D_arr_0p5, size=1) +random_st.wald(D_arr_like_0p5, 0.5) +random_st.wald(0.5, D_arr_like_0p5) +random_st.wald(D_arr_0p5, D_arr_0p5) +random_st.wald(D_arr_like_0p5, D_arr_like_0p5) +random_st.wald(D_arr_0p5, D_arr_0p5, size=1) +random_st.wald(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.uniform(0.5, 0.5) +random_st.uniform(0.5, 0.5, size=None) +random_st.uniform(0.5, 0.5, size=1) +random_st.uniform(D_arr_0p5, 0.5) +random_st.uniform(0.5, D_arr_0p5) +random_st.uniform(D_arr_0p5, 0.5, size=1) +random_st.uniform(0.5, D_arr_0p5, size=1) +random_st.uniform(D_arr_like_0p5, 0.5) +random_st.uniform(0.5, D_arr_like_0p5) +random_st.uniform(D_arr_0p5, D_arr_0p5) +random_st.uniform(D_arr_like_0p5, D_arr_like_0p5) +random_st.uniform(D_arr_0p5, D_arr_0p5, size=1) +random_st.uniform(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.beta(0.5, 0.5) +random_st.beta(0.5, 0.5, size=None) +random_st.beta(0.5, 0.5, size=1) +random_st.beta(D_arr_0p5, 0.5) +random_st.beta(0.5, D_arr_0p5) +random_st.beta(D_arr_0p5, 0.5, size=1) +random_st.beta(0.5, D_arr_0p5, size=1) +random_st.beta(D_arr_like_0p5, 0.5) +random_st.beta(0.5, D_arr_like_0p5) +random_st.beta(D_arr_0p5, D_arr_0p5) +random_st.beta(D_arr_like_0p5, D_arr_like_0p5) +random_st.beta(D_arr_0p5, D_arr_0p5, size=1) +random_st.beta(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.f(0.5, 0.5) +random_st.f(0.5, 0.5, size=None) +random_st.f(0.5, 0.5, size=1) +random_st.f(D_arr_0p5, 0.5) +random_st.f(0.5, D_arr_0p5) +random_st.f(D_arr_0p5, 0.5, size=1) +random_st.f(0.5, D_arr_0p5, size=1) +random_st.f(D_arr_like_0p5, 0.5) +random_st.f(0.5, D_arr_like_0p5) +random_st.f(D_arr_0p5, D_arr_0p5) +random_st.f(D_arr_like_0p5, D_arr_like_0p5) +random_st.f(D_arr_0p5, D_arr_0p5, size=1) +random_st.f(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.gamma(0.5, 0.5) +random_st.gamma(0.5, 0.5, size=None) +random_st.gamma(0.5, 0.5, size=1) +random_st.gamma(D_arr_0p5, 0.5) +random_st.gamma(0.5, D_arr_0p5) +random_st.gamma(D_arr_0p5, 0.5, size=1) +random_st.gamma(0.5, D_arr_0p5, size=1) +random_st.gamma(D_arr_like_0p5, 0.5) +random_st.gamma(0.5, D_arr_like_0p5) +random_st.gamma(D_arr_0p5, D_arr_0p5) +random_st.gamma(D_arr_like_0p5, D_arr_like_0p5) +random_st.gamma(D_arr_0p5, D_arr_0p5, size=1) +random_st.gamma(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.gumbel(0.5, 0.5) +random_st.gumbel(0.5, 0.5, size=None) +random_st.gumbel(0.5, 0.5, size=1) +random_st.gumbel(D_arr_0p5, 0.5) +random_st.gumbel(0.5, D_arr_0p5) +random_st.gumbel(D_arr_0p5, 0.5, size=1) +random_st.gumbel(0.5, D_arr_0p5, size=1) +random_st.gumbel(D_arr_like_0p5, 0.5) +random_st.gumbel(0.5, D_arr_like_0p5) +random_st.gumbel(D_arr_0p5, D_arr_0p5) +random_st.gumbel(D_arr_like_0p5, D_arr_like_0p5) +random_st.gumbel(D_arr_0p5, D_arr_0p5, size=1) +random_st.gumbel(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.laplace(0.5, 0.5) +random_st.laplace(0.5, 0.5, size=None) +random_st.laplace(0.5, 0.5, size=1) +random_st.laplace(D_arr_0p5, 0.5) +random_st.laplace(0.5, D_arr_0p5) +random_st.laplace(D_arr_0p5, 0.5, size=1) +random_st.laplace(0.5, D_arr_0p5, size=1) +random_st.laplace(D_arr_like_0p5, 0.5) +random_st.laplace(0.5, D_arr_like_0p5) +random_st.laplace(D_arr_0p5, D_arr_0p5) +random_st.laplace(D_arr_like_0p5, D_arr_like_0p5) +random_st.laplace(D_arr_0p5, D_arr_0p5, size=1) +random_st.laplace(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.logistic(0.5, 0.5) +random_st.logistic(0.5, 0.5, size=None) +random_st.logistic(0.5, 0.5, size=1) +random_st.logistic(D_arr_0p5, 0.5) +random_st.logistic(0.5, D_arr_0p5) +random_st.logistic(D_arr_0p5, 0.5, size=1) +random_st.logistic(0.5, D_arr_0p5, size=1) +random_st.logistic(D_arr_like_0p5, 0.5) +random_st.logistic(0.5, D_arr_like_0p5) +random_st.logistic(D_arr_0p5, D_arr_0p5) +random_st.logistic(D_arr_like_0p5, D_arr_like_0p5) +random_st.logistic(D_arr_0p5, D_arr_0p5, size=1) +random_st.logistic(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.lognormal(0.5, 0.5) +random_st.lognormal(0.5, 0.5, size=None) +random_st.lognormal(0.5, 0.5, size=1) +random_st.lognormal(D_arr_0p5, 0.5) +random_st.lognormal(0.5, D_arr_0p5) +random_st.lognormal(D_arr_0p5, 0.5, size=1) +random_st.lognormal(0.5, D_arr_0p5, size=1) +random_st.lognormal(D_arr_like_0p5, 0.5) +random_st.lognormal(0.5, D_arr_like_0p5) +random_st.lognormal(D_arr_0p5, D_arr_0p5) +random_st.lognormal(D_arr_like_0p5, D_arr_like_0p5) +random_st.lognormal(D_arr_0p5, D_arr_0p5, size=1) +random_st.lognormal(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.noncentral_chisquare(0.5, 0.5) +random_st.noncentral_chisquare(0.5, 0.5, size=None) +random_st.noncentral_chisquare(0.5, 0.5, size=1) +random_st.noncentral_chisquare(D_arr_0p5, 0.5) +random_st.noncentral_chisquare(0.5, D_arr_0p5) +random_st.noncentral_chisquare(D_arr_0p5, 0.5, size=1) +random_st.noncentral_chisquare(0.5, D_arr_0p5, size=1) +random_st.noncentral_chisquare(D_arr_like_0p5, 0.5) +random_st.noncentral_chisquare(0.5, D_arr_like_0p5) +random_st.noncentral_chisquare(D_arr_0p5, D_arr_0p5) +random_st.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5) +random_st.noncentral_chisquare(D_arr_0p5, D_arr_0p5, size=1) +random_st.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.normal(0.5, 0.5) +random_st.normal(0.5, 0.5, size=None) +random_st.normal(0.5, 0.5, size=1) +random_st.normal(D_arr_0p5, 0.5) +random_st.normal(0.5, D_arr_0p5) +random_st.normal(D_arr_0p5, 0.5, size=1) +random_st.normal(0.5, D_arr_0p5, size=1) +random_st.normal(D_arr_like_0p5, 0.5) +random_st.normal(0.5, D_arr_like_0p5) +random_st.normal(D_arr_0p5, D_arr_0p5) +random_st.normal(D_arr_like_0p5, D_arr_like_0p5) +random_st.normal(D_arr_0p5, D_arr_0p5, size=1) +random_st.normal(D_arr_like_0p5, D_arr_like_0p5, size=1) + +random_st.triangular(0.1, 0.5, 0.9) +random_st.triangular(0.1, 0.5, 0.9, size=None) +random_st.triangular(0.1, 0.5, 0.9, size=1) +random_st.triangular(D_arr_0p1, 0.5, 0.9) +random_st.triangular(0.1, D_arr_0p5, 0.9) +random_st.triangular(D_arr_0p1, 0.5, D_arr_like_0p9, size=1) +random_st.triangular(0.1, D_arr_0p5, 0.9, size=1) +random_st.triangular(D_arr_like_0p1, 0.5, D_arr_0p9) +random_st.triangular(0.5, D_arr_like_0p5, 0.9) +random_st.triangular(D_arr_0p1, D_arr_0p5, 0.9) +random_st.triangular(D_arr_like_0p1, D_arr_like_0p5, 0.9) +random_st.triangular(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1) +random_st.triangular(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1) + +random_st.noncentral_f(0.1, 0.5, 0.9) +random_st.noncentral_f(0.1, 0.5, 0.9, size=None) +random_st.noncentral_f(0.1, 0.5, 0.9, size=1) +random_st.noncentral_f(D_arr_0p1, 0.5, 0.9) +random_st.noncentral_f(0.1, D_arr_0p5, 0.9) +random_st.noncentral_f(D_arr_0p1, 0.5, D_arr_like_0p9, size=1) +random_st.noncentral_f(0.1, D_arr_0p5, 0.9, size=1) +random_st.noncentral_f(D_arr_like_0p1, 0.5, D_arr_0p9) +random_st.noncentral_f(0.5, D_arr_like_0p5, 0.9) +random_st.noncentral_f(D_arr_0p1, D_arr_0p5, 0.9) +random_st.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, 0.9) +random_st.noncentral_f(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1) +random_st.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1) + +random_st.binomial(10, 0.5) +random_st.binomial(10, 0.5, size=None) +random_st.binomial(10, 0.5, size=1) +random_st.binomial(I_arr_10, 0.5) +random_st.binomial(10, D_arr_0p5) +random_st.binomial(I_arr_10, 0.5, size=1) +random_st.binomial(10, D_arr_0p5, size=1) +random_st.binomial(I_arr_like_10, 0.5) +random_st.binomial(10, D_arr_like_0p5) +random_st.binomial(I_arr_10, D_arr_0p5) +random_st.binomial(I_arr_like_10, D_arr_like_0p5) +random_st.binomial(I_arr_10, D_arr_0p5, size=1) +random_st.binomial(I_arr_like_10, D_arr_like_0p5, size=1) + +random_st.negative_binomial(10, 0.5) +random_st.negative_binomial(10, 0.5, size=None) +random_st.negative_binomial(10, 0.5, size=1) +random_st.negative_binomial(I_arr_10, 0.5) +random_st.negative_binomial(10, D_arr_0p5) +random_st.negative_binomial(I_arr_10, 0.5, size=1) +random_st.negative_binomial(10, D_arr_0p5, size=1) +random_st.negative_binomial(I_arr_like_10, 0.5) +random_st.negative_binomial(10, D_arr_like_0p5) +random_st.negative_binomial(I_arr_10, D_arr_0p5) +random_st.negative_binomial(I_arr_like_10, D_arr_like_0p5) +random_st.negative_binomial(I_arr_10, D_arr_0p5, size=1) +random_st.negative_binomial(I_arr_like_10, D_arr_like_0p5, size=1) + +random_st.hypergeometric(20, 20, 10) +random_st.hypergeometric(20, 20, 10, size=None) +random_st.hypergeometric(20, 20, 10, size=1) +random_st.hypergeometric(I_arr_20, 20, 10) +random_st.hypergeometric(20, I_arr_20, 10) +random_st.hypergeometric(I_arr_20, 20, I_arr_like_10, size=1) +random_st.hypergeometric(20, I_arr_20, 10, size=1) +random_st.hypergeometric(I_arr_like_20, 20, I_arr_10) +random_st.hypergeometric(20, I_arr_like_20, 10) +random_st.hypergeometric(I_arr_20, I_arr_20, 10) +random_st.hypergeometric(I_arr_like_20, I_arr_like_20, 10) +random_st.hypergeometric(I_arr_20, I_arr_20, I_arr_10, size=1) +random_st.hypergeometric(I_arr_like_20, I_arr_like_20, I_arr_like_10, size=1) + +random_st.randint(0, 100) +random_st.randint(100) +random_st.randint([100]) +random_st.randint(0, [100]) + +random_st.randint(2, dtype=bool) +random_st.randint(0, 2, dtype=bool) +random_st.randint(I_bool_high_open, dtype=bool) +random_st.randint(I_bool_low, I_bool_high_open, dtype=bool) +random_st.randint(0, I_bool_high_open, dtype=bool) + +random_st.randint(2, dtype=np.bool_) +random_st.randint(0, 2, dtype=np.bool_) +random_st.randint(I_bool_high_open, dtype=np.bool_) +random_st.randint(I_bool_low, I_bool_high_open, dtype=np.bool_) +random_st.randint(0, I_bool_high_open, dtype=np.bool_) + +random_st.randint(256, dtype="u1") +random_st.randint(0, 256, dtype="u1") +random_st.randint(I_u1_high_open, dtype="u1") +random_st.randint(I_u1_low, I_u1_high_open, dtype="u1") +random_st.randint(0, I_u1_high_open, dtype="u1") + +random_st.randint(256, dtype="uint8") +random_st.randint(0, 256, dtype="uint8") +random_st.randint(I_u1_high_open, dtype="uint8") +random_st.randint(I_u1_low, I_u1_high_open, dtype="uint8") +random_st.randint(0, I_u1_high_open, dtype="uint8") + +random_st.randint(256, dtype=np.uint8) +random_st.randint(0, 256, dtype=np.uint8) +random_st.randint(I_u1_high_open, dtype=np.uint8) +random_st.randint(I_u1_low, I_u1_high_open, dtype=np.uint8) +random_st.randint(0, I_u1_high_open, dtype=np.uint8) + +random_st.randint(65536, dtype="u2") +random_st.randint(0, 65536, dtype="u2") +random_st.randint(I_u2_high_open, dtype="u2") +random_st.randint(I_u2_low, I_u2_high_open, dtype="u2") +random_st.randint(0, I_u2_high_open, dtype="u2") + +random_st.randint(65536, dtype="uint16") +random_st.randint(0, 65536, dtype="uint16") +random_st.randint(I_u2_high_open, dtype="uint16") +random_st.randint(I_u2_low, I_u2_high_open, dtype="uint16") +random_st.randint(0, I_u2_high_open, dtype="uint16") + +random_st.randint(65536, dtype=np.uint16) +random_st.randint(0, 65536, dtype=np.uint16) +random_st.randint(I_u2_high_open, dtype=np.uint16) +random_st.randint(I_u2_low, I_u2_high_open, dtype=np.uint16) +random_st.randint(0, I_u2_high_open, dtype=np.uint16) + +random_st.randint(4294967296, dtype="u4") +random_st.randint(0, 4294967296, dtype="u4") +random_st.randint(I_u4_high_open, dtype="u4") +random_st.randint(I_u4_low, I_u4_high_open, dtype="u4") +random_st.randint(0, I_u4_high_open, dtype="u4") + +random_st.randint(4294967296, dtype="uint32") +random_st.randint(0, 4294967296, dtype="uint32") +random_st.randint(I_u4_high_open, dtype="uint32") +random_st.randint(I_u4_low, I_u4_high_open, dtype="uint32") +random_st.randint(0, I_u4_high_open, dtype="uint32") + +random_st.randint(4294967296, dtype=np.uint32) +random_st.randint(0, 4294967296, dtype=np.uint32) +random_st.randint(I_u4_high_open, dtype=np.uint32) +random_st.randint(I_u4_low, I_u4_high_open, dtype=np.uint32) +random_st.randint(0, I_u4_high_open, dtype=np.uint32) + + +random_st.randint(18446744073709551616, dtype="u8") +random_st.randint(0, 18446744073709551616, dtype="u8") +random_st.randint(I_u8_high_open, dtype="u8") +random_st.randint(I_u8_low, I_u8_high_open, dtype="u8") +random_st.randint(0, I_u8_high_open, dtype="u8") + +random_st.randint(18446744073709551616, dtype="uint64") +random_st.randint(0, 18446744073709551616, dtype="uint64") +random_st.randint(I_u8_high_open, dtype="uint64") +random_st.randint(I_u8_low, I_u8_high_open, dtype="uint64") +random_st.randint(0, I_u8_high_open, dtype="uint64") + +random_st.randint(18446744073709551616, dtype=np.uint64) +random_st.randint(0, 18446744073709551616, dtype=np.uint64) +random_st.randint(I_u8_high_open, dtype=np.uint64) +random_st.randint(I_u8_low, I_u8_high_open, dtype=np.uint64) +random_st.randint(0, I_u8_high_open, dtype=np.uint64) + +random_st.randint(128, dtype="i1") +random_st.randint(-128, 128, dtype="i1") +random_st.randint(I_i1_high_open, dtype="i1") +random_st.randint(I_i1_low, I_i1_high_open, dtype="i1") +random_st.randint(-128, I_i1_high_open, dtype="i1") + +random_st.randint(128, dtype="int8") +random_st.randint(-128, 128, dtype="int8") +random_st.randint(I_i1_high_open, dtype="int8") +random_st.randint(I_i1_low, I_i1_high_open, dtype="int8") +random_st.randint(-128, I_i1_high_open, dtype="int8") + +random_st.randint(128, dtype=np.int8) +random_st.randint(-128, 128, dtype=np.int8) +random_st.randint(I_i1_high_open, dtype=np.int8) +random_st.randint(I_i1_low, I_i1_high_open, dtype=np.int8) +random_st.randint(-128, I_i1_high_open, dtype=np.int8) + +random_st.randint(32768, dtype="i2") +random_st.randint(-32768, 32768, dtype="i2") +random_st.randint(I_i2_high_open, dtype="i2") +random_st.randint(I_i2_low, I_i2_high_open, dtype="i2") +random_st.randint(-32768, I_i2_high_open, dtype="i2") +random_st.randint(32768, dtype="int16") +random_st.randint(-32768, 32768, dtype="int16") +random_st.randint(I_i2_high_open, dtype="int16") +random_st.randint(I_i2_low, I_i2_high_open, dtype="int16") +random_st.randint(-32768, I_i2_high_open, dtype="int16") +random_st.randint(32768, dtype=np.int16) +random_st.randint(-32768, 32768, dtype=np.int16) +random_st.randint(I_i2_high_open, dtype=np.int16) +random_st.randint(I_i2_low, I_i2_high_open, dtype=np.int16) +random_st.randint(-32768, I_i2_high_open, dtype=np.int16) + +random_st.randint(2147483648, dtype="i4") +random_st.randint(-2147483648, 2147483648, dtype="i4") +random_st.randint(I_i4_high_open, dtype="i4") +random_st.randint(I_i4_low, I_i4_high_open, dtype="i4") +random_st.randint(-2147483648, I_i4_high_open, dtype="i4") + +random_st.randint(2147483648, dtype="int32") +random_st.randint(-2147483648, 2147483648, dtype="int32") +random_st.randint(I_i4_high_open, dtype="int32") +random_st.randint(I_i4_low, I_i4_high_open, dtype="int32") +random_st.randint(-2147483648, I_i4_high_open, dtype="int32") + +random_st.randint(2147483648, dtype=np.int32) +random_st.randint(-2147483648, 2147483648, dtype=np.int32) +random_st.randint(I_i4_high_open, dtype=np.int32) +random_st.randint(I_i4_low, I_i4_high_open, dtype=np.int32) +random_st.randint(-2147483648, I_i4_high_open, dtype=np.int32) + +random_st.randint(9223372036854775808, dtype="i8") +random_st.randint(-9223372036854775808, 9223372036854775808, dtype="i8") +random_st.randint(I_i8_high_open, dtype="i8") +random_st.randint(I_i8_low, I_i8_high_open, dtype="i8") +random_st.randint(-9223372036854775808, I_i8_high_open, dtype="i8") + +random_st.randint(9223372036854775808, dtype="int64") +random_st.randint(-9223372036854775808, 9223372036854775808, dtype="int64") +random_st.randint(I_i8_high_open, dtype="int64") +random_st.randint(I_i8_low, I_i8_high_open, dtype="int64") +random_st.randint(-9223372036854775808, I_i8_high_open, dtype="int64") + +random_st.randint(9223372036854775808, dtype=np.int64) +random_st.randint(-9223372036854775808, 9223372036854775808, dtype=np.int64) +random_st.randint(I_i8_high_open, dtype=np.int64) +random_st.randint(I_i8_low, I_i8_high_open, dtype=np.int64) +random_st.randint(-9223372036854775808, I_i8_high_open, dtype=np.int64) + +bg: np.random.BitGenerator = random_st._bit_generator + +random_st.bytes(2) + +random_st.choice(5) +random_st.choice(5, 3) +random_st.choice(5, 3, replace=True) +random_st.choice(5, 3, p=[1 / 5] * 5) +random_st.choice(5, 3, p=[1 / 5] * 5, replace=False) + +random_st.choice(["pooh", "rabbit", "piglet", "Christopher"]) +random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3) +random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, p=[1 / 4] * 4) +random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=True) +random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=False, p=np.array([1 / 8, 1 / 8, 1 / 2, 1 / 4])) + +random_st.dirichlet([0.5, 0.5]) +random_st.dirichlet(np.array([0.5, 0.5])) +random_st.dirichlet(np.array([0.5, 0.5]), size=3) + +random_st.multinomial(20, [1 / 6.0] * 6) +random_st.multinomial(20, np.array([0.5, 0.5])) +random_st.multinomial(20, [1 / 6.0] * 6, size=2) + +random_st.multivariate_normal([0.0], [[1.0]]) +random_st.multivariate_normal([0.0], np.array([[1.0]])) +random_st.multivariate_normal(np.array([0.0]), [[1.0]]) +random_st.multivariate_normal([0.0], np.array([[1.0]])) + +random_st.permutation(10) +random_st.permutation([1, 2, 3, 4]) +random_st.permutation(np.array([1, 2, 3, 4])) +random_st.permutation(D_2D) + +random_st.shuffle(np.arange(10)) +random_st.shuffle([1, 2, 3, 4, 5]) +random_st.shuffle(D_2D) + +np.random.RandomState(SEED_PCG64) +np.random.RandomState(0) +np.random.RandomState([0, 1, 2]) +random_st.__str__() +random_st.__repr__() +random_st_state = random_st.__getstate__() +random_st.__setstate__(random_st_state) +random_st.seed() +random_st.seed(1) +random_st.seed([0, 1]) +random_st_get_state = random_st.get_state() +random_st_get_state_legacy = random_st.get_state(legacy=True) +random_st.set_state(random_st_get_state) + +random_st.rand() +random_st.rand(1) +random_st.rand(1, 2) +random_st.randn() +random_st.randn(1) +random_st.randn(1, 2) +random_st.random_sample() +random_st.random_sample(1) +random_st.random_sample(size=(1, 2)) + +random_st.tomaxint() +random_st.tomaxint(1) +random_st.tomaxint((1,)) diff --git a/numpy/typing/tests/data/pass/scalars.py b/numpy/typing/tests/data/pass/scalars.py index b7f7880e4561..b258db49fd7c 100644 --- a/numpy/typing/tests/data/pass/scalars.py +++ b/numpy/typing/tests/data/pass/scalars.py @@ -4,6 +4,14 @@ import pytest import numpy as np +b = np.bool_() +u8 = np.uint64() +i8 = np.int64() +f8 = np.float64() +c16 = np.complex128() +U = np.str_() +S = np.bytes_() + # Construction class D: @@ -84,6 +92,7 @@ def __float__(self) -> float: np.datetime64("2019", "D") np.datetime64(np.datetime64()) np.datetime64(dt.datetime(2000, 5, 3)) +np.datetime64(dt.date(2000, 5, 3)) np.datetime64(None) np.datetime64(None, "D") @@ -163,3 +172,83 @@ def __float__(self) -> float: c16.squeeze() c16.byteswap() c16.transpose() + +# Aliases +np.str0() +np.bool8() +np.bytes0() +np.string_() +np.object0() +np.void0(0) + +np.byte() +np.short() +np.intc() +np.intp() +np.int0() +np.int_() +np.longlong() + +np.ubyte() +np.ushort() +np.uintc() +np.uintp() +np.uint0() +np.uint() +np.ulonglong() + +np.half() +np.single() +np.double() +np.float_() +np.longdouble() +np.longfloat() + +np.csingle() +np.singlecomplex() +np.cdouble() +np.complex_() +np.cfloat() +np.clongdouble() +np.clongfloat() +np.longcomplex() + +b.item() +i8.item() +u8.item() +f8.item() +c16.item() +U.item() +S.item() + +b.tolist() +i8.tolist() +u8.tolist() +f8.tolist() +c16.tolist() +U.tolist() +S.tolist() + +b.ravel() +i8.ravel() +u8.ravel() +f8.ravel() +c16.ravel() +U.ravel() +S.ravel() + +b.flatten() +i8.flatten() +u8.flatten() +f8.flatten() +c16.flatten() +U.flatten() +S.flatten() + +b.reshape(1) +i8.reshape(1) +u8.reshape(1) +f8.reshape(1) +c16.reshape(1) +U.reshape(1) +S.reshape(1) diff --git a/numpy/typing/tests/data/pass/ufunclike.py b/numpy/typing/tests/data/pass/ufunclike.py new file mode 100644 index 000000000000..7eac89e8f9aa --- /dev/null +++ b/numpy/typing/tests/data/pass/ufunclike.py @@ -0,0 +1,46 @@ +from __future__ import annotations +from typing import Any +import numpy as np + + +class Object: + def __ceil__(self) -> Object: + return self + + def __floor__(self) -> Object: + return self + + def __ge__(self, value: object) -> bool: + return True + + def __array__(self) -> np.ndarray[Any, np.dtype[np.object_]]: + ret = np.empty((), dtype=object) + ret[()] = self + return ret + + +AR_LIKE_b = [True, True, False] +AR_LIKE_u = [np.uint32(1), np.uint32(2), np.uint32(3)] +AR_LIKE_i = [1, 2, 3] +AR_LIKE_f = [1.0, 2.0, 3.0] +AR_LIKE_O = [Object(), Object(), Object()] +AR_U: np.ndarray[Any, np.dtype[np.str_]] = np.zeros(3, dtype="U5") + +np.fix(AR_LIKE_b) +np.fix(AR_LIKE_u) +np.fix(AR_LIKE_i) +np.fix(AR_LIKE_f) +np.fix(AR_LIKE_O) +np.fix(AR_LIKE_f, out=AR_U) + +np.isposinf(AR_LIKE_b) +np.isposinf(AR_LIKE_u) +np.isposinf(AR_LIKE_i) +np.isposinf(AR_LIKE_f) +np.isposinf(AR_LIKE_f, out=AR_U) + +np.isneginf(AR_LIKE_b) +np.isneginf(AR_LIKE_u) +np.isneginf(AR_LIKE_i) +np.isneginf(AR_LIKE_f) +np.isneginf(AR_LIKE_f, out=AR_U) diff --git a/numpy/typing/tests/data/pass/ufuncs.py b/numpy/typing/tests/data/pass/ufuncs.py index ad4d483d4a70..3cc31ae5e305 100644 --- a/numpy/typing/tests/data/pass/ufuncs.py +++ b/numpy/typing/tests/data/pass/ufuncs.py @@ -4,7 +4,7 @@ np.sin([1, 2, 3]) np.sin(1, out=np.empty(1)) np.matmul(np.ones((2, 2, 2)), np.ones((2, 2, 2)), axes=[(0, 1), (0, 1), (0, 1)]) -np.sin(1, signature="D") +np.sin(1, signature="D->D") np.sin(1, extobj=[16, 1, lambda: None]) # NOTE: `np.generic` subclasses are not guaranteed to support addition; # re-enable this we can infer the exact return type of `np.sin(...)`. @@ -12,5 +12,6 @@ # np.sin(1) + np.sin(1) np.sin.types[0] np.sin.__name__ +np.sin.__doc__ np.abs(np.array([1])) diff --git a/numpy/typing/tests/data/reveal/arithmetic.py b/numpy/typing/tests/data/reveal/arithmetic.py index 20310e691647..f5d185206c05 100644 --- a/numpy/typing/tests/data/reveal/arithmetic.py +++ b/numpy/typing/tests/data/reveal/arithmetic.py @@ -1,4 +1,9 @@ +from typing import Any, List import numpy as np +import numpy.typing as npt + +# Can't directly import `np.float128` as it is not available on all platforms +f16: np.floating[npt._128Bit] c16 = np.complex128() f8 = np.float64() @@ -20,44 +25,284 @@ f = float() i = int() -AR = np.array([0], dtype=np.float64) -AR.setflags(write=False) +AR_b: np.ndarray[Any, np.dtype[np.bool_]] +AR_u: np.ndarray[Any, np.dtype[np.uint32]] +AR_i: np.ndarray[Any, np.dtype[np.int64]] +AR_f: np.ndarray[Any, np.dtype[np.float64]] +AR_c: np.ndarray[Any, np.dtype[np.complex128]] +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] +AR_M: np.ndarray[Any, np.dtype[np.datetime64]] +AR_O: np.ndarray[Any, np.dtype[np.object_]] + +AR_LIKE_b: List[bool] +AR_LIKE_u: List[np.uint32] +AR_LIKE_i: List[int] +AR_LIKE_f: List[float] +AR_LIKE_c: List[complex] +AR_LIKE_m: List[np.timedelta64] +AR_LIKE_M: List[np.datetime64] +AR_LIKE_O: List[np.object_] + +# Array subtraction + +reveal_type(AR_b - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_b - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_b - AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_b - AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_b - AR_LIKE_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_b - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_u - AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_LIKE_i - AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_f - AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c - AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_m - AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_M - AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_LIKE_O - AR_b) # E: Any + +reveal_type(AR_u - AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_u - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_u - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_u - AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_u - AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_u - AR_LIKE_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_u - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_LIKE_u - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_LIKE_i - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_f - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_m - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_M - AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_LIKE_O - AR_u) # E: Any + +reveal_type(AR_i - AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_i - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_i - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_i - AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_i - AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_i - AR_LIKE_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_i - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_u - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_i - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_f - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_m - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_M - AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_LIKE_O - AR_i) # E: Any + +reveal_type(AR_f - AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f - AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f - AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_f - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b - AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_u - AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_i - AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_f - AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c - AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_O - AR_f) # E: Any + +reveal_type(AR_c - AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c - AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c - AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b - AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_u - AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_i - AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_f - AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_c - AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_O - AR_c) # E: Any + +reveal_type(AR_m - AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m - AR_LIKE_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b - AR_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_u - AR_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_i - AR_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_m - AR_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_M - AR_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_LIKE_O - AR_m) # E: Any + +reveal_type(AR_M - AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_M - AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_M - AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_M - AR_LIKE_m) # E: numpy.ndarray[Any, numpy.dtype[numpy.datetime64]] +reveal_type(AR_M - AR_LIKE_M) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_M - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_M - AR_M) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_O - AR_M) # E: Any + +reveal_type(AR_O - AR_LIKE_b) # E: Any +reveal_type(AR_O - AR_LIKE_u) # E: Any +reveal_type(AR_O - AR_LIKE_i) # E: Any +reveal_type(AR_O - AR_LIKE_f) # E: Any +reveal_type(AR_O - AR_LIKE_c) # E: Any +reveal_type(AR_O - AR_LIKE_m) # E: Any +reveal_type(AR_O - AR_LIKE_M) # E: Any +reveal_type(AR_O - AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b - AR_O) # E: Any +reveal_type(AR_LIKE_u - AR_O) # E: Any +reveal_type(AR_LIKE_i - AR_O) # E: Any +reveal_type(AR_LIKE_f - AR_O) # E: Any +reveal_type(AR_LIKE_c - AR_O) # E: Any +reveal_type(AR_LIKE_m - AR_O) # E: Any +reveal_type(AR_LIKE_M - AR_O) # E: Any +reveal_type(AR_LIKE_O - AR_O) # E: Any + +# Array floor division + +reveal_type(AR_b // AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[{int8}]] +reveal_type(AR_b // AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_b // AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_b // AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_b // AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_b // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b // AR_b) # E: numpy.ndarray[Any, numpy.dtype[{int8}]] +reveal_type(AR_LIKE_u // AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_LIKE_i // AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_f // AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c // AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_O // AR_b) # E: Any + +reveal_type(AR_u // AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_u // AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_u // AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_u // AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_u // AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_u // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b // AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_LIKE_u // AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[Any]]] +reveal_type(AR_LIKE_i // AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_f // AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c // AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_m // AR_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_O // AR_u) # E: Any + +reveal_type(AR_i // AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_i // AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_i // AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_i // AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_i // AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_i // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b // AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_u // AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_i // AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] +reveal_type(AR_LIKE_f // AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c // AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_m // AR_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_O // AR_i) # E: Any + +reveal_type(AR_f // AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f // AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f // AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f // AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_f // AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_f // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b // AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_u // AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_i // AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_f // AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(AR_LIKE_c // AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_m // AR_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_LIKE_O // AR_f) # E: Any + +reveal_type(AR_c // AR_LIKE_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c // AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c // AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c // AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c // AR_LIKE_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_c // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b // AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_u // AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_i // AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_f // AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_c // AR_c) # E: numpy.ndarray[Any, numpy.dtype[numpy.complexfloating[Any, Any]]] +reveal_type(AR_LIKE_O // AR_c) # E: Any + +reveal_type(AR_m // AR_LIKE_u) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m // AR_LIKE_i) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m // AR_LIKE_f) # E: numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]] +reveal_type(AR_m // AR_LIKE_m) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] +reveal_type(AR_m // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_m // AR_m) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] +reveal_type(AR_LIKE_O // AR_m) # E: Any + +reveal_type(AR_O // AR_LIKE_b) # E: Any +reveal_type(AR_O // AR_LIKE_u) # E: Any +reveal_type(AR_O // AR_LIKE_i) # E: Any +reveal_type(AR_O // AR_LIKE_f) # E: Any +reveal_type(AR_O // AR_LIKE_c) # E: Any +reveal_type(AR_O // AR_LIKE_m) # E: Any +reveal_type(AR_O // AR_LIKE_M) # E: Any +reveal_type(AR_O // AR_LIKE_O) # E: Any + +reveal_type(AR_LIKE_b // AR_O) # E: Any +reveal_type(AR_LIKE_u // AR_O) # E: Any +reveal_type(AR_LIKE_i // AR_O) # E: Any +reveal_type(AR_LIKE_f // AR_O) # E: Any +reveal_type(AR_LIKE_c // AR_O) # E: Any +reveal_type(AR_LIKE_m // AR_O) # E: Any +reveal_type(AR_LIKE_M // AR_O) # E: Any +reveal_type(AR_LIKE_O // AR_O) # E: Any # unary ops -reveal_type(-c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(-c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(-f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(-f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(-i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(-i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(-u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(-u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(-f16) # E: {float128} +reveal_type(-c16) # E: {complex128} +reveal_type(-c8) # E: {complex64} +reveal_type(-f8) # E: {float64} +reveal_type(-f4) # E: {float32} +reveal_type(-i8) # E: {int64} +reveal_type(-i4) # E: {int32} +reveal_type(-u8) # E: {uint64} +reveal_type(-u4) # E: {uint32} reveal_type(-td) # E: numpy.timedelta64 -reveal_type(-AR) # E: Any - -reveal_type(+c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(+c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(+f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(+f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(+i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(+i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(+u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(+u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(-AR_f) # E: Any + +reveal_type(+f16) # E: {float128} +reveal_type(+c16) # E: {complex128} +reveal_type(+c8) # E: {complex64} +reveal_type(+f8) # E: {float64} +reveal_type(+f4) # E: {float32} +reveal_type(+i8) # E: {int64} +reveal_type(+i4) # E: {int32} +reveal_type(+u8) # E: {uint64} +reveal_type(+u4) # E: {uint32} reveal_type(+td) # E: numpy.timedelta64 -reveal_type(+AR) # E: Any - -reveal_type(abs(c16)) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(abs(c8)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(abs(f8)) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(abs(f4)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(abs(i8)) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(abs(i4)) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(abs(u8)) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(abs(u4)) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(+AR_f) # E: Any + +reveal_type(abs(f16)) # E: {float128} +reveal_type(abs(c16)) # E: {float64} +reveal_type(abs(c8)) # E: {float32} +reveal_type(abs(f8)) # E: {float64} +reveal_type(abs(f4)) # E: {float32} +reveal_type(abs(i8)) # E: {int64} +reveal_type(abs(i4)) # E: {int32} +reveal_type(abs(u8)) # E: {uint64} +reveal_type(abs(u4)) # E: {uint32} reveal_type(abs(td)) # E: numpy.timedelta64 reveal_type(abs(b_)) # E: numpy.bool_ -reveal_type(abs(AR)) # E: Any +reveal_type(abs(AR_f)) # E: Any # Time structures @@ -81,211 +326,221 @@ reveal_type(td / f) # E: numpy.timedelta64 reveal_type(td / f4) # E: numpy.timedelta64 reveal_type(td / f8) # E: numpy.timedelta64 -reveal_type(td / td) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(td // td) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(td / td) # E: {float64} +reveal_type(td // td) # E: {int64} # boolean -reveal_type(b_ / b) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / i) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / i8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / i4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / u8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / u4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(b_ / c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(b_ / c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(b_ / c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] - -reveal_type(b / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i4 / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(u8 / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(u4 / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 / b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 / b_) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(c / b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 / b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 / b_) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(b_ / b) # E: {float64} +reveal_type(b_ / b_) # E: {float64} +reveal_type(b_ / i) # E: {float64} +reveal_type(b_ / i8) # E: {float64} +reveal_type(b_ / i4) # E: {float64} +reveal_type(b_ / u8) # E: {float64} +reveal_type(b_ / u4) # E: {float64} +reveal_type(b_ / f) # E: {float64} +reveal_type(b_ / f16) # E: {float128} +reveal_type(b_ / f8) # E: {float64} +reveal_type(b_ / f4) # E: {float32} +reveal_type(b_ / c) # E: {complex128} +reveal_type(b_ / c16) # E: {complex128} +reveal_type(b_ / c8) # E: {complex64} + +reveal_type(b / b_) # E: {float64} +reveal_type(b_ / b_) # E: {float64} +reveal_type(i / b_) # E: {float64} +reveal_type(i8 / b_) # E: {float64} +reveal_type(i4 / b_) # E: {float64} +reveal_type(u8 / b_) # E: {float64} +reveal_type(u4 / b_) # E: {float64} +reveal_type(f / b_) # E: {float64} +reveal_type(f16 / b_) # E: {float128} +reveal_type(f8 / b_) # E: {float64} +reveal_type(f4 / b_) # E: {float32} +reveal_type(c / b_) # E: {complex128} +reveal_type(c16 / b_) # E: {complex128} +reveal_type(c8 / b_) # E: {complex64} # Complex -reveal_type(c16 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + f4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + i4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + b_) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + b) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + f) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c16 + i) # E: numpy.complexfloating[Any, Any] -reveal_type(c16 + AR) # E: Any - -reveal_type(c16 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(i8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f4 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(i4 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(b_ + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(b + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(i + c16) # E: numpy.complexfloating[Any, Any] -reveal_type(AR + c16) # E: Any - -reveal_type(c8 + c16) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(c8 + f4) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(c8 + i4) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(c8 + b_) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(c8 + b) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(c8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + f) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + i) # E: numpy.complexfloating[Any, Any] -reveal_type(c8 + AR) # E: Any - -reveal_type(c16 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f8 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(i8 + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(c8 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(f4 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(i4 + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(b_ + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(b + c8) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(c + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f + c8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(i + c8) # E: numpy.complexfloating[Any, Any] -reveal_type(AR + c8) # E: Any +reveal_type(c16 + f16) # E: {complex256} +reveal_type(c16 + c16) # E: {complex128} +reveal_type(c16 + f8) # E: {complex128} +reveal_type(c16 + i8) # E: {complex128} +reveal_type(c16 + c8) # E: {complex128} +reveal_type(c16 + f4) # E: {complex128} +reveal_type(c16 + i4) # E: {complex128} +reveal_type(c16 + b_) # E: {complex128} +reveal_type(c16 + b) # E: {complex128} +reveal_type(c16 + c) # E: {complex128} +reveal_type(c16 + f) # E: {complex128} +reveal_type(c16 + i) # E: {complex128} +reveal_type(c16 + AR_f) # E: Any + +reveal_type(f16 + c16) # E: {complex256} +reveal_type(c16 + c16) # E: {complex128} +reveal_type(f8 + c16) # E: {complex128} +reveal_type(i8 + c16) # E: {complex128} +reveal_type(c8 + c16) # E: {complex128} +reveal_type(f4 + c16) # E: {complex128} +reveal_type(i4 + c16) # E: {complex128} +reveal_type(b_ + c16) # E: {complex128} +reveal_type(b + c16) # E: {complex128} +reveal_type(c + c16) # E: {complex128} +reveal_type(f + c16) # E: {complex128} +reveal_type(i + c16) # E: {complex128} +reveal_type(AR_f + c16) # E: Any + +reveal_type(c8 + f16) # E: {complex256} +reveal_type(c8 + c16) # E: {complex128} +reveal_type(c8 + f8) # E: {complex128} +reveal_type(c8 + i8) # E: {complex128} +reveal_type(c8 + c8) # E: {complex64} +reveal_type(c8 + f4) # E: {complex64} +reveal_type(c8 + i4) # E: {complex64} +reveal_type(c8 + b_) # E: {complex64} +reveal_type(c8 + b) # E: {complex64} +reveal_type(c8 + c) # E: {complex128} +reveal_type(c8 + f) # E: {complex128} +reveal_type(c8 + i) # E: numpy.complexfloating[{_NBitInt}, {_NBitInt}] +reveal_type(c8 + AR_f) # E: Any + +reveal_type(f16 + c8) # E: {complex256} +reveal_type(c16 + c8) # E: {complex128} +reveal_type(f8 + c8) # E: {complex128} +reveal_type(i8 + c8) # E: {complex128} +reveal_type(c8 + c8) # E: {complex64} +reveal_type(f4 + c8) # E: {complex64} +reveal_type(i4 + c8) # E: {complex64} +reveal_type(b_ + c8) # E: {complex64} +reveal_type(b + c8) # E: {complex64} +reveal_type(c + c8) # E: {complex128} +reveal_type(f + c8) # E: {complex128} +reveal_type(i + c8) # E: numpy.complexfloating[{_NBitInt}, {_NBitInt}] +reveal_type(AR_f + c8) # E: Any # Float -reveal_type(f8 + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + i8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + f4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + i4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + b) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f8 + f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 + i) # E: numpy.floating[Any] -reveal_type(f8 + AR) # E: Any - -reveal_type(f8 + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i4 + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(c + f8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i + f8) # E: numpy.floating[Any] -reveal_type(AR + f8) # E: Any - -reveal_type(f4 + f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 + i8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 + f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(f4 + i4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(f4 + b_) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(f4 + b) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(f4 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f4 + f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 + i) # E: numpy.floating[Any] -reveal_type(f4 + AR) # E: Any - -reveal_type(f8 + f4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 + f4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 + f4) # E: umpy.floating[numpy.typing._32Bit] -reveal_type(i4 + f4) # E: umpy.floating[numpy.typing._32Bit] -reveal_type(b_ + f4) # E: umpy.floating[numpy.typing._32Bit] -reveal_type(b + f4) # E: umpy.floating[numpy.typing._32Bit] -reveal_type(c + f4) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f + f4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i + f4) # E: numpy.floating[Any] -reveal_type(AR + f4) # E: Any +reveal_type(f8 + f16) # E: {float128} +reveal_type(f8 + f8) # E: {float64} +reveal_type(f8 + i8) # E: {float64} +reveal_type(f8 + f4) # E: {float64} +reveal_type(f8 + i4) # E: {float64} +reveal_type(f8 + b_) # E: {float64} +reveal_type(f8 + b) # E: {float64} +reveal_type(f8 + c) # E: {complex128} +reveal_type(f8 + f) # E: {float64} +reveal_type(f8 + i) # E: {float64} +reveal_type(f8 + AR_f) # E: Any + +reveal_type(f16 + f8) # E: {float128} +reveal_type(f8 + f8) # E: {float64} +reveal_type(i8 + f8) # E: {float64} +reveal_type(f4 + f8) # E: {float64} +reveal_type(i4 + f8) # E: {float64} +reveal_type(b_ + f8) # E: {float64} +reveal_type(b + f8) # E: {float64} +reveal_type(c + f8) # E: {complex128} +reveal_type(f + f8) # E: {float64} +reveal_type(i + f8) # E: {float64} +reveal_type(AR_f + f8) # E: Any + +reveal_type(f4 + f16) # E: {float128} +reveal_type(f4 + f8) # E: {float64} +reveal_type(f4 + i8) # E: {float64} +reveal_type(f4 + f4) # E: {float32} +reveal_type(f4 + i4) # E: {float32} +reveal_type(f4 + b_) # E: {float32} +reveal_type(f4 + b) # E: {float32} +reveal_type(f4 + c) # E: {complex128} +reveal_type(f4 + f) # E: {float64} +reveal_type(f4 + i) # E: numpy.floating[{_NBitInt}] +reveal_type(f4 + AR_f) # E: Any + +reveal_type(f16 + f4) # E: {float128} +reveal_type(f8 + f4) # E: {float64} +reveal_type(i8 + f4) # E: {float64} +reveal_type(f4 + f4) # E: {float32} +reveal_type(i4 + f4) # E: {float32} +reveal_type(b_ + f4) # E: {float32} +reveal_type(b + f4) # E: {float32} +reveal_type(c + f4) # E: {complex128} +reveal_type(f + f4) # E: {float64} +reveal_type(i + f4) # E: numpy.floating[{_NBitInt}] +reveal_type(AR_f + f4) # E: Any # Int -reveal_type(i8 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(i8 + i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(i8 + b_) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 + b) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(i8 + f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 + i) # E: numpy.signedinteger[Any] -reveal_type(i8 + AR) # E: Any - -reveal_type(u8 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 + i4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u8 + u4) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 + b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 + b) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 + c) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(u8 + f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(u8 + i) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u8 + AR) # E: Any - -reveal_type(i8 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(u8 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(i4 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(u4 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(b_ + i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(b + i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(c + i8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f + i8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i + i8) # E: numpy.signedinteger[Any] -reveal_type(AR + i8) # E: Any - -reveal_type(u8 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(i4 + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u4 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(b_ + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(b + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(c + u8) # E: numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit] -reveal_type(f + u8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i + u8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(AR + u8) # E: Any - -reveal_type(i4 + i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i4 + i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 + i) # E: numpy.signedinteger[Any] -reveal_type(i4 + b_) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 + b) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 + AR) # E: Any - -reveal_type(u4 + i8) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u4 + i4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u4 + u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u4 + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 + i) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u4 + b_) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 + b) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 + AR) # E: Any - -reveal_type(i8 + i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i4 + i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i + i4) # E: numpy.signedinteger[Any] -reveal_type(b_ + i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(b + i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(AR + i4) # E: Any - -reveal_type(i8 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(i4 + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(u8 + u4) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u4 + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(b_ + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(b + u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(i + u4) # E: Union[numpy.signedinteger[Any], numpy.floating[numpy.typing._64Bit]] -reveal_type(AR + u4) # E: Any +reveal_type(i8 + i8) # E: {int64} +reveal_type(i8 + u8) # E: Any +reveal_type(i8 + i4) # E: {int64} +reveal_type(i8 + u4) # E: Any +reveal_type(i8 + b_) # E: {int64} +reveal_type(i8 + b) # E: {int64} +reveal_type(i8 + c) # E: {complex128} +reveal_type(i8 + f) # E: {float64} +reveal_type(i8 + i) # E: {int64} +reveal_type(i8 + AR_f) # E: Any + +reveal_type(u8 + u8) # E: {uint64} +reveal_type(u8 + i4) # E: Any +reveal_type(u8 + u4) # E: {uint64} +reveal_type(u8 + b_) # E: {uint64} +reveal_type(u8 + b) # E: {uint64} +reveal_type(u8 + c) # E: {complex128} +reveal_type(u8 + f) # E: {float64} +reveal_type(u8 + i) # E: Any +reveal_type(u8 + AR_f) # E: Any + +reveal_type(i8 + i8) # E: {int64} +reveal_type(u8 + i8) # E: Any +reveal_type(i4 + i8) # E: {int64} +reveal_type(u4 + i8) # E: Any +reveal_type(b_ + i8) # E: {int64} +reveal_type(b + i8) # E: {int64} +reveal_type(c + i8) # E: {complex128} +reveal_type(f + i8) # E: {float64} +reveal_type(i + i8) # E: {int64} +reveal_type(AR_f + i8) # E: Any + +reveal_type(u8 + u8) # E: {uint64} +reveal_type(i4 + u8) # E: Any +reveal_type(u4 + u8) # E: {uint64} +reveal_type(b_ + u8) # E: {uint64} +reveal_type(b + u8) # E: {uint64} +reveal_type(c + u8) # E: {complex128} +reveal_type(f + u8) # E: {float64} +reveal_type(i + u8) # E: Any +reveal_type(AR_f + u8) # E: Any + +reveal_type(i4 + i8) # E: {int64} +reveal_type(i4 + i4) # E: {int32} +reveal_type(i4 + i) # E: {int_} +reveal_type(i4 + b_) # E: {int32} +reveal_type(i4 + b) # E: {int32} +reveal_type(i4 + AR_f) # E: Any + +reveal_type(u4 + i8) # E: Any +reveal_type(u4 + i4) # E: Any +reveal_type(u4 + u8) # E: {uint64} +reveal_type(u4 + u4) # E: {uint32} +reveal_type(u4 + i) # E: Any +reveal_type(u4 + b_) # E: {uint32} +reveal_type(u4 + b) # E: {uint32} +reveal_type(u4 + AR_f) # E: Any + +reveal_type(i8 + i4) # E: {int64} +reveal_type(i4 + i4) # E: {int32} +reveal_type(i + i4) # E: {int_} +reveal_type(b_ + i4) # E: {int32} +reveal_type(b + i4) # E: {int32} +reveal_type(AR_f + i4) # E: Any + +reveal_type(i8 + u4) # E: Any +reveal_type(i4 + u4) # E: Any +reveal_type(u8 + u4) # E: {uint64} +reveal_type(u4 + u4) # E: {uint32} +reveal_type(b_ + u4) # E: {uint32} +reveal_type(b + u4) # E: {uint32} +reveal_type(i + u4) # E: Any +reveal_type(AR_f + u4) # E: Any diff --git a/numpy/typing/tests/data/reveal/array_constructors.py b/numpy/typing/tests/data/reveal/array_constructors.py index 106174736085..2e803a365ce8 100644 --- a/numpy/typing/tests/data/reveal/array_constructors.py +++ b/numpy/typing/tests/data/reveal/array_constructors.py @@ -11,92 +11,92 @@ class SubClass(np.ndarray): ... def func(i: int, j: int, **kwargs: Any) -> SubClass: ... -reveal_type(np.asarray(A)) # E: ndarray -reveal_type(np.asarray(B)) # E: ndarray -reveal_type(np.asarray(C)) # E: ndarray +reveal_type(np.asarray(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.asarray(B)) # E: numpy.ndarray[Any, Any] +reveal_type(np.asarray(C)) # E: numpy.ndarray[Any, Any] -reveal_type(np.asanyarray(A)) # E: ndarray +reveal_type(np.asanyarray(A)) # E: numpy.ndarray[Any, Any] reveal_type(np.asanyarray(B)) # E: SubClass -reveal_type(np.asanyarray(B, dtype=int)) # E: ndarray -reveal_type(np.asanyarray(C)) # E: ndarray +reveal_type(np.asanyarray(B, dtype=int)) # E: numpy.ndarray[Any, Any] +reveal_type(np.asanyarray(C)) # E: numpy.ndarray[Any, Any] -reveal_type(np.ascontiguousarray(A)) # E: ndarray -reveal_type(np.ascontiguousarray(B)) # E: ndarray -reveal_type(np.ascontiguousarray(C)) # E: ndarray +reveal_type(np.ascontiguousarray(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ascontiguousarray(B)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ascontiguousarray(C)) # E: numpy.ndarray[Any, Any] -reveal_type(np.asfortranarray(A)) # E: ndarray -reveal_type(np.asfortranarray(B)) # E: ndarray -reveal_type(np.asfortranarray(C)) # E: ndarray +reveal_type(np.asfortranarray(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.asfortranarray(B)) # E: numpy.ndarray[Any, Any] +reveal_type(np.asfortranarray(C)) # E: numpy.ndarray[Any, Any] -reveal_type(np.require(A)) # E: ndarray +reveal_type(np.require(A)) # E: numpy.ndarray[Any, Any] reveal_type(np.require(B)) # E: SubClass reveal_type(np.require(B, requirements=None)) # E: SubClass -reveal_type(np.require(B, dtype=int)) # E: ndarray -reveal_type(np.require(B, requirements="E")) # E: ndarray -reveal_type(np.require(B, requirements=["ENSUREARRAY"])) # E: ndarray -reveal_type(np.require(B, requirements={"F", "E"})) # E: ndarray +reveal_type(np.require(B, dtype=int)) # E: numpy.ndarray[Any, Any] +reveal_type(np.require(B, requirements="E")) # E: numpy.ndarray[Any, Any] +reveal_type(np.require(B, requirements=["ENSUREARRAY"])) # E: numpy.ndarray[Any, Any] +reveal_type(np.require(B, requirements={"F", "E"})) # E: numpy.ndarray[Any, Any] reveal_type(np.require(B, requirements=["C", "OWNDATA"])) # E: SubClass reveal_type(np.require(B, requirements="W")) # E: SubClass reveal_type(np.require(B, requirements="A")) # E: SubClass -reveal_type(np.require(C)) # E: ndarray +reveal_type(np.require(C)) # E: numpy.ndarray[Any, Any] -reveal_type(np.linspace(0, 10)) # E: numpy.ndarray -reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray, numpy.inexact[Any]] -reveal_type(np.logspace(0, 10)) # E: numpy.ndarray -reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray +reveal_type(np.linspace(0, 10)) # E: numpy.ndarray[Any, Any] +reveal_type(np.linspace(0, 10, retstep=True)) # E: Tuple[numpy.ndarray[Any, Any], Any] +reveal_type(np.logspace(0, 10)) # E: numpy.ndarray[Any, Any] +reveal_type(np.geomspace(1, 10)) # E: numpy.ndarray[Any, Any] -reveal_type(np.zeros_like(A)) # E: numpy.ndarray -reveal_type(np.zeros_like(C)) # E: numpy.ndarray +reveal_type(np.zeros_like(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.zeros_like(C)) # E: numpy.ndarray[Any, Any] reveal_type(np.zeros_like(B)) # E: SubClass -reveal_type(np.zeros_like(B, dtype=np.int64)) # E: numpy.ndarray +reveal_type(np.zeros_like(B, dtype=np.int64)) # E: numpy.ndarray[Any, Any] -reveal_type(np.ones_like(A)) # E: numpy.ndarray -reveal_type(np.ones_like(C)) # E: numpy.ndarray +reveal_type(np.ones_like(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ones_like(C)) # E: numpy.ndarray[Any, Any] reveal_type(np.ones_like(B)) # E: SubClass -reveal_type(np.ones_like(B, dtype=np.int64)) # E: numpy.ndarray +reveal_type(np.ones_like(B, dtype=np.int64)) # E: numpy.ndarray[Any, Any] -reveal_type(np.empty_like(A)) # E: numpy.ndarray -reveal_type(np.empty_like(C)) # E: numpy.ndarray +reveal_type(np.empty_like(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.empty_like(C)) # E: numpy.ndarray[Any, Any] reveal_type(np.empty_like(B)) # E: SubClass -reveal_type(np.empty_like(B, dtype=np.int64)) # E: numpy.ndarray +reveal_type(np.empty_like(B, dtype=np.int64)) # E: numpy.ndarray[Any, Any] -reveal_type(np.full_like(A, i8)) # E: numpy.ndarray -reveal_type(np.full_like(C, i8)) # E: numpy.ndarray +reveal_type(np.full_like(A, i8)) # E: numpy.ndarray[Any, Any] +reveal_type(np.full_like(C, i8)) # E: numpy.ndarray[Any, Any] reveal_type(np.full_like(B, i8)) # E: SubClass -reveal_type(np.full_like(B, i8, dtype=np.int64)) # E: numpy.ndarray +reveal_type(np.full_like(B, i8, dtype=np.int64)) # E: numpy.ndarray[Any, Any] -reveal_type(np.ones(1)) # E: numpy.ndarray -reveal_type(np.ones([1, 1, 1])) # E: numpy.ndarray +reveal_type(np.ones(1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ones([1, 1, 1])) # E: numpy.ndarray[Any, Any] -reveal_type(np.full(1, i8)) # E: numpy.ndarray -reveal_type(np.full([1, 1, 1], i8)) # E: numpy.ndarray +reveal_type(np.full(1, i8)) # E: numpy.ndarray[Any, Any] +reveal_type(np.full([1, 1, 1], i8)) # E: numpy.ndarray[Any, Any] -reveal_type(np.indices([1, 2, 3])) # E: numpy.ndarray -reveal_type(np.indices([1, 2, 3], sparse=True)) # E: tuple[numpy.ndarray] +reveal_type(np.indices([1, 2, 3])) # E: numpy.ndarray[Any, Any] +reveal_type(np.indices([1, 2, 3], sparse=True)) # E: tuple[numpy.ndarray[Any, Any]] reveal_type(np.fromfunction(func, (3, 5))) # E: SubClass -reveal_type(np.identity(10)) # E: numpy.ndarray +reveal_type(np.identity(10)) # E: numpy.ndarray[Any, Any] -reveal_type(np.atleast_1d(A)) # E: numpy.ndarray -reveal_type(np.atleast_1d(C)) # E: numpy.ndarray -reveal_type(np.atleast_1d(A, A)) # E: list[numpy.ndarray] -reveal_type(np.atleast_1d(A, C)) # E: list[numpy.ndarray] -reveal_type(np.atleast_1d(C, C)) # E: list[numpy.ndarray] +reveal_type(np.atleast_1d(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.atleast_1d(C)) # E: numpy.ndarray[Any, Any] +reveal_type(np.atleast_1d(A, A)) # E: list[numpy.ndarray[Any, Any]] +reveal_type(np.atleast_1d(A, C)) # E: list[numpy.ndarray[Any, Any]] +reveal_type(np.atleast_1d(C, C)) # E: list[numpy.ndarray[Any, Any]] -reveal_type(np.atleast_2d(A)) # E: numpy.ndarray +reveal_type(np.atleast_2d(A)) # E: numpy.ndarray[Any, Any] -reveal_type(np.atleast_3d(A)) # E: numpy.ndarray +reveal_type(np.atleast_3d(A)) # E: numpy.ndarray[Any, Any] -reveal_type(np.vstack([A, A])) # E: numpy.ndarray -reveal_type(np.vstack([A, C])) # E: numpy.ndarray -reveal_type(np.vstack([C, C])) # E: numpy.ndarray +reveal_type(np.vstack([A, A])) # E: numpy.ndarray[Any, Any] +reveal_type(np.vstack([A, C])) # E: numpy.ndarray[Any, Any] +reveal_type(np.vstack([C, C])) # E: numpy.ndarray[Any, Any] -reveal_type(np.hstack([A, A])) # E: numpy.ndarray +reveal_type(np.hstack([A, A])) # E: numpy.ndarray[Any, Any] -reveal_type(np.stack([A, A])) # E: numpy.ndarray -reveal_type(np.stack([A, A], axis=0)) # E: numpy.ndarray +reveal_type(np.stack([A, A])) # E: numpy.ndarray[Any, Any] +reveal_type(np.stack([A, A], axis=0)) # E: numpy.ndarray[Any, Any] reveal_type(np.stack([A, A], out=B)) # E: SubClass -reveal_type(np.block([[A, A], [A, A]])) # E: numpy.ndarray -reveal_type(np.block(C)) # E: numpy.ndarray +reveal_type(np.block([[A, A], [A, A]])) # E: numpy.ndarray[Any, Any] +reveal_type(np.block(C)) # E: numpy.ndarray[Any, Any] diff --git a/numpy/typing/tests/data/reveal/arrayprint.py b/numpy/typing/tests/data/reveal/arrayprint.py new file mode 100644 index 000000000000..e797097ebb94 --- /dev/null +++ b/numpy/typing/tests/data/reveal/arrayprint.py @@ -0,0 +1,19 @@ +from typing import Any, Callable +import numpy as np + +AR: np.ndarray[Any, Any] +func_float: Callable[[np.floating[Any]], str] +func_int: Callable[[np.integer[Any]], str] + +reveal_type(np.get_printoptions()) # E: TypedDict +reveal_type(np.array2string( # E: str + AR, formatter={'float_kind': func_float, 'int_kind': func_int} +)) +reveal_type(np.format_float_scientific(1.0)) # E: str +reveal_type(np.format_float_positional(1)) # E: str +reveal_type(np.array_repr(AR)) # E: str +reveal_type(np.array_str(AR)) # E: str + +reveal_type(np.printoptions()) # E: contextlib._GeneratorContextManager +with np.printoptions() as dct: + reveal_type(dct) # E: TypedDict diff --git a/numpy/typing/tests/data/reveal/arrayterator.py b/numpy/typing/tests/data/reveal/arrayterator.py new file mode 100644 index 000000000000..b57861d00f05 --- /dev/null +++ b/numpy/typing/tests/data/reveal/arrayterator.py @@ -0,0 +1,24 @@ +from typing import Any +import numpy as np + +AR_i8: np.ndarray[Any, np.dtype[np.int64]] +ar_iter = np.lib.Arrayterator(AR_i8) + +reveal_type(ar_iter.var) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] +reveal_type(ar_iter.buf_size) # E: Union[None, builtins.int] +reveal_type(ar_iter.start) # E: builtins.list[builtins.int] +reveal_type(ar_iter.stop) # E: builtins.list[builtins.int] +reveal_type(ar_iter.step) # E: builtins.list[builtins.int] +reveal_type(ar_iter.shape) # E: builtins.tuple[builtins.int] +reveal_type(ar_iter.flat) # E: 'typing.Generator[{int64}, None, None] + +reveal_type(ar_iter.__array__()) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] + +for i in ar_iter: + reveal_type(i) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] + +reveal_type(ar_iter[0]) # E: numpy.lib.arrayterator.Arrayterator[Any, numpy.dtype[{int64}]] +reveal_type(ar_iter[...]) # E: numpy.lib.arrayterator.Arrayterator[Any, numpy.dtype[{int64}]] +reveal_type(ar_iter[:]) # E: numpy.lib.arrayterator.Arrayterator[Any, numpy.dtype[{int64}]] +reveal_type(ar_iter[0, 0, 0]) # E: numpy.lib.arrayterator.Arrayterator[Any, numpy.dtype[{int64}]] +reveal_type(ar_iter[..., 0, :]) # E: numpy.lib.arrayterator.Arrayterator[Any, numpy.dtype[{int64}]] diff --git a/numpy/typing/tests/data/reveal/bitwise_ops.py b/numpy/typing/tests/data/reveal/bitwise_ops.py index cb9131a96938..6b9969568c8e 100644 --- a/numpy/typing/tests/data/reveal/bitwise_ops.py +++ b/numpy/typing/tests/data/reveal/bitwise_ops.py @@ -15,11 +15,11 @@ AR.setflags(write=False) -reveal_type(i8 << i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 >> i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 | i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 ^ i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 & i8) # E: numpy.signedinteger[numpy.typing._64Bit] +reveal_type(i8 << i8) # E: {int64} +reveal_type(i8 >> i8) # E: {int64} +reveal_type(i8 | i8) # E: {int64} +reveal_type(i8 ^ i8) # E: {int64} +reveal_type(i8 & i8) # E: {int64} reveal_type(i8 << AR) # E: Any reveal_type(i8 >> AR) # E: Any @@ -27,41 +27,41 @@ reveal_type(i8 ^ AR) # E: Any reveal_type(i8 & AR) # E: Any -reveal_type(i4 << i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 >> i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 | i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 ^ i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 & i4) # E: numpy.signedinteger[numpy.typing._32Bit] - -reveal_type(i8 << i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 >> i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 | i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 ^ i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 & i4) # E: numpy.signedinteger[numpy.typing._64Bit] - -reveal_type(i8 << i) # E: numpy.signedinteger[Any] -reveal_type(i8 >> i) # E: numpy.signedinteger[Any] -reveal_type(i8 | i) # E: numpy.signedinteger[Any] -reveal_type(i8 ^ i) # E: numpy.signedinteger[Any] -reveal_type(i8 & i) # E: numpy.signedinteger[Any] - -reveal_type(i8 << b_) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 >> b_) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 | b_) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 ^ b_) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 & b_) # E: numpy.signedinteger[numpy.typing._64Bit] - -reveal_type(i8 << b) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 >> b) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 | b) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 ^ b) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 & b) # E: numpy.signedinteger[numpy.typing._64Bit] - -reveal_type(u8 << u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 >> u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 | u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 ^ u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 & u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(i4 << i4) # E: {int32} +reveal_type(i4 >> i4) # E: {int32} +reveal_type(i4 | i4) # E: {int32} +reveal_type(i4 ^ i4) # E: {int32} +reveal_type(i4 & i4) # E: {int32} + +reveal_type(i8 << i4) # E: {int64} +reveal_type(i8 >> i4) # E: {int64} +reveal_type(i8 | i4) # E: {int64} +reveal_type(i8 ^ i4) # E: {int64} +reveal_type(i8 & i4) # E: {int64} + +reveal_type(i8 << i) # E: {int64} +reveal_type(i8 >> i) # E: {int64} +reveal_type(i8 | i) # E: {int64} +reveal_type(i8 ^ i) # E: {int64} +reveal_type(i8 & i) # E: {int64} + +reveal_type(i8 << b_) # E: {int64} +reveal_type(i8 >> b_) # E: {int64} +reveal_type(i8 | b_) # E: {int64} +reveal_type(i8 ^ b_) # E: {int64} +reveal_type(i8 & b_) # E: {int64} + +reveal_type(i8 << b) # E: {int64} +reveal_type(i8 >> b) # E: {int64} +reveal_type(i8 | b) # E: {int64} +reveal_type(i8 ^ b) # E: {int64} +reveal_type(i8 & b) # E: {int64} + +reveal_type(u8 << u8) # E: {uint64} +reveal_type(u8 >> u8) # E: {uint64} +reveal_type(u8 | u8) # E: {uint64} +reveal_type(u8 ^ u8) # E: {uint64} +reveal_type(u8 & u8) # E: {uint64} reveal_type(u8 << AR) # E: Any reveal_type(u8 >> AR) # E: Any @@ -69,11 +69,11 @@ reveal_type(u8 ^ AR) # E: Any reveal_type(u8 & AR) # E: Any -reveal_type(u4 << u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 >> u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 | u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 ^ u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] -reveal_type(u4 & u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(u4 << u4) # E: {uint32} +reveal_type(u4 >> u4) # E: {uint32} +reveal_type(u4 | u4) # E: {uint32} +reveal_type(u4 ^ u4) # E: {uint32} +reveal_type(u4 & u4) # E: {uint32} reveal_type(u4 << i4) # E: numpy.signedinteger[Any] reveal_type(u4 >> i4) # E: numpy.signedinteger[Any] @@ -87,20 +87,20 @@ reveal_type(u4 ^ i) # E: numpy.signedinteger[Any] reveal_type(u4 & i) # E: numpy.signedinteger[Any] -reveal_type(u8 << b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 >> b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 | b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 ^ b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 & b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 << b_) # E: {uint64} +reveal_type(u8 >> b_) # E: {uint64} +reveal_type(u8 | b_) # E: {uint64} +reveal_type(u8 ^ b_) # E: {uint64} +reveal_type(u8 & b_) # E: {uint64} -reveal_type(u8 << b) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 >> b) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 | b) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 ^ b) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(u8 & b) # E: numpy.unsignedinteger[numpy.typing._64Bit] +reveal_type(u8 << b) # E: {uint64} +reveal_type(u8 >> b) # E: {uint64} +reveal_type(u8 | b) # E: {uint64} +reveal_type(u8 ^ b) # E: {uint64} +reveal_type(u8 & b) # E: {uint64} -reveal_type(b_ << b_) # E: numpy.signedinteger[numpy.typing._8Bit] -reveal_type(b_ >> b_) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ << b_) # E: {int8} +reveal_type(b_ >> b_) # E: {int8} reveal_type(b_ | b_) # E: numpy.bool_ reveal_type(b_ ^ b_) # E: numpy.bool_ reveal_type(b_ & b_) # E: numpy.bool_ @@ -111,21 +111,21 @@ reveal_type(b_ ^ AR) # E: Any reveal_type(b_ & AR) # E: Any -reveal_type(b_ << b) # E: numpy.signedinteger[numpy.typing._8Bit] -reveal_type(b_ >> b) # E: numpy.signedinteger[numpy.typing._8Bit] +reveal_type(b_ << b) # E: {int8} +reveal_type(b_ >> b) # E: {int8} reveal_type(b_ | b) # E: numpy.bool_ reveal_type(b_ ^ b) # E: numpy.bool_ reveal_type(b_ & b) # E: numpy.bool_ -reveal_type(b_ << i) # E: numpy.signedinteger[Any] -reveal_type(b_ >> i) # E: numpy.signedinteger[Any] -reveal_type(b_ | i) # E: numpy.signedinteger[Any] -reveal_type(b_ ^ i) # E: numpy.signedinteger[Any] -reveal_type(b_ & i) # E: numpy.signedinteger[Any] +reveal_type(b_ << i) # E: {int_} +reveal_type(b_ >> i) # E: {int_} +reveal_type(b_ | i) # E: {int_} +reveal_type(b_ ^ i) # E: {int_} +reveal_type(b_ & i) # E: {int_} -reveal_type(~i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(~i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(~u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(~u4) # E: numpy.unsignedinteger[numpy.typing._32Bit] +reveal_type(~i8) # E: {int64} +reveal_type(~i4) # E: {int32} +reveal_type(~u8) # E: {uint64} +reveal_type(~u4) # E: {uint32} reveal_type(~b_) # E: numpy.bool_ reveal_type(~AR) # E: Any diff --git a/numpy/typing/tests/data/reveal/comparisons.py b/numpy/typing/tests/data/reveal/comparisons.py new file mode 100644 index 000000000000..16f21cc39349 --- /dev/null +++ b/numpy/typing/tests/data/reveal/comparisons.py @@ -0,0 +1,252 @@ +import numpy as np + +c16 = np.complex128() +f8 = np.float64() +i8 = np.int64() +u8 = np.uint64() + +c8 = np.complex64() +f4 = np.float32() +i4 = np.int32() +u4 = np.uint32() + +dt = np.datetime64(0, "D") +td = np.timedelta64(0, "D") + +b_ = np.bool_() + +b = bool() +c = complex() +f = float() +i = int() + +AR = np.array([0], dtype=np.int64) +AR.setflags(write=False) + +SEQ = (0, 1, 2, 3, 4) + +# Time structures + +reveal_type(dt > dt) # E: numpy.bool_ + +reveal_type(td > td) # E: numpy.bool_ +reveal_type(td > i) # E: numpy.bool_ +reveal_type(td > i4) # E: numpy.bool_ +reveal_type(td > i8) # E: numpy.bool_ + +reveal_type(td > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(td > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(AR > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(AR > td) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > td) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +# boolean + +reveal_type(b_ > b) # E: numpy.bool_ +reveal_type(b_ > b_) # E: numpy.bool_ +reveal_type(b_ > i) # E: numpy.bool_ +reveal_type(b_ > i8) # E: numpy.bool_ +reveal_type(b_ > i4) # E: numpy.bool_ +reveal_type(b_ > u8) # E: numpy.bool_ +reveal_type(b_ > u4) # E: numpy.bool_ +reveal_type(b_ > f) # E: numpy.bool_ +reveal_type(b_ > f8) # E: numpy.bool_ +reveal_type(b_ > f4) # E: numpy.bool_ +reveal_type(b_ > c) # E: numpy.bool_ +reveal_type(b_ > c16) # E: numpy.bool_ +reveal_type(b_ > c8) # E: numpy.bool_ +reveal_type(b_ > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(b_ > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +# Complex + +reveal_type(c16 > c16) # E: numpy.bool_ +reveal_type(c16 > f8) # E: numpy.bool_ +reveal_type(c16 > i8) # E: numpy.bool_ +reveal_type(c16 > c8) # E: numpy.bool_ +reveal_type(c16 > f4) # E: numpy.bool_ +reveal_type(c16 > i4) # E: numpy.bool_ +reveal_type(c16 > b_) # E: numpy.bool_ +reveal_type(c16 > b) # E: numpy.bool_ +reveal_type(c16 > c) # E: numpy.bool_ +reveal_type(c16 > f) # E: numpy.bool_ +reveal_type(c16 > i) # E: numpy.bool_ +reveal_type(c16 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(c16 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(c16 > c16) # E: numpy.bool_ +reveal_type(f8 > c16) # E: numpy.bool_ +reveal_type(i8 > c16) # E: numpy.bool_ +reveal_type(c8 > c16) # E: numpy.bool_ +reveal_type(f4 > c16) # E: numpy.bool_ +reveal_type(i4 > c16) # E: numpy.bool_ +reveal_type(b_ > c16) # E: numpy.bool_ +reveal_type(b > c16) # E: numpy.bool_ +reveal_type(c > c16) # E: numpy.bool_ +reveal_type(f > c16) # E: numpy.bool_ +reveal_type(i > c16) # E: numpy.bool_ +reveal_type(AR > c16) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > c16) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(c8 > c16) # E: numpy.bool_ +reveal_type(c8 > f8) # E: numpy.bool_ +reveal_type(c8 > i8) # E: numpy.bool_ +reveal_type(c8 > c8) # E: numpy.bool_ +reveal_type(c8 > f4) # E: numpy.bool_ +reveal_type(c8 > i4) # E: numpy.bool_ +reveal_type(c8 > b_) # E: numpy.bool_ +reveal_type(c8 > b) # E: numpy.bool_ +reveal_type(c8 > c) # E: numpy.bool_ +reveal_type(c8 > f) # E: numpy.bool_ +reveal_type(c8 > i) # E: numpy.bool_ +reveal_type(c8 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(c8 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(c16 > c8) # E: numpy.bool_ +reveal_type(f8 > c8) # E: numpy.bool_ +reveal_type(i8 > c8) # E: numpy.bool_ +reveal_type(c8 > c8) # E: numpy.bool_ +reveal_type(f4 > c8) # E: numpy.bool_ +reveal_type(i4 > c8) # E: numpy.bool_ +reveal_type(b_ > c8) # E: numpy.bool_ +reveal_type(b > c8) # E: numpy.bool_ +reveal_type(c > c8) # E: numpy.bool_ +reveal_type(f > c8) # E: numpy.bool_ +reveal_type(i > c8) # E: numpy.bool_ +reveal_type(AR > c8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > c8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +# Float + +reveal_type(f8 > f8) # E: numpy.bool_ +reveal_type(f8 > i8) # E: numpy.bool_ +reveal_type(f8 > f4) # E: numpy.bool_ +reveal_type(f8 > i4) # E: numpy.bool_ +reveal_type(f8 > b_) # E: numpy.bool_ +reveal_type(f8 > b) # E: numpy.bool_ +reveal_type(f8 > c) # E: numpy.bool_ +reveal_type(f8 > f) # E: numpy.bool_ +reveal_type(f8 > i) # E: numpy.bool_ +reveal_type(f8 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(f8 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(f8 > f8) # E: numpy.bool_ +reveal_type(i8 > f8) # E: numpy.bool_ +reveal_type(f4 > f8) # E: numpy.bool_ +reveal_type(i4 > f8) # E: numpy.bool_ +reveal_type(b_ > f8) # E: numpy.bool_ +reveal_type(b > f8) # E: numpy.bool_ +reveal_type(c > f8) # E: numpy.bool_ +reveal_type(f > f8) # E: numpy.bool_ +reveal_type(i > f8) # E: numpy.bool_ +reveal_type(AR > f8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > f8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(f4 > f8) # E: numpy.bool_ +reveal_type(f4 > i8) # E: numpy.bool_ +reveal_type(f4 > f4) # E: numpy.bool_ +reveal_type(f4 > i4) # E: numpy.bool_ +reveal_type(f4 > b_) # E: numpy.bool_ +reveal_type(f4 > b) # E: numpy.bool_ +reveal_type(f4 > c) # E: numpy.bool_ +reveal_type(f4 > f) # E: numpy.bool_ +reveal_type(f4 > i) # E: numpy.bool_ +reveal_type(f4 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(f4 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(f8 > f4) # E: numpy.bool_ +reveal_type(i8 > f4) # E: numpy.bool_ +reveal_type(f4 > f4) # E: numpy.bool_ +reveal_type(i4 > f4) # E: numpy.bool_ +reveal_type(b_ > f4) # E: numpy.bool_ +reveal_type(b > f4) # E: numpy.bool_ +reveal_type(c > f4) # E: numpy.bool_ +reveal_type(f > f4) # E: numpy.bool_ +reveal_type(i > f4) # E: numpy.bool_ +reveal_type(AR > f4) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > f4) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +# Int + +reveal_type(i8 > i8) # E: numpy.bool_ +reveal_type(i8 > u8) # E: numpy.bool_ +reveal_type(i8 > i4) # E: numpy.bool_ +reveal_type(i8 > u4) # E: numpy.bool_ +reveal_type(i8 > b_) # E: numpy.bool_ +reveal_type(i8 > b) # E: numpy.bool_ +reveal_type(i8 > c) # E: numpy.bool_ +reveal_type(i8 > f) # E: numpy.bool_ +reveal_type(i8 > i) # E: numpy.bool_ +reveal_type(i8 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(i8 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(u8 > u8) # E: numpy.bool_ +reveal_type(u8 > i4) # E: numpy.bool_ +reveal_type(u8 > u4) # E: numpy.bool_ +reveal_type(u8 > b_) # E: numpy.bool_ +reveal_type(u8 > b) # E: numpy.bool_ +reveal_type(u8 > c) # E: numpy.bool_ +reveal_type(u8 > f) # E: numpy.bool_ +reveal_type(u8 > i) # E: numpy.bool_ +reveal_type(u8 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(u8 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(i8 > i8) # E: numpy.bool_ +reveal_type(u8 > i8) # E: numpy.bool_ +reveal_type(i4 > i8) # E: numpy.bool_ +reveal_type(u4 > i8) # E: numpy.bool_ +reveal_type(b_ > i8) # E: numpy.bool_ +reveal_type(b > i8) # E: numpy.bool_ +reveal_type(c > i8) # E: numpy.bool_ +reveal_type(f > i8) # E: numpy.bool_ +reveal_type(i > i8) # E: numpy.bool_ +reveal_type(AR > i8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > i8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(u8 > u8) # E: numpy.bool_ +reveal_type(i4 > u8) # E: numpy.bool_ +reveal_type(u4 > u8) # E: numpy.bool_ +reveal_type(b_ > u8) # E: numpy.bool_ +reveal_type(b > u8) # E: numpy.bool_ +reveal_type(c > u8) # E: numpy.bool_ +reveal_type(f > u8) # E: numpy.bool_ +reveal_type(i > u8) # E: numpy.bool_ +reveal_type(AR > u8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > u8) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(i4 > i8) # E: numpy.bool_ +reveal_type(i4 > i4) # E: numpy.bool_ +reveal_type(i4 > i) # E: numpy.bool_ +reveal_type(i4 > b_) # E: numpy.bool_ +reveal_type(i4 > b) # E: numpy.bool_ +reveal_type(i4 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(i4 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(u4 > i8) # E: numpy.bool_ +reveal_type(u4 > i4) # E: numpy.bool_ +reveal_type(u4 > u8) # E: numpy.bool_ +reveal_type(u4 > u4) # E: numpy.bool_ +reveal_type(u4 > i) # E: numpy.bool_ +reveal_type(u4 > b_) # E: numpy.bool_ +reveal_type(u4 > b) # E: numpy.bool_ +reveal_type(u4 > AR) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(u4 > SEQ) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(i8 > i4) # E: numpy.bool_ +reveal_type(i4 > i4) # E: numpy.bool_ +reveal_type(i > i4) # E: numpy.bool_ +reveal_type(b_ > i4) # E: numpy.bool_ +reveal_type(b > i4) # E: numpy.bool_ +reveal_type(AR > i4) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > i4) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] + +reveal_type(i8 > u4) # E: numpy.bool_ +reveal_type(i4 > u4) # E: numpy.bool_ +reveal_type(u8 > u4) # E: numpy.bool_ +reveal_type(u4 > u4) # E: numpy.bool_ +reveal_type(b_ > u4) # E: numpy.bool_ +reveal_type(b > u4) # E: numpy.bool_ +reveal_type(i > u4) # E: numpy.bool_ +reveal_type(AR > u4) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(SEQ > u4) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] diff --git a/numpy/typing/tests/data/reveal/datasource.py b/numpy/typing/tests/data/reveal/datasource.py new file mode 100644 index 000000000000..245ac7649e96 --- /dev/null +++ b/numpy/typing/tests/data/reveal/datasource.py @@ -0,0 +1,21 @@ +from pathlib import Path +import numpy as np + +path1: Path +path2: str + +d1 = np.DataSource(path1) +d2 = np.DataSource(path2) +d3 = np.DataSource(None) + +reveal_type(d1.abspath("...")) # E: str +reveal_type(d2.abspath("...")) # E: str +reveal_type(d3.abspath("...")) # E: str + +reveal_type(d1.exists("...")) # E: bool +reveal_type(d2.exists("...")) # E: bool +reveal_type(d3.exists("...")) # E: bool + +reveal_type(d1.open("...", "r")) # E: IO[Any] +reveal_type(d2.open("...", encoding="utf8")) # E: IO[Any] +reveal_type(d3.open("...", newline="/n")) # E: IO[Any] diff --git a/numpy/typing/tests/data/reveal/dtype.py b/numpy/typing/tests/data/reveal/dtype.py index d414f2c4934f..215d89ead66c 100644 --- a/numpy/typing/tests/data/reveal/dtype.py +++ b/numpy/typing/tests/data/reveal/dtype.py @@ -1,33 +1,57 @@ import numpy as np -reveal_type(np.dtype(np.float64)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] -reveal_type(np.dtype(np.int64)) # E: numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]] +dtype_obj: np.dtype[np.str_] +void_dtype_obj: np.dtype[np.void] + +reveal_type(np.dtype(np.float64)) # E: numpy.dtype[{float64}] +reveal_type(np.dtype(np.int64)) # E: numpy.dtype[{int64}] # String aliases -reveal_type(np.dtype("float64")) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] -reveal_type(np.dtype("float32")) # E: numpy.dtype[numpy.floating[numpy.typing._32Bit]] -reveal_type(np.dtype("int64")) # E: numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(np.dtype("int32")) # E: numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]] +reveal_type(np.dtype("float64")) # E: numpy.dtype[{float64}] +reveal_type(np.dtype("float32")) # E: numpy.dtype[{float32}] +reveal_type(np.dtype("int64")) # E: numpy.dtype[{int64}] +reveal_type(np.dtype("int32")) # E: numpy.dtype[{int32}] reveal_type(np.dtype("bool")) # E: numpy.dtype[numpy.bool_] reveal_type(np.dtype("bytes")) # E: numpy.dtype[numpy.bytes_] reveal_type(np.dtype("str")) # E: numpy.dtype[numpy.str_] # Python types -reveal_type(np.dtype(complex)) # E: numpy.dtype[numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit]] -reveal_type(np.dtype(float)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] -reveal_type(np.dtype(int)) # E: numpy.dtype +reveal_type(np.dtype(complex)) # E: numpy.dtype[{cdouble}] +reveal_type(np.dtype(float)) # E: numpy.dtype[{double}] +reveal_type(np.dtype(int)) # E: numpy.dtype[{int_}] reveal_type(np.dtype(bool)) # E: numpy.dtype[numpy.bool_] reveal_type(np.dtype(str)) # E: numpy.dtype[numpy.str_] reveal_type(np.dtype(bytes)) # E: numpy.dtype[numpy.bytes_] # Special case for None -reveal_type(np.dtype(None)) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype(None)) # E: numpy.dtype[{double}] # Dtypes of dtypes -reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(np.dtype(np.dtype(np.float64))) # E: numpy.dtype[{float64}] # Parameterized dtypes reveal_type(np.dtype("S8")) # E: numpy.dtype # Void reveal_type(np.dtype(("U", 10))) # E: numpy.dtype[numpy.void] + +# Methods and attributes +reveal_type(dtype_obj.base) # E: numpy.dtype[numpy.str_] +reveal_type(dtype_obj.subdtype) # E: Union[Tuple[numpy.dtype[numpy.str_], builtins.tuple[builtins.int]], None] +reveal_type(dtype_obj.newbyteorder()) # E: numpy.dtype[numpy.str_] +reveal_type(dtype_obj.type) # E: Type[numpy.str_] +reveal_type(dtype_obj.name) # E: str +reveal_type(dtype_obj.names) # E: Union[builtins.tuple[builtins.str], None] + +reveal_type(dtype_obj * 0) # E: None +reveal_type(dtype_obj * 1) # E: numpy.dtype[numpy.str_] +reveal_type(dtype_obj * 2) # E: numpy.dtype[numpy.void] + +reveal_type(0 * dtype_obj) # E: Any +reveal_type(1 * dtype_obj) # E: Any +reveal_type(2 * dtype_obj) # E: Any + +reveal_type(void_dtype_obj["f0"]) # E: numpy.dtype[Any] +reveal_type(void_dtype_obj[0]) # E: numpy.dtype[Any] +reveal_type(void_dtype_obj[["f0", "f1"]]) # E: numpy.dtype[numpy.void] +reveal_type(void_dtype_obj[["f0"]]) # E: numpy.dtype[numpy.void] diff --git a/numpy/typing/tests/data/reveal/einsumfunc.py b/numpy/typing/tests/data/reveal/einsumfunc.py new file mode 100644 index 000000000000..f1a90428de34 --- /dev/null +++ b/numpy/typing/tests/data/reveal/einsumfunc.py @@ -0,0 +1,32 @@ +from typing import List, Any +import numpy as np + +AR_LIKE_b: List[bool] +AR_LIKE_u: List[np.uint32] +AR_LIKE_i: List[int] +AR_LIKE_f: List[float] +AR_LIKE_c: List[complex] +AR_LIKE_U: List[str] + +OUT_f: np.ndarray[Any, np.dtype[np.float64]] + +reveal_type(np.einsum("i,i->i", AR_LIKE_b, AR_LIKE_b)) # E: Any +reveal_type(np.einsum("i,i->i", AR_LIKE_u, AR_LIKE_u)) # E: Any +reveal_type(np.einsum("i,i->i", AR_LIKE_i, AR_LIKE_i)) # E: Any +reveal_type(np.einsum("i,i->i", AR_LIKE_f, AR_LIKE_f)) # E: Any +reveal_type(np.einsum("i,i->i", AR_LIKE_c, AR_LIKE_c)) # E: Any +reveal_type(np.einsum("i,i->i", AR_LIKE_b, AR_LIKE_i)) # E: Any +reveal_type(np.einsum("i,i,i,i->i", AR_LIKE_b, AR_LIKE_u, AR_LIKE_i, AR_LIKE_c)) # E: Any + +reveal_type(np.einsum("i,i->i", AR_LIKE_c, AR_LIKE_c, out=OUT_f)) # E: numpy.ndarray[Any, numpy.dtype[{float64}] +reveal_type(np.einsum("i,i->i", AR_LIKE_U, AR_LIKE_U, dtype=bool, casting="unsafe", out=OUT_f)) # E: numpy.ndarray[Any, numpy.dtype[{float64}] +reveal_type(np.einsum("i,i->i", AR_LIKE_f, AR_LIKE_f, dtype="c16")) # E: Any +reveal_type(np.einsum("i,i->i", AR_LIKE_U, AR_LIKE_U, dtype=bool, casting="unsafe")) # E: Any + +reveal_type(np.einsum_path("i,i->i", AR_LIKE_b, AR_LIKE_b)) # E: Tuple[builtins.list[Any], builtins.str] +reveal_type(np.einsum_path("i,i->i", AR_LIKE_u, AR_LIKE_u)) # E: Tuple[builtins.list[Any], builtins.str] +reveal_type(np.einsum_path("i,i->i", AR_LIKE_i, AR_LIKE_i)) # E: Tuple[builtins.list[Any], builtins.str] +reveal_type(np.einsum_path("i,i->i", AR_LIKE_f, AR_LIKE_f)) # E: Tuple[builtins.list[Any], builtins.str] +reveal_type(np.einsum_path("i,i->i", AR_LIKE_c, AR_LIKE_c)) # E: Tuple[builtins.list[Any], builtins.str] +reveal_type(np.einsum_path("i,i->i", AR_LIKE_b, AR_LIKE_i)) # E: Tuple[builtins.list[Any], builtins.str] +reveal_type(np.einsum_path("i,i,i,i->i", AR_LIKE_b, AR_LIKE_u, AR_LIKE_i, AR_LIKE_c)) # E: Tuple[builtins.list[Any], builtins.str] diff --git a/numpy/typing/tests/data/reveal/flatiter.py b/numpy/typing/tests/data/reveal/flatiter.py index 56cdc7a0ea74..97776dd9f4f5 100644 --- a/numpy/typing/tests/data/reveal/flatiter.py +++ b/numpy/typing/tests/data/reveal/flatiter.py @@ -1,14 +1,17 @@ +from typing import Any import numpy as np -a: "np.flatiter[np.ndarray]" +a: np.flatiter[np.ndarray[Any, np.dtype[np.str_]]] -reveal_type(a.base) # E: numpy.ndarray* -reveal_type(a.copy()) # E: numpy.ndarray* +reveal_type(a.base) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(a.copy()) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] reveal_type(a.coords) # E: tuple[builtins.int] reveal_type(a.index) # E: int -reveal_type(iter(a)) # E: Iterator[numpy.generic*] -reveal_type(next(a)) # E: numpy.generic -reveal_type(a[0]) # E: numpy.generic -reveal_type(a[[0, 1, 2]]) # E: numpy.ndarray* -reveal_type(a[...]) # E: numpy.ndarray* -reveal_type(a[:]) # E: numpy.ndarray* +reveal_type(iter(a)) # E: Iterator[numpy.str_] +reveal_type(next(a)) # E: numpy.str_ +reveal_type(a[0]) # E: numpy.str_ +reveal_type(a[[0, 1, 2]]) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(a[...]) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(a[:]) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(a.__array__()) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(a.__array__(np.dtype(np.float64))) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] diff --git a/numpy/typing/tests/data/reveal/fromnumeric.py b/numpy/typing/tests/data/reveal/fromnumeric.py index 75865c285f36..bbcfbb85ad3d 100644 --- a/numpy/typing/tests/data/reveal/fromnumeric.py +++ b/numpy/typing/tests/data/reveal/fromnumeric.py @@ -12,116 +12,102 @@ c = 1.0 d = np.array(1.0, dtype=np.float32) # writeable -reveal_type(np.take(a, 0)) # E: numpy.bool_ -reveal_type(np.take(b, 0)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type( - np.take(c, 0) # E: Union[numpy.generic, datetime.datetime, datetime.timedelta] -) -reveal_type( - np.take(A, 0) # E: Union[numpy.generic, datetime.datetime, datetime.timedelta] -) -reveal_type( - np.take(B, 0) # E: Union[numpy.generic, datetime.datetime, datetime.timedelta] -) -reveal_type( - np.take( # E: Union[Union[numpy.generic, datetime.datetime, datetime.timedelta], numpy.ndarray] - A, [0] - ) -) -reveal_type( - np.take( # E: Union[Union[numpy.generic, datetime.datetime, datetime.timedelta], numpy.ndarray] - B, [0] - ) -) - -reveal_type(np.reshape(a, 1)) # E: numpy.ndarray -reveal_type(np.reshape(b, 1)) # E: numpy.ndarray -reveal_type(np.reshape(c, 1)) # E: numpy.ndarray -reveal_type(np.reshape(A, 1)) # E: numpy.ndarray -reveal_type(np.reshape(B, 1)) # E: numpy.ndarray - -reveal_type(np.choose(a, [True, True])) # E: numpy.bool_ -reveal_type(np.choose(A, [True, True])) # E: numpy.ndarray - -reveal_type(np.repeat(a, 1)) # E: numpy.ndarray -reveal_type(np.repeat(b, 1)) # E: numpy.ndarray -reveal_type(np.repeat(c, 1)) # E: numpy.ndarray -reveal_type(np.repeat(A, 1)) # E: numpy.ndarray -reveal_type(np.repeat(B, 1)) # E: numpy.ndarray +reveal_type(np.take(a, 0)) # E: Any +reveal_type(np.take(b, 0)) # E: Any +reveal_type(np.take(c, 0)) # E: Any +reveal_type(np.take(A, 0)) # E: Any +reveal_type(np.take(B, 0)) # E: Any +reveal_type(np.take(A, [0])) # E: Any +reveal_type(np.take(B, [0])) # E: Any + +reveal_type(np.reshape(a, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.reshape(b, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.reshape(c, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.reshape(A, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.reshape(B, 1)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.choose(a, [True, True])) # E: Any +reveal_type(np.choose(A, [True, True])) # E: Any + +reveal_type(np.repeat(a, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.repeat(b, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.repeat(c, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.repeat(A, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.repeat(B, 1)) # E: numpy.ndarray[Any, Any] # TODO: Add tests for np.put() -reveal_type(np.swapaxes(A, 0, 0)) # E: numpy.ndarray -reveal_type(np.swapaxes(B, 0, 0)) # E: numpy.ndarray - -reveal_type(np.transpose(a)) # E: numpy.ndarray -reveal_type(np.transpose(b)) # E: numpy.ndarray -reveal_type(np.transpose(c)) # E: numpy.ndarray -reveal_type(np.transpose(A)) # E: numpy.ndarray -reveal_type(np.transpose(B)) # E: numpy.ndarray - -reveal_type(np.partition(a, 0, axis=None)) # E: numpy.ndarray -reveal_type(np.partition(b, 0, axis=None)) # E: numpy.ndarray -reveal_type(np.partition(c, 0, axis=None)) # E: numpy.ndarray -reveal_type(np.partition(A, 0)) # E: numpy.ndarray -reveal_type(np.partition(B, 0)) # E: numpy.ndarray - -reveal_type(np.argpartition(a, 0)) # E: numpy.integer[Any] -reveal_type(np.argpartition(b, 0)) # E: numpy.integer[Any] -reveal_type(np.argpartition(c, 0)) # E: numpy.ndarray -reveal_type(np.argpartition(A, 0)) # E: numpy.ndarray -reveal_type(np.argpartition(B, 0)) # E: numpy.ndarray - -reveal_type(np.sort(A, 0)) # E: numpy.ndarray -reveal_type(np.sort(B, 0)) # E: numpy.ndarray - -reveal_type(np.argsort(A, 0)) # E: numpy.ndarray -reveal_type(np.argsort(B, 0)) # E: numpy.ndarray - -reveal_type(np.argmax(A)) # E: numpy.integer[Any] -reveal_type(np.argmax(B)) # E: numpy.integer[Any] -reveal_type(np.argmax(A, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] -reveal_type(np.argmax(B, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] - -reveal_type(np.argmin(A)) # E: numpy.integer[Any] -reveal_type(np.argmin(B)) # E: numpy.integer[Any] -reveal_type(np.argmin(A, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] -reveal_type(np.argmin(B, axis=0)) # E: Union[numpy.integer[Any], numpy.ndarray] - -reveal_type(np.searchsorted(A[0], 0)) # E: numpy.integer[Any] -reveal_type(np.searchsorted(B[0], 0)) # E: numpy.integer[Any] -reveal_type(np.searchsorted(A[0], [0])) # E: numpy.ndarray -reveal_type(np.searchsorted(B[0], [0])) # E: numpy.ndarray - -reveal_type(np.resize(a, (5, 5))) # E: numpy.ndarray -reveal_type(np.resize(b, (5, 5))) # E: numpy.ndarray -reveal_type(np.resize(c, (5, 5))) # E: numpy.ndarray -reveal_type(np.resize(A, (5, 5))) # E: numpy.ndarray -reveal_type(np.resize(B, (5, 5))) # E: numpy.ndarray +reveal_type(np.swapaxes(A, 0, 0)) # E: numpy.ndarray[Any, Any] +reveal_type(np.swapaxes(B, 0, 0)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.transpose(a)) # E: numpy.ndarray[Any, Any] +reveal_type(np.transpose(b)) # E: numpy.ndarray[Any, Any] +reveal_type(np.transpose(c)) # E: numpy.ndarray[Any, Any] +reveal_type(np.transpose(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.transpose(B)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.partition(a, 0, axis=None)) # E: numpy.ndarray[Any, Any] +reveal_type(np.partition(b, 0, axis=None)) # E: numpy.ndarray[Any, Any] +reveal_type(np.partition(c, 0, axis=None)) # E: numpy.ndarray[Any, Any] +reveal_type(np.partition(A, 0)) # E: numpy.ndarray[Any, Any] +reveal_type(np.partition(B, 0)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.argpartition(a, 0)) # E: Any +reveal_type(np.argpartition(b, 0)) # E: Any +reveal_type(np.argpartition(c, 0)) # E: Any +reveal_type(np.argpartition(A, 0)) # E: Any +reveal_type(np.argpartition(B, 0)) # E: Any + +reveal_type(np.sort(A, 0)) # E: numpy.ndarray[Any, Any] +reveal_type(np.sort(B, 0)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.argsort(A, 0)) # E: numpy.ndarray[Any, Any] +reveal_type(np.argsort(B, 0)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.argmax(A)) # E: {intp} +reveal_type(np.argmax(B)) # E: {intp} +reveal_type(np.argmax(A, axis=0)) # E: Any +reveal_type(np.argmax(B, axis=0)) # E: Any + +reveal_type(np.argmin(A)) # E: {intp} +reveal_type(np.argmin(B)) # E: {intp} +reveal_type(np.argmin(A, axis=0)) # E: Any +reveal_type(np.argmin(B, axis=0)) # E: Any + +reveal_type(np.searchsorted(A[0], 0)) # E: {intp} +reveal_type(np.searchsorted(B[0], 0)) # E: {intp} +reveal_type(np.searchsorted(A[0], [0])) # E: numpy.ndarray[Any, Any] +reveal_type(np.searchsorted(B[0], [0])) # E: numpy.ndarray[Any, Any] + +reveal_type(np.resize(a, (5, 5))) # E: numpy.ndarray[Any, Any] +reveal_type(np.resize(b, (5, 5))) # E: numpy.ndarray[Any, Any] +reveal_type(np.resize(c, (5, 5))) # E: numpy.ndarray[Any, Any] +reveal_type(np.resize(A, (5, 5))) # E: numpy.ndarray[Any, Any] +reveal_type(np.resize(B, (5, 5))) # E: numpy.ndarray[Any, Any] reveal_type(np.squeeze(a)) # E: numpy.bool_ -reveal_type(np.squeeze(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.squeeze(c)) # E: numpy.ndarray -reveal_type(np.squeeze(A)) # E: numpy.ndarray -reveal_type(np.squeeze(B)) # E: numpy.ndarray +reveal_type(np.squeeze(b)) # E: {float32} +reveal_type(np.squeeze(c)) # E: numpy.ndarray[Any, Any] +reveal_type(np.squeeze(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.squeeze(B)) # E: numpy.ndarray[Any, Any] -reveal_type(np.diagonal(A)) # E: numpy.ndarray -reveal_type(np.diagonal(B)) # E: numpy.ndarray +reveal_type(np.diagonal(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.diagonal(B)) # E: numpy.ndarray[Any, Any] -reveal_type(np.trace(A)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.trace(B)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.trace(A)) # E: Any +reveal_type(np.trace(B)) # E: Any -reveal_type(np.ravel(a)) # E: numpy.ndarray -reveal_type(np.ravel(b)) # E: numpy.ndarray -reveal_type(np.ravel(c)) # E: numpy.ndarray -reveal_type(np.ravel(A)) # E: numpy.ndarray -reveal_type(np.ravel(B)) # E: numpy.ndarray +reveal_type(np.ravel(a)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ravel(b)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ravel(c)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ravel(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.ravel(B)) # E: numpy.ndarray[Any, Any] -reveal_type(np.nonzero(a)) # E: tuple[numpy.ndarray] -reveal_type(np.nonzero(b)) # E: tuple[numpy.ndarray] -reveal_type(np.nonzero(c)) # E: tuple[numpy.ndarray] -reveal_type(np.nonzero(A)) # E: tuple[numpy.ndarray] -reveal_type(np.nonzero(B)) # E: tuple[numpy.ndarray] +reveal_type(np.nonzero(a)) # E: tuple[numpy.ndarray[Any, Any]] +reveal_type(np.nonzero(b)) # E: tuple[numpy.ndarray[Any, Any]] +reveal_type(np.nonzero(c)) # E: tuple[numpy.ndarray[Any, Any]] +reveal_type(np.nonzero(A)) # E: tuple[numpy.ndarray[Any, Any]] +reveal_type(np.nonzero(B)) # E: tuple[numpy.ndarray[Any, Any]] reveal_type(np.shape(a)) # E: tuple[builtins.int] reveal_type(np.shape(b)) # E: tuple[builtins.int] @@ -129,99 +115,99 @@ reveal_type(np.shape(A)) # E: tuple[builtins.int] reveal_type(np.shape(B)) # E: tuple[builtins.int] -reveal_type(np.compress([True], a)) # E: numpy.ndarray -reveal_type(np.compress([True], b)) # E: numpy.ndarray -reveal_type(np.compress([True], c)) # E: numpy.ndarray -reveal_type(np.compress([True], A)) # E: numpy.ndarray -reveal_type(np.compress([True], B)) # E: numpy.ndarray - -reveal_type(np.clip(a, 0, 1.0)) # E: numpy.number[Any] -reveal_type(np.clip(b, -1, 1)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.clip(c, 0, 1)) # E: numpy.number[Any] -reveal_type(np.clip(A, 0, 1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.clip(B, 0, 1)) # E: Union[numpy.number[Any], numpy.ndarray] - -reveal_type(np.sum(a)) # E: numpy.number[Any] -reveal_type(np.sum(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.sum(c)) # E: numpy.number[Any] -reveal_type(np.sum(A)) # E: numpy.number[Any] -reveal_type(np.sum(B)) # E: numpy.number[Any] -reveal_type(np.sum(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.sum(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] +reveal_type(np.compress([True], a)) # E: numpy.ndarray[Any, Any] +reveal_type(np.compress([True], b)) # E: numpy.ndarray[Any, Any] +reveal_type(np.compress([True], c)) # E: numpy.ndarray[Any, Any] +reveal_type(np.compress([True], A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.compress([True], B)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.clip(a, 0, 1.0)) # E: Any +reveal_type(np.clip(b, -1, 1)) # E: Any +reveal_type(np.clip(c, 0, 1)) # E: Any +reveal_type(np.clip(A, 0, 1)) # E: Any +reveal_type(np.clip(B, 0, 1)) # E: Any + +reveal_type(np.sum(a)) # E: Any +reveal_type(np.sum(b)) # E: Any +reveal_type(np.sum(c)) # E: Any +reveal_type(np.sum(A)) # E: Any +reveal_type(np.sum(B)) # E: Any +reveal_type(np.sum(A, axis=0)) # E: Any +reveal_type(np.sum(B, axis=0)) # E: Any reveal_type(np.all(a)) # E: numpy.bool_ reveal_type(np.all(b)) # E: numpy.bool_ reveal_type(np.all(c)) # E: numpy.bool_ reveal_type(np.all(A)) # E: numpy.bool_ reveal_type(np.all(B)) # E: numpy.bool_ -reveal_type(np.all(A, axis=0)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.all(B, axis=0)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.all(A, keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.all(B, keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(np.all(A, axis=0)) # E: Any +reveal_type(np.all(B, axis=0)) # E: Any +reveal_type(np.all(A, keepdims=True)) # E: Any +reveal_type(np.all(B, keepdims=True)) # E: Any reveal_type(np.any(a)) # E: numpy.bool_ reveal_type(np.any(b)) # E: numpy.bool_ reveal_type(np.any(c)) # E: numpy.bool_ reveal_type(np.any(A)) # E: numpy.bool_ reveal_type(np.any(B)) # E: numpy.bool_ -reveal_type(np.any(A, axis=0)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.any(B, axis=0)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.any(A, keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.any(B, keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] - -reveal_type(np.cumsum(a)) # E: numpy.ndarray -reveal_type(np.cumsum(b)) # E: numpy.ndarray -reveal_type(np.cumsum(c)) # E: numpy.ndarray -reveal_type(np.cumsum(A)) # E: numpy.ndarray -reveal_type(np.cumsum(B)) # E: numpy.ndarray - -reveal_type(np.ptp(a)) # E: numpy.number[Any] -reveal_type(np.ptp(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.ptp(c)) # E: numpy.number[Any] -reveal_type(np.ptp(A)) # E: numpy.number[Any] -reveal_type(np.ptp(B)) # E: numpy.number[Any] -reveal_type(np.ptp(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.ptp(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.ptp(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.ptp(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] - -reveal_type(np.amax(a)) # E: numpy.number[Any] -reveal_type(np.amax(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.amax(c)) # E: numpy.number[Any] -reveal_type(np.amax(A)) # E: numpy.number[Any] -reveal_type(np.amax(B)) # E: numpy.number[Any] -reveal_type(np.amax(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.amax(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.amax(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.amax(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] - -reveal_type(np.amin(a)) # E: numpy.number[Any] -reveal_type(np.amin(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.amin(c)) # E: numpy.number[Any] -reveal_type(np.amin(A)) # E: numpy.number[Any] -reveal_type(np.amin(B)) # E: numpy.number[Any] -reveal_type(np.amin(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.amin(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.amin(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.amin(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] - -reveal_type(np.prod(a)) # E: numpy.number[Any] -reveal_type(np.prod(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.prod(c)) # E: numpy.number[Any] -reveal_type(np.prod(A)) # E: numpy.number[Any] -reveal_type(np.prod(B)) # E: numpy.number[Any] -reveal_type(np.prod(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.prod(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.prod(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.prod(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.prod(b, out=d)) # E: numpy.ndarray -reveal_type(np.prod(B, out=d)) # E: numpy.ndarray - -reveal_type(np.cumprod(a)) # E: numpy.ndarray -reveal_type(np.cumprod(b)) # E: numpy.ndarray -reveal_type(np.cumprod(c)) # E: numpy.ndarray -reveal_type(np.cumprod(A)) # E: numpy.ndarray -reveal_type(np.cumprod(B)) # E: numpy.ndarray +reveal_type(np.any(A, axis=0)) # E: Any +reveal_type(np.any(B, axis=0)) # E: Any +reveal_type(np.any(A, keepdims=True)) # E: Any +reveal_type(np.any(B, keepdims=True)) # E: Any + +reveal_type(np.cumsum(a)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumsum(b)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumsum(c)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumsum(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumsum(B)) # E: numpy.ndarray[Any, Any] + +reveal_type(np.ptp(a)) # E: Any +reveal_type(np.ptp(b)) # E: Any +reveal_type(np.ptp(c)) # E: Any +reveal_type(np.ptp(A)) # E: Any +reveal_type(np.ptp(B)) # E: Any +reveal_type(np.ptp(A, axis=0)) # E: Any +reveal_type(np.ptp(B, axis=0)) # E: Any +reveal_type(np.ptp(A, keepdims=True)) # E: Any +reveal_type(np.ptp(B, keepdims=True)) # E: Any + +reveal_type(np.amax(a)) # E: Any +reveal_type(np.amax(b)) # E: Any +reveal_type(np.amax(c)) # E: Any +reveal_type(np.amax(A)) # E: Any +reveal_type(np.amax(B)) # E: Any +reveal_type(np.amax(A, axis=0)) # E: Any +reveal_type(np.amax(B, axis=0)) # E: Any +reveal_type(np.amax(A, keepdims=True)) # E: Any +reveal_type(np.amax(B, keepdims=True)) # E: Any + +reveal_type(np.amin(a)) # E: Any +reveal_type(np.amin(b)) # E: Any +reveal_type(np.amin(c)) # E: Any +reveal_type(np.amin(A)) # E: Any +reveal_type(np.amin(B)) # E: Any +reveal_type(np.amin(A, axis=0)) # E: Any +reveal_type(np.amin(B, axis=0)) # E: Any +reveal_type(np.amin(A, keepdims=True)) # E: Any +reveal_type(np.amin(B, keepdims=True)) # E: Any + +reveal_type(np.prod(a)) # E: Any +reveal_type(np.prod(b)) # E: Any +reveal_type(np.prod(c)) # E: Any +reveal_type(np.prod(A)) # E: Any +reveal_type(np.prod(B)) # E: Any +reveal_type(np.prod(A, axis=0)) # E: Any +reveal_type(np.prod(B, axis=0)) # E: Any +reveal_type(np.prod(A, keepdims=True)) # E: Any +reveal_type(np.prod(B, keepdims=True)) # E: Any +reveal_type(np.prod(b, out=d)) # E: Any +reveal_type(np.prod(B, out=d)) # E: Any + +reveal_type(np.cumprod(a)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumprod(b)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumprod(c)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumprod(A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cumprod(B)) # E: numpy.ndarray[Any, Any] reveal_type(np.ndim(a)) # E: int reveal_type(np.ndim(b)) # E: int @@ -235,44 +221,44 @@ reveal_type(np.size(A)) # E: int reveal_type(np.size(B)) # E: int -reveal_type(np.around(a)) # E: numpy.number[Any] -reveal_type(np.around(b)) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.around(c)) # E: numpy.number[Any] -reveal_type(np.around(A)) # E: numpy.ndarray -reveal_type(np.around(B)) # E: numpy.ndarray - -reveal_type(np.mean(a)) # E: numpy.number[Any] -reveal_type(np.mean(b)) # E: numpy.number[Any] -reveal_type(np.mean(c)) # E: numpy.number[Any] -reveal_type(np.mean(A)) # E: numpy.number[Any] -reveal_type(np.mean(B)) # E: numpy.number[Any] -reveal_type(np.mean(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.mean(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.mean(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.mean(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.mean(b, out=d)) # E: numpy.ndarray -reveal_type(np.mean(B, out=d)) # E: numpy.ndarray - -reveal_type(np.std(a)) # E: numpy.number[Any] -reveal_type(np.std(b)) # E: numpy.number[Any] -reveal_type(np.std(c)) # E: numpy.number[Any] -reveal_type(np.std(A)) # E: numpy.number[Any] -reveal_type(np.std(B)) # E: numpy.number[Any] -reveal_type(np.std(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.std(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.std(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.std(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.std(b, out=d)) # E: numpy.ndarray -reveal_type(np.std(B, out=d)) # E: numpy.ndarray - -reveal_type(np.var(a)) # E: numpy.number[Any] -reveal_type(np.var(b)) # E: numpy.number[Any] -reveal_type(np.var(c)) # E: numpy.number[Any] -reveal_type(np.var(A)) # E: numpy.number[Any] -reveal_type(np.var(B)) # E: numpy.number[Any] -reveal_type(np.var(A, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.var(B, axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.var(A, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.var(B, keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(np.var(b, out=d)) # E: numpy.ndarray -reveal_type(np.var(B, out=d)) # E: numpy.ndarray +reveal_type(np.around(a)) # E: Any +reveal_type(np.around(b)) # E: Any +reveal_type(np.around(c)) # E: Any +reveal_type(np.around(A)) # E: Any +reveal_type(np.around(B)) # E: Any + +reveal_type(np.mean(a)) # E: Any +reveal_type(np.mean(b)) # E: Any +reveal_type(np.mean(c)) # E: Any +reveal_type(np.mean(A)) # E: Any +reveal_type(np.mean(B)) # E: Any +reveal_type(np.mean(A, axis=0)) # E: Any +reveal_type(np.mean(B, axis=0)) # E: Any +reveal_type(np.mean(A, keepdims=True)) # E: Any +reveal_type(np.mean(B, keepdims=True)) # E: Any +reveal_type(np.mean(b, out=d)) # E: Any +reveal_type(np.mean(B, out=d)) # E: Any + +reveal_type(np.std(a)) # E: Any +reveal_type(np.std(b)) # E: Any +reveal_type(np.std(c)) # E: Any +reveal_type(np.std(A)) # E: Any +reveal_type(np.std(B)) # E: Any +reveal_type(np.std(A, axis=0)) # E: Any +reveal_type(np.std(B, axis=0)) # E: Any +reveal_type(np.std(A, keepdims=True)) # E: Any +reveal_type(np.std(B, keepdims=True)) # E: Any +reveal_type(np.std(b, out=d)) # E: Any +reveal_type(np.std(B, out=d)) # E: Any + +reveal_type(np.var(a)) # E: Any +reveal_type(np.var(b)) # E: Any +reveal_type(np.var(c)) # E: Any +reveal_type(np.var(A)) # E: Any +reveal_type(np.var(B)) # E: Any +reveal_type(np.var(A, axis=0)) # E: Any +reveal_type(np.var(B, axis=0)) # E: Any +reveal_type(np.var(A, keepdims=True)) # E: Any +reveal_type(np.var(B, keepdims=True)) # E: Any +reveal_type(np.var(b, out=d)) # E: Any +reveal_type(np.var(B, out=d)) # E: Any diff --git a/numpy/typing/tests/data/reveal/index_tricks.py b/numpy/typing/tests/data/reveal/index_tricks.py new file mode 100644 index 000000000000..863d60220937 --- /dev/null +++ b/numpy/typing/tests/data/reveal/index_tricks.py @@ -0,0 +1,64 @@ +from typing import Any, List +import numpy as np + +AR_LIKE_b: List[bool] +AR_LIKE_i: List[int] +AR_LIKE_f: List[float] +AR_LIKE_U: List[str] + +AR_i8: np.ndarray[Any, np.dtype[np.int64]] + +reveal_type(np.ndenumerate(AR_i8)) # E: numpy.ndenumerate[{int64}] +reveal_type(np.ndenumerate(AR_LIKE_f)) # E: numpy.ndenumerate[{double}] +reveal_type(np.ndenumerate(AR_LIKE_U)) # E: numpy.ndenumerate[numpy.str_] + +reveal_type(np.ndenumerate(AR_i8).iter) # E: numpy.flatiter[numpy.ndarray[Any, numpy.dtype[{int64}]]] +reveal_type(np.ndenumerate(AR_LIKE_f).iter) # E: numpy.flatiter[numpy.ndarray[Any, numpy.dtype[{double}]]] +reveal_type(np.ndenumerate(AR_LIKE_U).iter) # E: numpy.flatiter[numpy.ndarray[Any, numpy.dtype[numpy.str_]]] + +reveal_type(next(np.ndenumerate(AR_i8))) # E: Tuple[builtins.tuple[builtins.int], {int64}] +reveal_type(next(np.ndenumerate(AR_LIKE_f))) # E: Tuple[builtins.tuple[builtins.int], {double}] +reveal_type(next(np.ndenumerate(AR_LIKE_U))) # E: Tuple[builtins.tuple[builtins.int], numpy.str_] + +reveal_type(iter(np.ndenumerate(AR_i8))) # E: Iterator[Tuple[builtins.tuple[builtins.int], {int64}]] +reveal_type(iter(np.ndenumerate(AR_LIKE_f))) # E: Iterator[Tuple[builtins.tuple[builtins.int], {double}]] +reveal_type(iter(np.ndenumerate(AR_LIKE_U))) # E: Iterator[Tuple[builtins.tuple[builtins.int], numpy.str_]] + +reveal_type(iter(np.ndindex(1, 2, 3))) # E: Iterator[builtins.tuple[builtins.int]] +reveal_type(next(np.ndindex(1, 2, 3))) # E: builtins.tuple[builtins.int] + +reveal_type(np.unravel_index([22, 41, 37], (7, 6))) # E: tuple[numpy.ndarray[Any, numpy.dtype[{intp}]]] +reveal_type(np.unravel_index([31, 41, 13], (7, 6), order="F")) # E: tuple[numpy.ndarray[Any, numpy.dtype[{intp}]]] +reveal_type(np.unravel_index(1621, (6, 7, 8, 9))) # E: tuple[{intp}] + +reveal_type(np.ravel_multi_index([[1]], (7, 6))) # E: numpy.ndarray[Any, numpy.dtype[{intp}]] +reveal_type(np.ravel_multi_index(AR_LIKE_i, (7, 6))) # E: {intp} +reveal_type(np.ravel_multi_index(AR_LIKE_i, (7, 6), order="F")) # E: {intp} +reveal_type(np.ravel_multi_index(AR_LIKE_i, (4, 6), mode="clip")) # E: {intp} +reveal_type(np.ravel_multi_index(AR_LIKE_i, (4, 4), mode=("clip", "wrap"))) # E: {intp} +reveal_type(np.ravel_multi_index((3, 1, 4, 1), (6, 7, 8, 9))) # E: {intp} + +reveal_type(np.mgrid[1:1:2]) # E: numpy.ndarray[Any, numpy.dtype[Any]] +reveal_type(np.mgrid[1:1:2, None:10]) # E: numpy.ndarray[Any, numpy.dtype[Any]] + +reveal_type(np.ogrid[1:1:2]) # E: list[numpy.ndarray[Any, numpy.dtype[Any]]] +reveal_type(np.ogrid[1:1:2, None:10]) # E: list[numpy.ndarray[Any, numpy.dtype[Any]]] + +reveal_type(np.index_exp[0:1]) # E: Tuple[builtins.slice] +reveal_type(np.index_exp[0:1, None:3]) # E: Tuple[builtins.slice, builtins.slice] +reveal_type(np.index_exp[0, 0:1, ..., [0, 1, 3]]) # E: Tuple[Literal[0]?, builtins.slice, builtins.ellipsis, builtins.list[builtins.int]] + +reveal_type(np.s_[0:1]) # E: builtins.slice +reveal_type(np.s_[0:1, None:3]) # E: Tuple[builtins.slice, builtins.slice] +reveal_type(np.s_[0, 0:1, ..., [0, 1, 3]]) # E: Tuple[Literal[0]?, builtins.slice, builtins.ellipsis, builtins.list[builtins.int]] + +reveal_type(np.ix_(AR_LIKE_b)) # E: tuple[numpy.ndarray[Any, numpy.dtype[numpy.bool_]]] +reveal_type(np.ix_(AR_LIKE_i, AR_LIKE_f)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{double}]]] +reveal_type(np.ix_(AR_i8)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int64}]]] + +reveal_type(np.fill_diagonal(AR_i8, 5)) # E: None + +reveal_type(np.diag_indices(4)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]] +reveal_type(np.diag_indices(2, 3)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]] + +reveal_type(np.diag_indices_from(AR_i8)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]] diff --git a/numpy/typing/tests/data/reveal/lib_utils.py b/numpy/typing/tests/data/reveal/lib_utils.py new file mode 100644 index 000000000000..d820127078a3 --- /dev/null +++ b/numpy/typing/tests/data/reveal/lib_utils.py @@ -0,0 +1,30 @@ +from io import StringIO +from typing import Any, Dict + +import numpy as np + +AR: np.ndarray[Any, np.dtype[np.float64]] +AR_DICT: Dict[str, np.ndarray[Any, np.dtype[np.float64]]] +FILE: StringIO + +def func(a: int) -> bool: ... + +reveal_type(np.deprecate(func)) # E: def (a: builtins.int) -> builtins.bool +reveal_type(np.deprecate()) # E: _Deprecate + +reveal_type(np.deprecate_with_doc("test")) # E: _Deprecate +reveal_type(np.deprecate_with_doc(None)) # E: _Deprecate + +reveal_type(np.byte_bounds(AR)) # E: Tuple[builtins.int, builtins.int] +reveal_type(np.byte_bounds(np.float64())) # E: Tuple[builtins.int, builtins.int] + +reveal_type(np.who(None)) # E: None +reveal_type(np.who(AR_DICT)) # E: None + +reveal_type(np.info(1, output=FILE)) # E: None + +reveal_type(np.source(np.interp, output=FILE)) # E: None + +reveal_type(np.lookfor("binary representation", output=FILE)) # E: None + +reveal_type(np.safe_eval("1 + 1")) # E: Any diff --git a/numpy/typing/tests/data/reveal/lib_version.py b/numpy/typing/tests/data/reveal/lib_version.py new file mode 100644 index 000000000000..e6f695558a40 --- /dev/null +++ b/numpy/typing/tests/data/reveal/lib_version.py @@ -0,0 +1,18 @@ +from numpy.lib import NumpyVersion + +version = NumpyVersion("1.8.0") + +reveal_type(version.vstring) # E: str +reveal_type(version.version) # E: str +reveal_type(version.major) # E: int +reveal_type(version.minor) # E: int +reveal_type(version.bugfix) # E: int +reveal_type(version.pre_release) # E: str +reveal_type(version.is_devversion) # E: bool + +reveal_type(version == version) # E: bool +reveal_type(version != version) # E: bool +reveal_type(version < "1.8.0") # E: bool +reveal_type(version <= version) # E: bool +reveal_type(version > version) # E: bool +reveal_type(version >= "1.8.0") # E: bool diff --git a/numpy/typing/tests/data/reveal/mod.py b/numpy/typing/tests/data/reveal/mod.py index 4292041f8302..bf45b8c589b5 100644 --- a/numpy/typing/tests/data/reveal/mod.py +++ b/numpy/typing/tests/data/reveal/mod.py @@ -1,3 +1,4 @@ +from typing import Any import numpy as np f8 = np.float64() @@ -15,135 +16,132 @@ f = float() i = int() -AR = np.array([1], dtype=np.bool_) -AR.setflags(write=False) - -AR2 = np.array([1], dtype=np.timedelta64) -AR2.setflags(write=False) +AR_b: np.ndarray[Any, np.dtype[np.bool_]] +AR_m: np.ndarray[Any, np.dtype[np.timedelta64]] # Time structures reveal_type(td % td) # E: numpy.timedelta64 -reveal_type(AR2 % td) # E: Any -reveal_type(td % AR2) # E: Any +reveal_type(AR_m % td) # E: Any +reveal_type(td % AR_m) # E: Any -reveal_type(divmod(td, td)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.timedelta64] -reveal_type(divmod(AR2, td)) # E: Tuple[Any, Any] -reveal_type(divmod(td, AR2)) # E: Tuple[Any, Any] +reveal_type(divmod(td, td)) # E: Tuple[{int64}, numpy.timedelta64] +reveal_type(divmod(AR_m, td)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]], numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]]] +reveal_type(divmod(td, AR_m)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]], numpy.ndarray[Any, numpy.dtype[numpy.timedelta64]]] # Bool -reveal_type(b_ % b) # E: numpy.signedinteger[numpy.typing._8Bit] -reveal_type(b_ % i) # E: numpy.signedinteger[Any] -reveal_type(b_ % f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ % b_) # E: numpy.signedinteger[numpy.typing._8Bit] -reveal_type(b_ % i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(b_ % u8) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(b_ % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ % AR) # E: Any - -reveal_type(divmod(b_, b)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] -reveal_type(divmod(b_, i)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] -reveal_type(divmod(b_, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(b_, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] -reveal_type(divmod(b_, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(b_, u8)) # E: Tuple[numpy.unsignedinteger[numpy.typing._64Bit], numpy.unsignedinteger[numpy.typing._64Bit]] -reveal_type(divmod(b_, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(b_, AR)) # E: Tuple[Any, Any] - -reveal_type(b % b_) # E: numpy.signedinteger[numpy.typing._8Bit] -reveal_type(i % b_) # E: numpy.signedinteger[Any] -reveal_type(f % b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(b_ % b_) # E: numpy.signedinteger[numpy.typing._8Bit] -reveal_type(i8 % b_) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(u8 % b_) # E: numpy.unsignedinteger[numpy.typing._64Bit] -reveal_type(f8 % b_) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(AR % b_) # E: Any - -reveal_type(divmod(b, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] -reveal_type(divmod(i, b_)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] -reveal_type(divmod(f, b_)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(b_, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._8Bit], numpy.signedinteger[numpy.typing._8Bit]] -reveal_type(divmod(i8, b_)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(u8, b_)) # E: Tuple[numpy.unsignedinteger[numpy.typing._64Bit], numpy.unsignedinteger[numpy.typing._64Bit]] -reveal_type(divmod(f8, b_)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(AR, b_)) # E: Tuple[Any, Any] +reveal_type(b_ % b) # E: {int8} +reveal_type(b_ % i) # E: {int_} +reveal_type(b_ % f) # E: {float64} +reveal_type(b_ % b_) # E: {int8} +reveal_type(b_ % i8) # E: {int64} +reveal_type(b_ % u8) # E: {uint64} +reveal_type(b_ % f8) # E: {float64} +reveal_type(b_ % AR_b) # E: numpy.ndarray[Any, numpy.dtype[{int8}]] + +reveal_type(divmod(b_, b)) # E: Tuple[{int8}, {int8}] +reveal_type(divmod(b_, i)) # E: Tuple[{int_}, {int_}] +reveal_type(divmod(b_, f)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(b_, b_)) # E: Tuple[{int8}, {int8}] +reveal_type(divmod(b_, i8)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(b_, u8)) # E: Tuple[{uint64}, {uint64}] +reveal_type(divmod(b_, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(b_, AR_b)) # E: numpy.ndarray[Any, numpy.dtype[{int8}]], numpy.ndarray[Any, numpy.dtype[{int8}]]] + +reveal_type(b % b_) # E: {int8} +reveal_type(i % b_) # E: {int_} +reveal_type(f % b_) # E: {float64} +reveal_type(b_ % b_) # E: {int8} +reveal_type(i8 % b_) # E: {int64} +reveal_type(u8 % b_) # E: {uint64} +reveal_type(f8 % b_) # E: {float64} +reveal_type(AR_b % b_) # E: numpy.ndarray[Any, numpy.dtype[{int8}]] + +reveal_type(divmod(b, b_)) # E: Tuple[{int8}, {int8}] +reveal_type(divmod(i, b_)) # E: Tuple[{int_}, {int_}] +reveal_type(divmod(f, b_)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(b_, b_)) # E: Tuple[{int8}, {int8}] +reveal_type(divmod(i8, b_)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(u8, b_)) # E: Tuple[{uint64}, {uint64}] +reveal_type(divmod(f8, b_)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(AR_b, b_)) # E: numpy.ndarray[Any, numpy.dtype[{int8}]], numpy.ndarray[Any, numpy.dtype[{int8}]]] # int -reveal_type(i8 % b) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 % i) # E: numpy.signedinteger[Any] -reveal_type(i8 % f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i8 % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i4 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i4 % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i4 % i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(i4 % f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(i8 % AR) # E: Any - -reveal_type(divmod(i8, b)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(i8, i)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] -reveal_type(divmod(i8, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i8, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(i8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i8, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(i8, f4)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i4, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._32Bit], numpy.signedinteger[numpy.typing._32Bit]] -reveal_type(divmod(i4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] -reveal_type(divmod(i8, AR)) # E: Tuple[Any, Any] - -reveal_type(b % i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(i % i8) # E: numpy.signedinteger[Any] -reveal_type(f % i8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 % i8) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(f8 % i8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 % i4) # E: numpy.signedinteger[numpy.typing._64Bit] -reveal_type(f8 % i4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i4 % i4) # E: numpy.signedinteger[numpy.typing._32Bit] -reveal_type(f4 % i4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(AR % i8) # E: Any - -reveal_type(divmod(b, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(i, i8)) # E: Tuple[numpy.signedinteger[Any], numpy.signedinteger[Any]] -reveal_type(divmod(f, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i8, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(f8, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i4, i8)) # E: Tuple[numpy.signedinteger[numpy.typing._64Bit], numpy.signedinteger[numpy.typing._64Bit]] -reveal_type(divmod(f4, i8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i4, i4)) # E: Tuple[numpy.signedinteger[numpy.typing._32Bit], numpy.signedinteger[numpy.typing._32Bit]] -reveal_type(divmod(f4, i4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] -reveal_type(divmod(AR, i8)) # E: Tuple[Any, Any] +reveal_type(i8 % b) # E: {int64} +reveal_type(i8 % i) # E: {int64} +reveal_type(i8 % f) # E: {float64} +reveal_type(i8 % i8) # E: {int64} +reveal_type(i8 % f8) # E: {float64} +reveal_type(i4 % i8) # E: {int64} +reveal_type(i4 % f8) # E: {float64} +reveal_type(i4 % i4) # E: {int32} +reveal_type(i4 % f4) # E: {float32} +reveal_type(i8 % AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] + +reveal_type(divmod(i8, b)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(i8, i)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(i8, f)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i8, i8)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(i8, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i8, i4)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(i8, f4)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i4, i4)) # E: Tuple[{int32}, {int32}] +reveal_type(divmod(i4, f4)) # E: Tuple[{float32}, {float32}] +reveal_type(divmod(i8, AR_b)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]], numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]]] + +reveal_type(b % i8) # E: {int64} +reveal_type(i % i8) # E: {int64} +reveal_type(f % i8) # E: {float64} +reveal_type(i8 % i8) # E: {int64} +reveal_type(f8 % i8) # E: {float64} +reveal_type(i8 % i4) # E: {int64} +reveal_type(f8 % i4) # E: {float64} +reveal_type(i4 % i4) # E: {int32} +reveal_type(f4 % i4) # E: {float32} +reveal_type(AR_b % i8) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]] + +reveal_type(divmod(b, i8)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(i, i8)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(f, i8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i8, i8)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(f8, i8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i4, i8)) # E: Tuple[{int64}, {int64}] +reveal_type(divmod(f4, i8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i4, i4)) # E: Tuple[{int32}, {int32}] +reveal_type(divmod(f4, i4)) # E: Tuple[{float32}, {float32}] +reveal_type(divmod(AR_b, i8)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]], numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[Any]]]] # float -reveal_type(f8 % b) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 % i) # E: numpy.floating[Any] -reveal_type(f8 % f) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i8 % f4) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 % f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(f8 % AR) # E: Any - -reveal_type(divmod(f8, b)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f8, i)) # E: Tuple[numpy.floating[Any], numpy.floating[Any]] -reveal_type(divmod(f8, f)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f8, f4)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] -reveal_type(divmod(f8, AR)) # E: Tuple[Any, Any] - -reveal_type(b % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(i % f8) # E: numpy.floating[Any] -reveal_type(f % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f8 % f8) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(f4 % f4) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(AR % f8) # E: Any - -reveal_type(divmod(b, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(i, f8)) # E: Tuple[numpy.floating[Any], numpy.floating[Any]] -reveal_type(divmod(f, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f8, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f4, f8)) # E: Tuple[numpy.floating[numpy.typing._64Bit], numpy.floating[numpy.typing._64Bit]] -reveal_type(divmod(f4, f4)) # E: Tuple[numpy.floating[numpy.typing._32Bit], numpy.floating[numpy.typing._32Bit]] -reveal_type(divmod(AR, f8)) # E: Tuple[Any, Any] +reveal_type(f8 % b) # E: {float64} +reveal_type(f8 % i) # E: {float64} +reveal_type(f8 % f) # E: {float64} +reveal_type(i8 % f4) # E: {float64} +reveal_type(f4 % f4) # E: {float32} +reveal_type(f8 % AR_b) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] + +reveal_type(divmod(f8, b)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f8, i)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f8, f)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f8, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f8, f4)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f4, f4)) # E: Tuple[{float32}, {float32}] +reveal_type(divmod(f8, AR_b)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]], numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]]] + +reveal_type(b % f8) # E: {float64} +reveal_type(i % f8) # E: {float64} +reveal_type(f % f8) # E: {float64} +reveal_type(f8 % f8) # E: {float64} +reveal_type(f8 % f8) # E: {float64} +reveal_type(f4 % f4) # E: {float32} +reveal_type(AR_b % f8) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] + +reveal_type(divmod(b, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(i, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f8, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f4, f8)) # E: Tuple[{float64}, {float64}] +reveal_type(divmod(f4, f4)) # E: Tuple[{float32}, {float32}] +reveal_type(divmod(AR_b, f8)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]], numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]]] diff --git a/numpy/typing/tests/data/reveal/modules.py b/numpy/typing/tests/data/reveal/modules.py index 40646315270d..fa356969aa92 100644 --- a/numpy/typing/tests/data/reveal/modules.py +++ b/numpy/typing/tests/data/reveal/modules.py @@ -1,4 +1,5 @@ import numpy as np +from numpy import f2py reveal_type(np) # E: ModuleType @@ -16,5 +17,32 @@ reveal_type(np.testing) # E: ModuleType reveal_type(np.version) # E: ModuleType +reveal_type(np.lib.format) # E: ModuleType +reveal_type(np.lib.mixins) # E: ModuleType +reveal_type(np.lib.scimath) # E: ModuleType +reveal_type(np.lib.stride_tricks) # E: ModuleType +reveal_type(np.ma.extras) # E: ModuleType +reveal_type(np.polynomial.chebyshev) # E: ModuleType +reveal_type(np.polynomial.hermite) # E: ModuleType +reveal_type(np.polynomial.hermite_e) # E: ModuleType +reveal_type(np.polynomial.laguerre) # E: ModuleType +reveal_type(np.polynomial.legendre) # E: ModuleType +reveal_type(np.polynomial.polynomial) # E: ModuleType + # TODO: Remove when annotations have been added to `np.testing.assert_equal` reveal_type(np.testing.assert_equal) # E: Any + +reveal_type(np.__path__) # E: list[builtins.str] +reveal_type(np.__version__) # E: str +reveal_type(np.__git_version__) # E: str + +reveal_type(np.__all__) # E: list[builtins.str] +reveal_type(np.char.__all__) # E: list[builtins.str] +reveal_type(np.ctypeslib.__all__) # E: list[builtins.str] +reveal_type(np.emath.__all__) # E: list[builtins.str] +reveal_type(np.lib.__all__) # E: list[builtins.str] +reveal_type(np.ma.__all__) # E: list[builtins.str] +reveal_type(np.random.__all__) # E: list[builtins.str] +reveal_type(np.rec.__all__) # E: list[builtins.str] +reveal_type(np.testing.__all__) # E: list[builtins.str] +reveal_type(f2py.__all__) # E: list[builtins.str] diff --git a/numpy/typing/tests/data/reveal/multiarray.py b/numpy/typing/tests/data/reveal/multiarray.py new file mode 100644 index 000000000000..33e9ede7cc54 --- /dev/null +++ b/numpy/typing/tests/data/reveal/multiarray.py @@ -0,0 +1,35 @@ +from typing import Any +import numpy as np + +AR_f8: np.ndarray[Any, np.dtype[np.float64]] +AR_i8: np.ndarray[Any, np.dtype[np.int64]] + +b_f8 = np.broadcast(AR_f8) +b_i8_f8_f8 = np.broadcast(AR_i8, AR_f8, AR_f8) + +reveal_type(next(b_f8)) # E: tuple[Any] +reveal_type(next(b_i8_f8_f8)) # E: tuple[Any] + +reveal_type(b_f8.reset()) # E: None +reveal_type(b_i8_f8_f8.reset()) # E: None + +reveal_type(b_f8.index) # E: int +reveal_type(b_i8_f8_f8.index) # E: int + +reveal_type(b_f8.iters) # E: tuple[numpy.flatiter[Any]] +reveal_type(b_i8_f8_f8.iters) # E: tuple[numpy.flatiter[Any]] + +reveal_type(b_f8.nd) # E: int +reveal_type(b_i8_f8_f8.nd) # E: int + +reveal_type(b_f8.ndim) # E: int +reveal_type(b_i8_f8_f8.ndim) # E: int + +reveal_type(b_f8.numiter) # E: int +reveal_type(b_i8_f8_f8.numiter) # E: int + +reveal_type(b_f8.shape) # E: tuple[builtins.int] +reveal_type(b_i8_f8_f8.shape) # E: tuple[builtins.int] + +reveal_type(b_f8.size) # E: int +reveal_type(b_i8_f8_f8.size) # E: int diff --git a/numpy/typing/tests/data/reveal/nbit_base_example.py b/numpy/typing/tests/data/reveal/nbit_base_example.py index 0c4c53f9b1f1..d34f6f69a31d 100644 --- a/numpy/typing/tests/data/reveal/nbit_base_example.py +++ b/numpy/typing/tests/data/reveal/nbit_base_example.py @@ -2,9 +2,10 @@ import numpy as np import numpy.typing as npt -T = TypeVar("T", bound=npt.NBitBase) +T1 = TypeVar("T1", bound=npt.NBitBase) +T2 = TypeVar("T2", bound=npt.NBitBase) -def add(a: np.floating[T], b: np.integer[T]) -> np.floating[T]: +def add(a: np.floating[T1], b: np.integer[T2]) -> np.floating[Union[T1, T2]]: return a + b i8: np.int64 @@ -12,7 +13,7 @@ def add(a: np.floating[T], b: np.integer[T]) -> np.floating[T]: f8: np.float64 f4: np.float32 -reveal_type(add(f8, i8)) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(add(f4, i8)) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(add(f8, i4)) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(add(f4, i4)) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(add(f8, i8)) # E: {float64} +reveal_type(add(f4, i8)) # E: {float64} +reveal_type(add(f8, i4)) # E: {float64} +reveal_type(add(f4, i4)) # E: {float32} diff --git a/numpy/typing/tests/data/reveal/ndarray_misc.py b/numpy/typing/tests/data/reveal/ndarray_misc.py index 826c8aaa6d12..ea01b7aa4e43 100644 --- a/numpy/typing/tests/data/reveal/ndarray_misc.py +++ b/numpy/typing/tests/data/reveal/ndarray_misc.py @@ -6,145 +6,186 @@ """ +import operator +import ctypes as ct +from typing import Any + import numpy as np class SubClass(np.ndarray): ... f8: np.float64 -A: np.ndarray B: SubClass +AR_f8: np.ndarray[Any, np.dtype[np.float64]] +AR_i8: np.ndarray[Any, np.dtype[np.int64]] +AR_U: np.ndarray[Any, np.dtype[np.str_]] + +ctypes_obj = AR_f8.ctypes + +reveal_type(ctypes_obj.data) # E: int +reveal_type(ctypes_obj.shape) # E: ctypes.Array[ctypes.c_int64] +reveal_type(ctypes_obj.strides) # E: ctypes.Array[ctypes.c_int64] +reveal_type(ctypes_obj._as_parameter_) # E: ctypes.c_void_p + +reveal_type(ctypes_obj.data_as(ct.c_void_p)) # E: ctypes.c_void_p +reveal_type(ctypes_obj.shape_as(ct.c_longlong)) # E: ctypes.Array[ctypes.c_longlong] +reveal_type(ctypes_obj.strides_as(ct.c_ubyte)) # E: ctypes.Array[ctypes.c_ubyte] reveal_type(f8.all()) # E: numpy.bool_ -reveal_type(A.all()) # E: numpy.bool_ -reveal_type(A.all(axis=0)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(A.all(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(A.all(out=B)) # E: SubClass +reveal_type(AR_f8.all()) # E: numpy.bool_ +reveal_type(AR_f8.all(axis=0)) # E: Any +reveal_type(AR_f8.all(keepdims=True)) # E: Any +reveal_type(AR_f8.all(out=B)) # E: SubClass reveal_type(f8.any()) # E: numpy.bool_ -reveal_type(A.any()) # E: numpy.bool_ -reveal_type(A.any(axis=0)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(A.any(keepdims=True)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(A.any(out=B)) # E: SubClass - -reveal_type(f8.argmax()) # E: numpy.signedinteger[Any] -reveal_type(A.argmax()) # E: numpy.signedinteger[Any] -reveal_type(A.argmax(axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] -reveal_type(A.argmax(out=B)) # E: SubClass - -reveal_type(f8.argmin()) # E: numpy.signedinteger[Any] -reveal_type(A.argmin()) # E: numpy.signedinteger[Any] -reveal_type(A.argmin(axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] -reveal_type(A.argmin(out=B)) # E: SubClass - -reveal_type(f8.argsort()) # E: numpy.ndarray -reveal_type(A.argsort()) # E: numpy.ndarray - -reveal_type(f8.astype(np.int64).choose([()])) # E: numpy.ndarray -reveal_type(A.choose([0])) # E: numpy.ndarray -reveal_type(A.choose([0], out=B)) # E: SubClass - -reveal_type(f8.clip(1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.clip(1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.clip(None, 1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.clip(1, out=B)) # E: SubClass -reveal_type(A.clip(None, 1, out=B)) # E: SubClass - -reveal_type(f8.compress([0])) # E: numpy.ndarray -reveal_type(A.compress([0])) # E: numpy.ndarray -reveal_type(A.compress([0], out=B)) # E: SubClass - -reveal_type(f8.conj()) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(A.conj()) # E: numpy.ndarray +reveal_type(AR_f8.any()) # E: numpy.bool_ +reveal_type(AR_f8.any(axis=0)) # E: Any +reveal_type(AR_f8.any(keepdims=True)) # E: Any +reveal_type(AR_f8.any(out=B)) # E: SubClass + +reveal_type(f8.argmax()) # E: {intp} +reveal_type(AR_f8.argmax()) # E: {intp} +reveal_type(AR_f8.argmax(axis=0)) # E: Any +reveal_type(AR_f8.argmax(out=B)) # E: SubClass + +reveal_type(f8.argmin()) # E: {intp} +reveal_type(AR_f8.argmin()) # E: {intp} +reveal_type(AR_f8.argmin(axis=0)) # E: Any +reveal_type(AR_f8.argmin(out=B)) # E: SubClass + +reveal_type(f8.argsort()) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.argsort()) # E: numpy.ndarray[Any, Any] + +reveal_type(f8.astype(np.int64).choose([()])) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.choose([0])) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.choose([0], out=B)) # E: SubClass + +reveal_type(f8.clip(1)) # E: Any +reveal_type(AR_f8.clip(1)) # E: Any +reveal_type(AR_f8.clip(None, 1)) # E: Any +reveal_type(AR_f8.clip(1, out=B)) # E: SubClass +reveal_type(AR_f8.clip(None, 1, out=B)) # E: SubClass + +reveal_type(f8.compress([0])) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.compress([0])) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.compress([0], out=B)) # E: SubClass + +reveal_type(f8.conj()) # E: {float64} +reveal_type(AR_f8.conj()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] reveal_type(B.conj()) # E: SubClass -reveal_type(f8.conjugate()) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(A.conjugate()) # E: numpy.ndarray +reveal_type(f8.conjugate()) # E: {float64} +reveal_type(AR_f8.conjugate()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] reveal_type(B.conjugate()) # E: SubClass -reveal_type(f8.cumprod()) # E: numpy.ndarray -reveal_type(A.cumprod()) # E: numpy.ndarray -reveal_type(A.cumprod(out=B)) # E: SubClass - -reveal_type(f8.cumsum()) # E: numpy.ndarray -reveal_type(A.cumsum()) # E: numpy.ndarray -reveal_type(A.cumsum(out=B)) # E: SubClass - -reveal_type(f8.max()) # E: numpy.number[Any] -reveal_type(A.max()) # E: numpy.number[Any] -reveal_type(A.max(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.max(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.max(out=B)) # E: SubClass - -reveal_type(f8.mean()) # E: numpy.number[Any] -reveal_type(A.mean()) # E: numpy.number[Any] -reveal_type(A.mean(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.mean(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.mean(out=B)) # E: SubClass - -reveal_type(f8.min()) # E: numpy.number[Any] -reveal_type(A.min()) # E: numpy.number[Any] -reveal_type(A.min(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.min(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.min(out=B)) # E: SubClass - -reveal_type(f8.newbyteorder()) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(A.newbyteorder()) # E: numpy.ndarray +reveal_type(f8.cumprod()) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.cumprod()) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.cumprod(out=B)) # E: SubClass + +reveal_type(f8.cumsum()) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.cumsum()) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.cumsum(out=B)) # E: SubClass + +reveal_type(f8.max()) # E: Any +reveal_type(AR_f8.max()) # E: Any +reveal_type(AR_f8.max(axis=0)) # E: Any +reveal_type(AR_f8.max(keepdims=True)) # E: Any +reveal_type(AR_f8.max(out=B)) # E: SubClass + +reveal_type(f8.mean()) # E: Any +reveal_type(AR_f8.mean()) # E: Any +reveal_type(AR_f8.mean(axis=0)) # E: Any +reveal_type(AR_f8.mean(keepdims=True)) # E: Any +reveal_type(AR_f8.mean(out=B)) # E: SubClass + +reveal_type(f8.min()) # E: Any +reveal_type(AR_f8.min()) # E: Any +reveal_type(AR_f8.min(axis=0)) # E: Any +reveal_type(AR_f8.min(keepdims=True)) # E: Any +reveal_type(AR_f8.min(out=B)) # E: SubClass + +reveal_type(f8.newbyteorder()) # E: {float64} +reveal_type(AR_f8.newbyteorder()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] reveal_type(B.newbyteorder('|')) # E: SubClass -reveal_type(f8.prod()) # E: numpy.number[Any] -reveal_type(A.prod()) # E: numpy.number[Any] -reveal_type(A.prod(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.prod(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.prod(out=B)) # E: SubClass +reveal_type(f8.prod()) # E: Any +reveal_type(AR_f8.prod()) # E: Any +reveal_type(AR_f8.prod(axis=0)) # E: Any +reveal_type(AR_f8.prod(keepdims=True)) # E: Any +reveal_type(AR_f8.prod(out=B)) # E: SubClass + +reveal_type(f8.ptp()) # E: Any +reveal_type(AR_f8.ptp()) # E: Any +reveal_type(AR_f8.ptp(axis=0)) # E: Any +reveal_type(AR_f8.ptp(keepdims=True)) # E: Any +reveal_type(AR_f8.ptp(out=B)) # E: SubClass + +reveal_type(f8.round()) # E: {float64} +reveal_type(AR_f8.round()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(AR_f8.round(out=B)) # E: SubClass + +reveal_type(f8.repeat(1)) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(AR_f8.repeat(1)) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(B.repeat(1)) # E: numpy.ndarray[Any, Any] + +reveal_type(f8.std()) # E: Any +reveal_type(AR_f8.std()) # E: Any +reveal_type(AR_f8.std(axis=0)) # E: Any +reveal_type(AR_f8.std(keepdims=True)) # E: Any +reveal_type(AR_f8.std(out=B)) # E: SubClass + +reveal_type(f8.sum()) # E: Any +reveal_type(AR_f8.sum()) # E: Any +reveal_type(AR_f8.sum(axis=0)) # E: Any +reveal_type(AR_f8.sum(keepdims=True)) # E: Any +reveal_type(AR_f8.sum(out=B)) # E: SubClass + +reveal_type(f8.take(0)) # E: {float64} +reveal_type(AR_f8.take(0)) # E: {float64} +reveal_type(AR_f8.take([0])) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(AR_f8.take(0, out=B)) # E: SubClass +reveal_type(AR_f8.take([0], out=B)) # E: SubClass + +reveal_type(f8.var()) # E: Any +reveal_type(AR_f8.var()) # E: Any +reveal_type(AR_f8.var(axis=0)) # E: Any +reveal_type(AR_f8.var(keepdims=True)) # E: Any +reveal_type(AR_f8.var(out=B)) # E: SubClass + +reveal_type(AR_f8.argpartition([0])) # E: numpy.ndarray[Any, numpy.dtype[{intp}]] -reveal_type(f8.ptp()) # E: numpy.number[Any] -reveal_type(A.ptp()) # E: numpy.number[Any] -reveal_type(A.ptp(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.ptp(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.ptp(out=B)) # E: SubClass +reveal_type(AR_f8.diagonal()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] -reveal_type(f8.round()) # E: numpy.floating[numpy.typing._64Bit] -reveal_type(A.round()) # E: numpy.ndarray -reveal_type(A.round(out=B)) # E: SubClass +reveal_type(AR_f8.dot(1)) # E: numpy.ndarray[Any, Any] +reveal_type(AR_f8.dot([1])) # E: Any +reveal_type(AR_f8.dot(1, out=B)) # E: SubClass -reveal_type(f8.repeat(1)) # E: numpy.ndarray -reveal_type(A.repeat(1)) # E: numpy.ndarray -reveal_type(B.repeat(1)) # E: numpy.ndarray +reveal_type(AR_f8.nonzero()) # E: tuple[numpy.ndarray[Any, numpy.dtype[{intp}]]] -reveal_type(f8.std()) # E: numpy.number[Any] -reveal_type(A.std()) # E: numpy.number[Any] -reveal_type(A.std(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.std(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.std(out=B)) # E: SubClass +reveal_type(AR_f8.searchsorted(1)) # E: {intp} +reveal_type(AR_f8.searchsorted([1])) # E: numpy.ndarray[Any, numpy.dtype[{intp}]] -reveal_type(f8.sum()) # E: numpy.number[Any] -reveal_type(A.sum()) # E: numpy.number[Any] -reveal_type(A.sum(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.sum(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.sum(out=B)) # E: SubClass +reveal_type(AR_f8.trace()) # E: Any +reveal_type(AR_f8.trace(out=B)) # E: SubClass -reveal_type(f8.take(0)) # E: numpy.generic -reveal_type(A.take(0)) # E: numpy.generic -reveal_type(A.take([0])) # E: numpy.ndarray -reveal_type(A.take(0, out=B)) # E: SubClass -reveal_type(A.take([0], out=B)) # E: SubClass +reveal_type(AR_f8.item()) # E: float +reveal_type(AR_U.item()) # E: str -reveal_type(f8.var()) # E: numpy.number[Any] -reveal_type(A.var()) # E: numpy.number[Any] -reveal_type(A.var(axis=0)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.var(keepdims=True)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.var(out=B)) # E: SubClass +reveal_type(AR_f8.ravel()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(AR_U.ravel()) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] -reveal_type(A.argpartition([0])) # E: numpy.ndarray +reveal_type(AR_f8.flatten()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(AR_U.flatten()) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] -reveal_type(A.diagonal()) # E: numpy.ndarray +reveal_type(AR_f8.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(AR_U.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] -reveal_type(A.dot(1)) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.dot(1, out=B)) # E: SubClass +reveal_type(int(AR_f8)) # E: int +reveal_type(int(AR_U)) # E: int -reveal_type(A.nonzero()) # E: tuple[numpy.ndarray] +reveal_type(float(AR_f8)) # E: float +reveal_type(float(AR_U)) # E: float -reveal_type(A.searchsorted([1])) # E: numpy.ndarray +reveal_type(complex(AR_f8)) # E: complex -reveal_type(A.trace()) # E: Union[numpy.number[Any], numpy.ndarray] -reveal_type(A.trace(out=B)) # E: SubClass +reveal_type(operator.index(AR_i8)) # E: int diff --git a/numpy/typing/tests/data/reveal/numeric.py b/numpy/typing/tests/data/reveal/numeric.py index 5cbfa4ac765c..ec6e47ca05d0 100644 --- a/numpy/typing/tests/data/reveal/numeric.py +++ b/numpy/typing/tests/data/reveal/numeric.py @@ -20,53 +20,53 @@ class SubClass(np.ndarray): reveal_type(np.count_nonzero(i8)) # E: int reveal_type(np.count_nonzero(A)) # E: int reveal_type(np.count_nonzero(B)) # E: int -reveal_type(np.count_nonzero(A, keepdims=True)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] -reveal_type(np.count_nonzero(A, axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] +reveal_type(np.count_nonzero(A, keepdims=True)) # E: Any +reveal_type(np.count_nonzero(A, axis=0)) # E: Any reveal_type(np.isfortran(i8)) # E: bool reveal_type(np.isfortran(A)) # E: bool -reveal_type(np.argwhere(i8)) # E: numpy.ndarray -reveal_type(np.argwhere(A)) # E: numpy.ndarray +reveal_type(np.argwhere(i8)) # E: numpy.ndarray[Any, Any] +reveal_type(np.argwhere(A)) # E: numpy.ndarray[Any, Any] -reveal_type(np.flatnonzero(i8)) # E: numpy.ndarray -reveal_type(np.flatnonzero(A)) # E: numpy.ndarray +reveal_type(np.flatnonzero(i8)) # E: numpy.ndarray[Any, Any] +reveal_type(np.flatnonzero(A)) # E: numpy.ndarray[Any, Any] -reveal_type(np.correlate(B, A, mode="valid")) # E: numpy.ndarray -reveal_type(np.correlate(A, A, mode="same")) # E: numpy.ndarray +reveal_type(np.correlate(B, A, mode="valid")) # E: numpy.ndarray[Any, Any] +reveal_type(np.correlate(A, A, mode="same")) # E: numpy.ndarray[Any, Any] -reveal_type(np.convolve(B, A, mode="valid")) # E: numpy.ndarray -reveal_type(np.convolve(A, A, mode="same")) # E: numpy.ndarray +reveal_type(np.convolve(B, A, mode="valid")) # E: numpy.ndarray[Any, Any] +reveal_type(np.convolve(A, A, mode="same")) # E: numpy.ndarray[Any, Any] -reveal_type(np.outer(i8, A)) # E: numpy.ndarray -reveal_type(np.outer(B, A)) # E: numpy.ndarray -reveal_type(np.outer(A, A)) # E: numpy.ndarray +reveal_type(np.outer(i8, A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.outer(B, A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.outer(A, A)) # E: numpy.ndarray[Any, Any] reveal_type(np.outer(A, A, out=C)) # E: SubClass -reveal_type(np.tensordot(B, A)) # E: numpy.ndarray -reveal_type(np.tensordot(A, A)) # E: numpy.ndarray -reveal_type(np.tensordot(A, A, axes=0)) # E: numpy.ndarray -reveal_type(np.tensordot(A, A, axes=(0, 1))) # E: numpy.ndarray +reveal_type(np.tensordot(B, A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.tensordot(A, A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.tensordot(A, A, axes=0)) # E: numpy.ndarray[Any, Any] +reveal_type(np.tensordot(A, A, axes=(0, 1))) # E: numpy.ndarray[Any, Any] reveal_type(np.isscalar(i8)) # E: bool reveal_type(np.isscalar(A)) # E: bool reveal_type(np.isscalar(B)) # E: bool -reveal_type(np.roll(A, 1)) # E: numpy.ndarray -reveal_type(np.roll(A, (1, 2))) # E: numpy.ndarray -reveal_type(np.roll(B, 1)) # E: numpy.ndarray +reveal_type(np.roll(A, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.roll(A, (1, 2))) # E: numpy.ndarray[Any, Any] +reveal_type(np.roll(B, 1)) # E: numpy.ndarray[Any, Any] -reveal_type(np.rollaxis(A, 0, 1)) # E: numpy.ndarray +reveal_type(np.rollaxis(A, 0, 1)) # E: numpy.ndarray[Any, Any] -reveal_type(np.moveaxis(A, 0, 1)) # E: numpy.ndarray -reveal_type(np.moveaxis(A, (0, 1), (1, 2))) # E: numpy.ndarray +reveal_type(np.moveaxis(A, 0, 1)) # E: numpy.ndarray[Any, Any] +reveal_type(np.moveaxis(A, (0, 1), (1, 2))) # E: numpy.ndarray[Any, Any] -reveal_type(np.cross(B, A)) # E: numpy.ndarray -reveal_type(np.cross(A, A)) # E: numpy.ndarray +reveal_type(np.cross(B, A)) # E: numpy.ndarray[Any, Any] +reveal_type(np.cross(A, A)) # E: numpy.ndarray[Any, Any] -reveal_type(np.indices([0, 1, 2])) # E: numpy.ndarray -reveal_type(np.indices([0, 1, 2], sparse=False)) # E: numpy.ndarray -reveal_type(np.indices([0, 1, 2], sparse=True)) # E: tuple[numpy.ndarray] +reveal_type(np.indices([0, 1, 2])) # E: numpy.ndarray[Any, Any] +reveal_type(np.indices([0, 1, 2], sparse=False)) # E: numpy.ndarray[Any, Any] +reveal_type(np.indices([0, 1, 2], sparse=True)) # E: tuple[numpy.ndarray[Any, Any]] reveal_type(np.binary_repr(1)) # E: str @@ -76,9 +76,9 @@ class SubClass(np.ndarray): reveal_type(np.allclose(B, A)) # E: bool reveal_type(np.allclose(A, A)) # E: bool -reveal_type(np.isclose(i8, A)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.isclose(B, A)) # E: Union[numpy.bool_, numpy.ndarray] -reveal_type(np.isclose(A, A)) # E: Union[numpy.bool_, numpy.ndarray] +reveal_type(np.isclose(i8, A)) # E: Any +reveal_type(np.isclose(B, A)) # E: Any +reveal_type(np.isclose(A, A)) # E: Any reveal_type(np.array_equal(i8, A)) # E: bool reveal_type(np.array_equal(B, A)) # E: bool diff --git a/numpy/typing/tests/data/reveal/numerictypes.py b/numpy/typing/tests/data/reveal/numerictypes.py index e026158cd14f..0f886b3fbff4 100644 --- a/numpy/typing/tests/data/reveal/numerictypes.py +++ b/numpy/typing/tests/data/reveal/numerictypes.py @@ -16,3 +16,21 @@ reveal_type(np.sctype2char(list)) # E: str reveal_type(np.find_common_type([np.int64], [np.int64])) # E: numpy.dtype + +reveal_type(np.cast[int]) # E: _CastFunc +reveal_type(np.cast["i8"]) # E: _CastFunc +reveal_type(np.cast[np.int64]) # E: _CastFunc + +reveal_type(np.nbytes[int]) # E: int +reveal_type(np.nbytes["i8"]) # E: int +reveal_type(np.nbytes[np.int64]) # E: int + +reveal_type(np.ScalarType) # E: Tuple +reveal_type(np.ScalarType[0]) # E: Type[builtins.int] +reveal_type(np.ScalarType[4]) # E: Type[builtins.bool] +reveal_type(np.ScalarType[9]) # E: Type[{csingle}] +reveal_type(np.ScalarType[11]) # E: Type[{clongdouble}] + +reveal_type(np.typecodes["Character"]) # E: Literal['c'] +reveal_type(np.typecodes["Complex"]) # E: Literal['FDG'] +reveal_type(np.typecodes["All"]) # E: Literal['?bhilqpBHILQPefdgFDGSUVOMm'] diff --git a/numpy/typing/tests/data/reveal/random.py b/numpy/typing/tests/data/reveal/random.py new file mode 100644 index 000000000000..6fc35aced713 --- /dev/null +++ b/numpy/typing/tests/data/reveal/random.py @@ -0,0 +1,1539 @@ +from __future__ import annotations + +from typing import Any, List + +import numpy as np + +def_rng = np.random.default_rng() +seed_seq = np.random.SeedSequence() +mt19937 = np.random.MT19937() +pcg64 = np.random.PCG64() +sfc64 = np.random.SFC64() +philox = np.random.Philox() +seedless_seq = np.random.bit_generator.SeedlessSeedSequence() + +reveal_type(def_rng) # E: numpy.random._generator.Generator +reveal_type(mt19937) # E: numpy.random._mt19937.MT19937 +reveal_type(pcg64) # E: numpy.random._pcg64.PCG64 +reveal_type(sfc64) # E: numpy.random._sfc64.SFC64 +reveal_type(philox) # E: numpy.random._philox.Philox +reveal_type(seed_seq) # E: numpy.random.bit_generator.SeedSequence +reveal_type(seedless_seq) # E: numpy.random.bit_generator.SeedlessSeedSequence + +mt19937_jumped = mt19937.jumped() +mt19937_jumped3 = mt19937.jumped(3) +mt19937_raw = mt19937.random_raw() +mt19937_raw_arr = mt19937.random_raw(5) + +reveal_type(mt19937_jumped) # E: numpy.random._mt19937.MT19937 +reveal_type(mt19937_jumped3) # E: numpy.random._mt19937.MT19937 +reveal_type(mt19937_raw) # E: int +reveal_type(mt19937_raw_arr) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(mt19937.lock) # E: threading.Lock + +pcg64_jumped = pcg64.jumped() +pcg64_jumped3 = pcg64.jumped(3) +pcg64_adv = pcg64.advance(3) +pcg64_raw = pcg64.random_raw() +pcg64_raw_arr = pcg64.random_raw(5) + +reveal_type(pcg64_jumped) # E: numpy.random._pcg64.PCG64 +reveal_type(pcg64_jumped3) # E: numpy.random._pcg64.PCG64 +reveal_type(pcg64_adv) # E: numpy.random._pcg64.PCG64 +reveal_type(pcg64_raw) # E: int +reveal_type(pcg64_raw_arr) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(pcg64.lock) # E: threading.Lock + +philox_jumped = philox.jumped() +philox_jumped3 = philox.jumped(3) +philox_adv = philox.advance(3) +philox_raw = philox.random_raw() +philox_raw_arr = philox.random_raw(5) + +reveal_type(philox_jumped) # E: numpy.random._philox.Philox +reveal_type(philox_jumped3) # E: numpy.random._philox.Philox +reveal_type(philox_adv) # E: numpy.random._philox.Philox +reveal_type(philox_raw) # E: int +reveal_type(philox_raw_arr) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(philox.lock) # E: threading.Lock + +sfc64_raw = sfc64.random_raw() +sfc64_raw_arr = sfc64.random_raw(5) + +reveal_type(sfc64_raw) # E: int +reveal_type(sfc64_raw_arr) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(sfc64.lock) # E: threading.Lock + +reveal_type(seed_seq.pool) # numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(seed_seq.entropy) # E:Union[None, int, Sequence[int]] +reveal_type(seed_seq.spawn(1)) # E: list[numpy.random.bit_generator.SeedSequence] +reveal_type(seed_seq.generate_state(8, "uint32")) # E: numpy.ndarray[Any, numpy.dtype[Union[numpy.unsignedinteger[numpy.typing._32Bit], numpy.unsignedinteger[numpy.typing._64Bit]]]] +reveal_type(seed_seq.generate_state(8, "uint64")) # E: numpy.ndarray[Any, numpy.dtype[Union[numpy.unsignedinteger[numpy.typing._32Bit], numpy.unsignedinteger[numpy.typing._64Bit]]]] + + +def_gen: np.random.Generator = np.random.default_rng() + +D_arr_0p1: np.ndarray[Any, np.dtype[np.float64]] = np.array([0.1]) +D_arr_0p5: np.ndarray[Any, np.dtype[np.float64]] = np.array([0.5]) +D_arr_0p9: np.ndarray[Any, np.dtype[np.float64]] = np.array([0.9]) +D_arr_1p5: np.ndarray[Any, np.dtype[np.float64]] = np.array([1.5]) +I_arr_10: np.ndarray[Any, np.dtype[np.int_]] = np.array([10], dtype=np.int_) +I_arr_20: np.ndarray[Any, np.dtype[np.int_]] = np.array([20], dtype=np.int_) +D_arr_like_0p1: List[float] = [0.1] +D_arr_like_0p5: List[float] = [0.5] +D_arr_like_0p9: List[float] = [0.9] +D_arr_like_1p5: List[float] = [1.5] +I_arr_like_10: List[int] = [10] +I_arr_like_20: List[int] = [20] +D_2D_like: List[List[float]] = [[1, 2], [2, 3], [3, 4], [4, 5.1]] +D_2D: np.ndarray[Any, np.dtype[np.float64]] = np.array(D_2D_like) +S_out: np.ndarray[Any, np.dtype[np.float32]] = np.empty(1, dtype=np.float32) +D_out: np.ndarray[Any, np.dtype[np.float64]] = np.empty(1) + +reveal_type(def_gen.standard_normal()) # E: float +reveal_type(def_gen.standard_normal(dtype=np.float32)) # E: float +reveal_type(def_gen.standard_normal(dtype="float32")) # E: float +reveal_type(def_gen.standard_normal(dtype="double")) # E: float +reveal_type(def_gen.standard_normal(dtype=np.float64)) # E: float +reveal_type(def_gen.standard_normal(size=None)) # E: float +reveal_type(def_gen.standard_normal(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype=np.float32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype="f4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype="float32", out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_normal(dtype=np.float32, out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype=np.float64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype="float64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype="f8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_normal(out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype="float64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_normal(size=1, dtype="float64", out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(def_gen.random()) # E: float +reveal_type(def_gen.random(dtype=np.float32)) # E: float +reveal_type(def_gen.random(dtype="float32")) # E: float +reveal_type(def_gen.random(dtype="double")) # E: float +reveal_type(def_gen.random(dtype=np.float64)) # E: float +reveal_type(def_gen.random(size=None)) # E: float +reveal_type(def_gen.random(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.random(size=1, dtype=np.float32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.random(size=1, dtype="f4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.random(size=1, dtype="float32", out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.random(dtype=np.float32, out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.random(size=1, dtype=np.float64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.random(size=1, dtype="float64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.random(size=1, dtype="f8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.random(out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.random(size=1, dtype="float64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.random(size=1, dtype="float64", out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(def_gen.standard_cauchy()) # E: float +reveal_type(def_gen.standard_cauchy(size=None)) # E: float +reveal_type(def_gen.standard_cauchy(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.standard_exponential()) # E: float +reveal_type(def_gen.standard_exponential(method="inv")) # E: float +reveal_type(def_gen.standard_exponential(dtype=np.float32)) # E: float +reveal_type(def_gen.standard_exponential(dtype="float32")) # E: float +reveal_type(def_gen.standard_exponential(dtype="double")) # E: float +reveal_type(def_gen.standard_exponential(dtype=np.float64)) # E: float +reveal_type(def_gen.standard_exponential(size=None)) # E: float +reveal_type(def_gen.standard_exponential(size=None, method="inv")) # E: float +reveal_type(def_gen.standard_exponential(size=1, method="inv")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype=np.float32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype="f4", method="inv")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype="float32", out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_exponential(dtype=np.float32, out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype=np.float64, method="inv")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype="float64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype="f8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_exponential(out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype="float64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_exponential(size=1, dtype="float64", out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(def_gen.zipf(1.5)) # E: int +reveal_type(def_gen.zipf(1.5, size=None)) # E: int +reveal_type(def_gen.zipf(1.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.zipf(D_arr_1p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.zipf(D_arr_1p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.zipf(D_arr_like_1p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.zipf(D_arr_like_1p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.weibull(0.5)) # E: float +reveal_type(def_gen.weibull(0.5, size=None)) # E: float +reveal_type(def_gen.weibull(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.weibull(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.weibull(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.weibull(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.weibull(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.standard_t(0.5)) # E: float +reveal_type(def_gen.standard_t(0.5, size=None)) # E: float +reveal_type(def_gen.standard_t(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.standard_t(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.standard_t(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.standard_t(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.standard_t(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.poisson(0.5)) # E: int +reveal_type(def_gen.poisson(0.5, size=None)) # E: int +reveal_type(def_gen.poisson(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.poisson(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.poisson(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.poisson(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.poisson(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.power(0.5)) # E: float +reveal_type(def_gen.power(0.5, size=None)) # E: float +reveal_type(def_gen.power(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.power(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.power(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.power(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.power(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.pareto(0.5)) # E: float +reveal_type(def_gen.pareto(0.5, size=None)) # E: float +reveal_type(def_gen.pareto(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.pareto(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.pareto(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.pareto(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.pareto(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.chisquare(0.5)) # E: float +reveal_type(def_gen.chisquare(0.5, size=None)) # E: float +reveal_type(def_gen.chisquare(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.chisquare(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.chisquare(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.chisquare(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.chisquare(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.exponential(0.5)) # E: float +reveal_type(def_gen.exponential(0.5, size=None)) # E: float +reveal_type(def_gen.exponential(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.exponential(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.exponential(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.exponential(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.exponential(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.geometric(0.5)) # E: int +reveal_type(def_gen.geometric(0.5, size=None)) # E: int +reveal_type(def_gen.geometric(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.geometric(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.geometric(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.geometric(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.geometric(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.logseries(0.5)) # E: int +reveal_type(def_gen.logseries(0.5, size=None)) # E: int +reveal_type(def_gen.logseries(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.logseries(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.logseries(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.logseries(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.logseries(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.rayleigh(0.5)) # E: float +reveal_type(def_gen.rayleigh(0.5, size=None)) # E: float +reveal_type(def_gen.rayleigh(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.rayleigh(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.rayleigh(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.rayleigh(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.rayleigh(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.standard_gamma(0.5)) # E: float +reveal_type(def_gen.standard_gamma(0.5, size=None)) # E: float +reveal_type(def_gen.standard_gamma(0.5, dtype="float32")) # E: float +reveal_type(def_gen.standard_gamma(0.5, size=None, dtype="float32")) # E: float +reveal_type(def_gen.standard_gamma(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_0p5, dtype="f4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_gamma(0.5, size=1, dtype="float32", out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_0p5, dtype=np.float32, out=S_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._32Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(0.5, out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_like_0p5, out=D_out)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(def_gen.standard_gamma(D_arr_like_0p5, size=1, out=D_out, dtype=np.float64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(def_gen.vonmises(0.5, 0.5)) # E: float +reveal_type(def_gen.vonmises(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.vonmises(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.vonmises(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.wald(0.5, 0.5)) # E: float +reveal_type(def_gen.wald(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.wald(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.wald(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.uniform(0.5, 0.5)) # E: float +reveal_type(def_gen.uniform(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.uniform(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.uniform(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.beta(0.5, 0.5)) # E: float +reveal_type(def_gen.beta(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.beta(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.beta(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.f(0.5, 0.5)) # E: float +reveal_type(def_gen.f(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.f(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.f(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.gamma(0.5, 0.5)) # E: float +reveal_type(def_gen.gamma(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.gamma(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gamma(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.gumbel(0.5, 0.5)) # E: float +reveal_type(def_gen.gumbel(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.gumbel(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.gumbel(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.laplace(0.5, 0.5)) # E: float +reveal_type(def_gen.laplace(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.laplace(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.laplace(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.logistic(0.5, 0.5)) # E: float +reveal_type(def_gen.logistic(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.logistic(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.logistic(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.lognormal(0.5, 0.5)) # E: float +reveal_type(def_gen.lognormal(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.lognormal(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.lognormal(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.noncentral_chisquare(0.5, 0.5)) # E: float +reveal_type(def_gen.noncentral_chisquare(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.noncentral_chisquare(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.normal(0.5, 0.5)) # E: float +reveal_type(def_gen.normal(0.5, 0.5, size=None)) # E: float +reveal_type(def_gen.normal(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.normal(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.triangular(0.1, 0.5, 0.9)) # E: float +reveal_type(def_gen.triangular(0.1, 0.5, 0.9, size=None)) # E: float +reveal_type(def_gen.triangular(0.1, 0.5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_0p1, 0.5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(0.1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_0p1, 0.5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(0.1, D_arr_0p5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_like_0p1, 0.5, D_arr_0p9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(0.5, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_0p1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_like_0p1, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.triangular(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.noncentral_f(0.1, 0.5, 0.9)) # E: float +reveal_type(def_gen.noncentral_f(0.1, 0.5, 0.9, size=None)) # E: float +reveal_type(def_gen.noncentral_f(0.1, 0.5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_0p1, 0.5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(0.1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_0p1, 0.5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(0.1, D_arr_0p5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_like_0p1, 0.5, D_arr_0p9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(0.5, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_0p1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.binomial(10, 0.5)) # E: int +reveal_type(def_gen.binomial(10, 0.5, size=None)) # E: int +reveal_type(def_gen.binomial(10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_like_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_like_10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.binomial(I_arr_like_10, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.negative_binomial(10, 0.5)) # E: int +reveal_type(def_gen.negative_binomial(10, 0.5, size=None)) # E: int +reveal_type(def_gen.negative_binomial(10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_like_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_like_10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.negative_binomial(I_arr_like_10, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.hypergeometric(20, 20, 10)) # E: int +reveal_type(def_gen.hypergeometric(20, 20, 10, size=None)) # E: int +reveal_type(def_gen.hypergeometric(20, 20, 10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_20, 20, 10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(20, I_arr_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_20, 20, I_arr_like_10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(20, I_arr_20, 10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_like_20, 20, I_arr_10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(20, I_arr_like_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_20, I_arr_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_like_20, I_arr_like_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_20, I_arr_20, I_arr_10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.hypergeometric(I_arr_like_20, I_arr_like_20, I_arr_like_10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +I_int64_100: np.ndarray[Any, np.dtype[np.int64]] = np.array([100], dtype=np.int64) + +reveal_type(def_gen.integers(0, 100)) # E: int +reveal_type(def_gen.integers(100)) # E: int +reveal_type(def_gen.integers([100])) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, [100])) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +I_bool_low: np.ndarray[Any, np.dtype[np.bool_]] = np.array([0], dtype=np.bool_) +I_bool_low_like: List[int] = [0] +I_bool_high_open: np.ndarray[Any, np.dtype[np.bool_]] = np.array([1], dtype=np.bool_) +I_bool_high_closed: np.ndarray[Any, np.dtype[np.bool_]] = np.array([1], dtype=np.bool_) + +reveal_type(def_gen.integers(2, dtype=bool)) # E: builtins.bool +reveal_type(def_gen.integers(0, 2, dtype=bool)) # E: builtins.bool +reveal_type(def_gen.integers(1, dtype=bool, endpoint=True)) # E: builtins.bool +reveal_type(def_gen.integers(0, 1, dtype=bool, endpoint=True)) # E: builtins.bool +reveal_type(def_gen.integers(I_bool_low_like, 1, dtype=bool, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_high_open, dtype=bool)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_low, I_bool_high_open, dtype=bool)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(0, I_bool_high_open, dtype=bool)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_high_closed, dtype=bool, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_low, I_bool_high_closed, dtype=bool, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(0, I_bool_high_closed, dtype=bool, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] + +reveal_type(def_gen.integers(2, dtype=np.bool_)) # E: builtins.bool +reveal_type(def_gen.integers(0, 2, dtype=np.bool_)) # E: builtins.bool +reveal_type(def_gen.integers(1, dtype=np.bool_, endpoint=True)) # E: builtins.bool +reveal_type(def_gen.integers(0, 1, dtype=np.bool_, endpoint=True)) # E: builtins.bool +reveal_type(def_gen.integers(I_bool_low_like, 1, dtype=np.bool_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_high_open, dtype=np.bool_)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_low, I_bool_high_open, dtype=np.bool_)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(0, I_bool_high_open, dtype=np.bool_)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_high_closed, dtype=np.bool_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(I_bool_low, I_bool_high_closed, dtype=np.bool_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(def_gen.integers(0, I_bool_high_closed, dtype=np.bool_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] + +I_u1_low: np.ndarray[Any, np.dtype[np.uint8]] = np.array([0], dtype=np.uint8) +I_u1_low_like: List[int] = [0] +I_u1_high_open: np.ndarray[Any, np.dtype[np.uint8]] = np.array([255], dtype=np.uint8) +I_u1_high_closed: np.ndarray[Any, np.dtype[np.uint8]] = np.array([255], dtype=np.uint8) + +reveal_type(def_gen.integers(256, dtype="u1")) # E: int +reveal_type(def_gen.integers(0, 256, dtype="u1")) # E: int +reveal_type(def_gen.integers(255, dtype="u1", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 255, dtype="u1", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u1_low_like, 255, dtype="u1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_high_open, dtype="u1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_low, I_u1_high_open, dtype="u1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(0, I_u1_high_open, dtype="u1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_high_closed, dtype="u1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_low, I_u1_high_closed, dtype="u1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(0, I_u1_high_closed, dtype="u1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] + +reveal_type(def_gen.integers(256, dtype="uint8")) # E: int +reveal_type(def_gen.integers(0, 256, dtype="uint8")) # E: int +reveal_type(def_gen.integers(255, dtype="uint8", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 255, dtype="uint8", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u1_low_like, 255, dtype="uint8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_high_open, dtype="uint8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_low, I_u1_high_open, dtype="uint8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(0, I_u1_high_open, dtype="uint8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_high_closed, dtype="uint8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_low, I_u1_high_closed, dtype="uint8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(0, I_u1_high_closed, dtype="uint8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] + +reveal_type(def_gen.integers(256, dtype=np.uint8)) # E: int +reveal_type(def_gen.integers(0, 256, dtype=np.uint8)) # E: int +reveal_type(def_gen.integers(255, dtype=np.uint8, endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 255, dtype=np.uint8, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u1_low_like, 255, dtype=np.uint8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_high_open, dtype=np.uint8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_low, I_u1_high_open, dtype=np.uint8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(0, I_u1_high_open, dtype=np.uint8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_high_closed, dtype=np.uint8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_u1_low, I_u1_high_closed, dtype=np.uint8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(0, I_u1_high_closed, dtype=np.uint8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] + +I_u2_low: np.ndarray[Any, np.dtype[np.uint16]] = np.array([0], dtype=np.uint16) +I_u2_low_like: List[int] = [0] +I_u2_high_open: np.ndarray[Any, np.dtype[np.uint16]] = np.array([65535], dtype=np.uint16) +I_u2_high_closed: np.ndarray[Any, np.dtype[np.uint16]] = np.array([65535], dtype=np.uint16) + +reveal_type(def_gen.integers(65536, dtype="u2")) # E: int +reveal_type(def_gen.integers(0, 65536, dtype="u2")) # E: int +reveal_type(def_gen.integers(65535, dtype="u2", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 65535, dtype="u2", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u2_low_like, 65535, dtype="u2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_high_open, dtype="u2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_low, I_u2_high_open, dtype="u2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(0, I_u2_high_open, dtype="u2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_high_closed, dtype="u2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_low, I_u2_high_closed, dtype="u2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(0, I_u2_high_closed, dtype="u2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] + +reveal_type(def_gen.integers(65536, dtype="uint16")) # E: int +reveal_type(def_gen.integers(0, 65536, dtype="uint16")) # E: int +reveal_type(def_gen.integers(65535, dtype="uint16", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 65535, dtype="uint16", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u2_low_like, 65535, dtype="uint16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_high_open, dtype="uint16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_low, I_u2_high_open, dtype="uint16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(0, I_u2_high_open, dtype="uint16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_high_closed, dtype="uint16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_low, I_u2_high_closed, dtype="uint16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(0, I_u2_high_closed, dtype="uint16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] + +reveal_type(def_gen.integers(65536, dtype=np.uint16)) # E: int +reveal_type(def_gen.integers(0, 65536, dtype=np.uint16)) # E: int +reveal_type(def_gen.integers(65535, dtype=np.uint16, endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 65535, dtype=np.uint16, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u2_low_like, 65535, dtype=np.uint16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_high_open, dtype=np.uint16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_low, I_u2_high_open, dtype=np.uint16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(0, I_u2_high_open, dtype=np.uint16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_high_closed, dtype=np.uint16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_u2_low, I_u2_high_closed, dtype=np.uint16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(0, I_u2_high_closed, dtype=np.uint16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] + +I_u4_low: np.ndarray[Any, np.dtype[np.uint32]] = np.array([0], dtype=np.uint32) +I_u4_low_like: List[int] = [0] +I_u4_high_open: np.ndarray[Any, np.dtype[np.uint32]] = np.array([4294967295], dtype=np.uint32) +I_u4_high_closed: np.ndarray[Any, np.dtype[np.uint32]] = np.array([4294967295], dtype=np.uint32) + +reveal_type(def_gen.integers(4294967296, dtype=np.int_)) # E: int +reveal_type(def_gen.integers(0, 4294967296, dtype=np.int_)) # E: int +reveal_type(def_gen.integers(4294967295, dtype=np.int_, endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 4294967295, dtype=np.int_, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u4_low_like, 4294967295, dtype=np.int_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(def_gen.integers(I_u4_high_open, dtype=np.int_)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_open, dtype=np.int_)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(def_gen.integers(0, I_u4_high_open, dtype=np.int_)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(def_gen.integers(I_u4_high_closed, dtype=np.int_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_closed, dtype=np.int_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(def_gen.integers(0, I_u4_high_closed, dtype=np.int_, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + + +reveal_type(def_gen.integers(4294967296, dtype="u4")) # E: int +reveal_type(def_gen.integers(0, 4294967296, dtype="u4")) # E: int +reveal_type(def_gen.integers(4294967295, dtype="u4", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 4294967295, dtype="u4", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u4_low_like, 4294967295, dtype="u4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_high_open, dtype="u4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_open, dtype="u4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(0, I_u4_high_open, dtype="u4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_high_closed, dtype="u4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_closed, dtype="u4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(0, I_u4_high_closed, dtype="u4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] + +reveal_type(def_gen.integers(4294967296, dtype="uint32")) # E: int +reveal_type(def_gen.integers(0, 4294967296, dtype="uint32")) # E: int +reveal_type(def_gen.integers(4294967295, dtype="uint32", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 4294967295, dtype="uint32", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u4_low_like, 4294967295, dtype="uint32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_high_open, dtype="uint32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_open, dtype="uint32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(0, I_u4_high_open, dtype="uint32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_high_closed, dtype="uint32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_closed, dtype="uint32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(0, I_u4_high_closed, dtype="uint32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] + +reveal_type(def_gen.integers(4294967296, dtype=np.uint32)) # E: int +reveal_type(def_gen.integers(0, 4294967296, dtype=np.uint32)) # E: int +reveal_type(def_gen.integers(4294967295, dtype=np.uint32, endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 4294967295, dtype=np.uint32, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u4_low_like, 4294967295, dtype=np.uint32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_high_open, dtype=np.uint32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_open, dtype=np.uint32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(0, I_u4_high_open, dtype=np.uint32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_high_closed, dtype=np.uint32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_closed, dtype=np.uint32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(0, I_u4_high_closed, dtype=np.uint32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] + +reveal_type(def_gen.integers(4294967296, dtype=np.uint)) # E: int +reveal_type(def_gen.integers(0, 4294967296, dtype=np.uint)) # E: int +reveal_type(def_gen.integers(4294967295, dtype=np.uint, endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 4294967295, dtype=np.uint, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u4_low_like, 4294967295, dtype=np.uint, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(def_gen.integers(I_u4_high_open, dtype=np.uint)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_open, dtype=np.uint)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(def_gen.integers(0, I_u4_high_open, dtype=np.uint)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(def_gen.integers(I_u4_high_closed, dtype=np.uint, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(def_gen.integers(I_u4_low, I_u4_high_closed, dtype=np.uint, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(def_gen.integers(0, I_u4_high_closed, dtype=np.uint, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] + +I_u8_low: np.ndarray[Any, np.dtype[np.uint64]] = np.array([0], dtype=np.uint64) +I_u8_low_like: List[int] = [0] +I_u8_high_open: np.ndarray[Any, np.dtype[np.uint64]] = np.array([18446744073709551615], dtype=np.uint64) +I_u8_high_closed: np.ndarray[Any, np.dtype[np.uint64]] = np.array([18446744073709551615], dtype=np.uint64) + +reveal_type(def_gen.integers(18446744073709551616, dtype="u8")) # E: int +reveal_type(def_gen.integers(0, 18446744073709551616, dtype="u8")) # E: int +reveal_type(def_gen.integers(18446744073709551615, dtype="u8", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 18446744073709551615, dtype="u8", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u8_low_like, 18446744073709551615, dtype="u8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_high_open, dtype="u8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_low, I_u8_high_open, dtype="u8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, I_u8_high_open, dtype="u8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_high_closed, dtype="u8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_low, I_u8_high_closed, dtype="u8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, I_u8_high_closed, dtype="u8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.integers(18446744073709551616, dtype="uint64")) # E: int +reveal_type(def_gen.integers(0, 18446744073709551616, dtype="uint64")) # E: int +reveal_type(def_gen.integers(18446744073709551615, dtype="uint64", endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 18446744073709551615, dtype="uint64", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u8_low_like, 18446744073709551615, dtype="uint64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_high_open, dtype="uint64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_low, I_u8_high_open, dtype="uint64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, I_u8_high_open, dtype="uint64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_high_closed, dtype="uint64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_low, I_u8_high_closed, dtype="uint64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, I_u8_high_closed, dtype="uint64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.integers(18446744073709551616, dtype=np.uint64)) # E: int +reveal_type(def_gen.integers(0, 18446744073709551616, dtype=np.uint64)) # E: int +reveal_type(def_gen.integers(18446744073709551615, dtype=np.uint64, endpoint=True)) # E: int +reveal_type(def_gen.integers(0, 18446744073709551615, dtype=np.uint64, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_u8_low_like, 18446744073709551615, dtype=np.uint64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_high_open, dtype=np.uint64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_low, I_u8_high_open, dtype=np.uint64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, I_u8_high_open, dtype=np.uint64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_high_closed, dtype=np.uint64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_u8_low, I_u8_high_closed, dtype=np.uint64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(0, I_u8_high_closed, dtype=np.uint64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] + +I_i1_low: np.ndarray[Any, np.dtype[np.int8]] = np.array([-128], dtype=np.int8) +I_i1_low_like: List[int] = [-128] +I_i1_high_open: np.ndarray[Any, np.dtype[np.int8]] = np.array([127], dtype=np.int8) +I_i1_high_closed: np.ndarray[Any, np.dtype[np.int8]] = np.array([127], dtype=np.int8) + +reveal_type(def_gen.integers(128, dtype="i1")) # E: int +reveal_type(def_gen.integers(-128, 128, dtype="i1")) # E: int +reveal_type(def_gen.integers(127, dtype="i1", endpoint=True)) # E: int +reveal_type(def_gen.integers(-128, 127, dtype="i1", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i1_low_like, 127, dtype="i1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_high_open, dtype="i1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_low, I_i1_high_open, dtype="i1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(-128, I_i1_high_open, dtype="i1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_high_closed, dtype="i1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_low, I_i1_high_closed, dtype="i1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(-128, I_i1_high_closed, dtype="i1", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] + +reveal_type(def_gen.integers(128, dtype="int8")) # E: int +reveal_type(def_gen.integers(-128, 128, dtype="int8")) # E: int +reveal_type(def_gen.integers(127, dtype="int8", endpoint=True)) # E: int +reveal_type(def_gen.integers(-128, 127, dtype="int8", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i1_low_like, 127, dtype="int8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_high_open, dtype="int8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_low, I_i1_high_open, dtype="int8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(-128, I_i1_high_open, dtype="int8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_high_closed, dtype="int8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_low, I_i1_high_closed, dtype="int8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(-128, I_i1_high_closed, dtype="int8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] + +reveal_type(def_gen.integers(128, dtype=np.int8)) # E: int +reveal_type(def_gen.integers(-128, 128, dtype=np.int8)) # E: int +reveal_type(def_gen.integers(127, dtype=np.int8, endpoint=True)) # E: int +reveal_type(def_gen.integers(-128, 127, dtype=np.int8, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i1_low_like, 127, dtype=np.int8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_high_open, dtype=np.int8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_low, I_i1_high_open, dtype=np.int8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(-128, I_i1_high_open, dtype=np.int8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_high_closed, dtype=np.int8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(I_i1_low, I_i1_high_closed, dtype=np.int8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(def_gen.integers(-128, I_i1_high_closed, dtype=np.int8, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] + +I_i2_low: np.ndarray[Any, np.dtype[np.int16]] = np.array([-32768], dtype=np.int16) +I_i2_low_like: List[int] = [-32768] +I_i2_high_open: np.ndarray[Any, np.dtype[np.int16]] = np.array([32767], dtype=np.int16) +I_i2_high_closed: np.ndarray[Any, np.dtype[np.int16]] = np.array([32767], dtype=np.int16) + +reveal_type(def_gen.integers(32768, dtype="i2")) # E: int +reveal_type(def_gen.integers(-32768, 32768, dtype="i2")) # E: int +reveal_type(def_gen.integers(32767, dtype="i2", endpoint=True)) # E: int +reveal_type(def_gen.integers(-32768, 32767, dtype="i2", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i2_low_like, 32767, dtype="i2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_high_open, dtype="i2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_low, I_i2_high_open, dtype="i2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(-32768, I_i2_high_open, dtype="i2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_high_closed, dtype="i2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_low, I_i2_high_closed, dtype="i2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(-32768, I_i2_high_closed, dtype="i2", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] + +reveal_type(def_gen.integers(32768, dtype="int16")) # E: int +reveal_type(def_gen.integers(-32768, 32768, dtype="int16")) # E: int +reveal_type(def_gen.integers(32767, dtype="int16", endpoint=True)) # E: int +reveal_type(def_gen.integers(-32768, 32767, dtype="int16", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i2_low_like, 32767, dtype="int16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_high_open, dtype="int16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_low, I_i2_high_open, dtype="int16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(-32768, I_i2_high_open, dtype="int16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_high_closed, dtype="int16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_low, I_i2_high_closed, dtype="int16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(-32768, I_i2_high_closed, dtype="int16", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] + +reveal_type(def_gen.integers(32768, dtype=np.int16)) # E: int +reveal_type(def_gen.integers(-32768, 32768, dtype=np.int16)) # E: int +reveal_type(def_gen.integers(32767, dtype=np.int16, endpoint=True)) # E: int +reveal_type(def_gen.integers(-32768, 32767, dtype=np.int16, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i2_low_like, 32767, dtype=np.int16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_high_open, dtype=np.int16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_low, I_i2_high_open, dtype=np.int16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(-32768, I_i2_high_open, dtype=np.int16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_high_closed, dtype=np.int16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(I_i2_low, I_i2_high_closed, dtype=np.int16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(def_gen.integers(-32768, I_i2_high_closed, dtype=np.int16, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] + +I_i4_low: np.ndarray[Any, np.dtype[np.int32]] = np.array([-2147483648], dtype=np.int32) +I_i4_low_like: List[int] = [-2147483648] +I_i4_high_open: np.ndarray[Any, np.dtype[np.int32]] = np.array([2147483647], dtype=np.int32) +I_i4_high_closed: np.ndarray[Any, np.dtype[np.int32]] = np.array([2147483647], dtype=np.int32) + +reveal_type(def_gen.integers(2147483648, dtype="i4")) # E: int +reveal_type(def_gen.integers(-2147483648, 2147483648, dtype="i4")) # E: int +reveal_type(def_gen.integers(2147483647, dtype="i4", endpoint=True)) # E: int +reveal_type(def_gen.integers(-2147483648, 2147483647, dtype="i4", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i4_low_like, 2147483647, dtype="i4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_high_open, dtype="i4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_low, I_i4_high_open, dtype="i4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(-2147483648, I_i4_high_open, dtype="i4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_high_closed, dtype="i4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_low, I_i4_high_closed, dtype="i4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(-2147483648, I_i4_high_closed, dtype="i4", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] + +reveal_type(def_gen.integers(2147483648, dtype="int32")) # E: int +reveal_type(def_gen.integers(-2147483648, 2147483648, dtype="int32")) # E: int +reveal_type(def_gen.integers(2147483647, dtype="int32", endpoint=True)) # E: int +reveal_type(def_gen.integers(-2147483648, 2147483647, dtype="int32", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i4_low_like, 2147483647, dtype="int32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_high_open, dtype="int32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_low, I_i4_high_open, dtype="int32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(-2147483648, I_i4_high_open, dtype="int32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_high_closed, dtype="int32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_low, I_i4_high_closed, dtype="int32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(-2147483648, I_i4_high_closed, dtype="int32", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] + +reveal_type(def_gen.integers(2147483648, dtype=np.int32)) # E: int +reveal_type(def_gen.integers(-2147483648, 2147483648, dtype=np.int32)) # E: int +reveal_type(def_gen.integers(2147483647, dtype=np.int32, endpoint=True)) # E: int +reveal_type(def_gen.integers(-2147483648, 2147483647, dtype=np.int32, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i4_low_like, 2147483647, dtype=np.int32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_high_open, dtype=np.int32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_low, I_i4_high_open, dtype=np.int32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(-2147483648, I_i4_high_open, dtype=np.int32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_high_closed, dtype=np.int32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(I_i4_low, I_i4_high_closed, dtype=np.int32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(def_gen.integers(-2147483648, I_i4_high_closed, dtype=np.int32, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] + +I_i8_low: np.ndarray[Any, np.dtype[np.int64]] = np.array([-9223372036854775808], dtype=np.int64) +I_i8_low_like: List[int] = [-9223372036854775808] +I_i8_high_open: np.ndarray[Any, np.dtype[np.int64]] = np.array([9223372036854775807], dtype=np.int64) +I_i8_high_closed: np.ndarray[Any, np.dtype[np.int64]] = np.array([9223372036854775807], dtype=np.int64) + +reveal_type(def_gen.integers(9223372036854775808, dtype="i8")) # E: int +reveal_type(def_gen.integers(-9223372036854775808, 9223372036854775808, dtype="i8")) # E: int +reveal_type(def_gen.integers(9223372036854775807, dtype="i8", endpoint=True)) # E: int +reveal_type(def_gen.integers(-9223372036854775808, 9223372036854775807, dtype="i8", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i8_low_like, 9223372036854775807, dtype="i8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_high_open, dtype="i8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_low, I_i8_high_open, dtype="i8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(-9223372036854775808, I_i8_high_open, dtype="i8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_high_closed, dtype="i8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_low, I_i8_high_closed, dtype="i8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(-9223372036854775808, I_i8_high_closed, dtype="i8", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.integers(9223372036854775808, dtype="int64")) # E: int +reveal_type(def_gen.integers(-9223372036854775808, 9223372036854775808, dtype="int64")) # E: int +reveal_type(def_gen.integers(9223372036854775807, dtype="int64", endpoint=True)) # E: int +reveal_type(def_gen.integers(-9223372036854775808, 9223372036854775807, dtype="int64", endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i8_low_like, 9223372036854775807, dtype="int64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_high_open, dtype="int64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_low, I_i8_high_open, dtype="int64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(-9223372036854775808, I_i8_high_open, dtype="int64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_high_closed, dtype="int64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_low, I_i8_high_closed, dtype="int64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(-9223372036854775808, I_i8_high_closed, dtype="int64", endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.integers(9223372036854775808, dtype=np.int64)) # E: int +reveal_type(def_gen.integers(-9223372036854775808, 9223372036854775808, dtype=np.int64)) # E: int +reveal_type(def_gen.integers(9223372036854775807, dtype=np.int64, endpoint=True)) # E: int +reveal_type(def_gen.integers(-9223372036854775808, 9223372036854775807, dtype=np.int64, endpoint=True)) # E: int +reveal_type(def_gen.integers(I_i8_low_like, 9223372036854775807, dtype=np.int64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_high_open, dtype=np.int64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_low, I_i8_high_open, dtype=np.int64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(-9223372036854775808, I_i8_high_open, dtype=np.int64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_high_closed, dtype=np.int64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(I_i8_low, I_i8_high_closed, dtype=np.int64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.integers(-9223372036854775808, I_i8_high_closed, dtype=np.int64, endpoint=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + + +reveal_type(def_gen.bit_generator) # E: BitGenerator + +reveal_type(def_gen.bytes(2)) # E: bytes + +reveal_type(def_gen.choice(5)) # E: int +reveal_type(def_gen.choice(5, 3)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.choice(5, 3, replace=True)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.choice(5, 3, p=[1 / 5] * 5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.choice(5, 3, p=[1 / 5] * 5, replace=False)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"])) # E: Any +reveal_type(def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, p=[1 / 4] * 4)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=True)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=False, p=np.array([1 / 8, 1 / 8, 1 / 2, 1 / 4]))) # E: numpy.ndarray[Any, Any] + +reveal_type(def_gen.dirichlet([0.5, 0.5])) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.dirichlet(np.array([0.5, 0.5]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.dirichlet(np.array([0.5, 0.5]), size=3)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.multinomial(20, [1 / 6.0] * 6)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multinomial(20, np.array([0.5, 0.5]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multinomial(20, [1 / 6.0] * 6, size=2)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multinomial([[10], [20]], [1 / 6.0] * 6, size=(2, 2))) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multinomial(np.array([[10], [20]]), np.array([0.5, 0.5]), size=(2, 2))) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.multivariate_hypergeometric([3, 5, 7], 2)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2, size=4)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2, size=(4, 7))) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multivariate_hypergeometric([3, 5, 7], 2, method="count")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.multivariate_hypergeometric(np.array([3, 5, 7]), 2, method="marginals")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(def_gen.multivariate_normal([0.0], [[1.0]])) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.multivariate_normal([0.0], np.array([[1.0]]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.multivariate_normal(np.array([0.0]), [[1.0]])) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(def_gen.multivariate_normal([0.0], np.array([[1.0]]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(def_gen.permutation(10)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(def_gen.permutation([1, 2, 3, 4])) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permutation(np.array([1, 2, 3, 4]))) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permutation(D_2D, axis=1)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D_like)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D, axis=1)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D, out=D_2D)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D_like, out=D_2D)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D_like, out=D_2D)) # E: numpy.ndarray[Any, Any] +reveal_type(def_gen.permuted(D_2D, axis=1, out=D_2D)) # E: numpy.ndarray[Any, Any] + +reveal_type(def_gen.shuffle(np.arange(10))) # E: None +reveal_type(def_gen.shuffle([1, 2, 3, 4, 5])) # E: None +reveal_type(def_gen.shuffle(D_2D, axis=1)) # E: None + +reveal_type(np.random.Generator(pcg64)) # E: Generator +reveal_type(def_gen.__str__()) # E: str +reveal_type(def_gen.__repr__()) # E: str +def_gen_state = def_gen.__getstate__() +reveal_type(def_gen_state) # E: builtins.dict[builtins.str, Any] +reveal_type(def_gen.__setstate__(def_gen_state)) # E: None + +# RandomState +random_st: np.random.RandomState = np.random.RandomState() + +reveal_type(random_st.standard_normal()) # E: float +reveal_type(random_st.standard_normal(size=None)) # E: float +reveal_type(random_st.standard_normal(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(random_st.random()) # E: float +reveal_type(random_st.random(size=None)) # E: float +reveal_type(random_st.random(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(random_st.standard_cauchy()) # E: float +reveal_type(random_st.standard_cauchy(size=None)) # E: float +reveal_type(random_st.standard_cauchy(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.standard_exponential()) # E: float +reveal_type(random_st.standard_exponential(size=None)) # E: float +reveal_type(random_st.standard_exponential(size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(random_st.zipf(1.5)) # E: int +reveal_type(random_st.zipf(1.5, size=None)) # E: int +reveal_type(random_st.zipf(1.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.zipf(D_arr_1p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.zipf(D_arr_1p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.zipf(D_arr_like_1p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.zipf(D_arr_like_1p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.weibull(0.5)) # E: float +reveal_type(random_st.weibull(0.5, size=None)) # E: float +reveal_type(random_st.weibull(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.weibull(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.weibull(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.weibull(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.weibull(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.standard_t(0.5)) # E: float +reveal_type(random_st.standard_t(0.5, size=None)) # E: float +reveal_type(random_st.standard_t(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.standard_t(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.standard_t(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.standard_t(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.standard_t(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.poisson(0.5)) # E: int +reveal_type(random_st.poisson(0.5, size=None)) # E: int +reveal_type(random_st.poisson(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.poisson(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.poisson(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.poisson(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.poisson(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.power(0.5)) # E: float +reveal_type(random_st.power(0.5, size=None)) # E: float +reveal_type(random_st.power(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.power(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.power(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.power(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.power(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.pareto(0.5)) # E: float +reveal_type(random_st.pareto(0.5, size=None)) # E: float +reveal_type(random_st.pareto(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.pareto(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.pareto(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.pareto(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.pareto(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.chisquare(0.5)) # E: float +reveal_type(random_st.chisquare(0.5, size=None)) # E: float +reveal_type(random_st.chisquare(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.chisquare(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.chisquare(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.chisquare(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.chisquare(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.exponential(0.5)) # E: float +reveal_type(random_st.exponential(0.5, size=None)) # E: float +reveal_type(random_st.exponential(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.exponential(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.exponential(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.exponential(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.exponential(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.geometric(0.5)) # E: int +reveal_type(random_st.geometric(0.5, size=None)) # E: int +reveal_type(random_st.geometric(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.geometric(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.geometric(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.geometric(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.geometric(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.logseries(0.5)) # E: int +reveal_type(random_st.logseries(0.5, size=None)) # E: int +reveal_type(random_st.logseries(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.logseries(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.logseries(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.logseries(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.logseries(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.rayleigh(0.5)) # E: float +reveal_type(random_st.rayleigh(0.5, size=None)) # E: float +reveal_type(random_st.rayleigh(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.rayleigh(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.rayleigh(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.rayleigh(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.rayleigh(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.standard_gamma(0.5)) # E: float +reveal_type(random_st.standard_gamma(0.5, size=None)) # E: float +reveal_type(random_st.standard_gamma(0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(random_st.standard_gamma(D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(random_st.standard_gamma(D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(random_st.standard_gamma(D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(random_st.standard_gamma(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] +reveal_type(random_st.standard_gamma(D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]]] + +reveal_type(random_st.vonmises(0.5, 0.5)) # E: float +reveal_type(random_st.vonmises(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.vonmises(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.vonmises(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.wald(0.5, 0.5)) # E: float +reveal_type(random_st.wald(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.wald(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.wald(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.uniform(0.5, 0.5)) # E: float +reveal_type(random_st.uniform(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.uniform(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.uniform(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.beta(0.5, 0.5)) # E: float +reveal_type(random_st.beta(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.beta(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.beta(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.f(0.5, 0.5)) # E: float +reveal_type(random_st.f(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.f(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.f(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.gamma(0.5, 0.5)) # E: float +reveal_type(random_st.gamma(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.gamma(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gamma(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.gumbel(0.5, 0.5)) # E: float +reveal_type(random_st.gumbel(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.gumbel(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.gumbel(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.laplace(0.5, 0.5)) # E: float +reveal_type(random_st.laplace(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.laplace(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.laplace(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.logistic(0.5, 0.5)) # E: float +reveal_type(random_st.logistic(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.logistic(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.logistic(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.lognormal(0.5, 0.5)) # E: float +reveal_type(random_st.lognormal(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.lognormal(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.lognormal(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.noncentral_chisquare(0.5, 0.5)) # E: float +reveal_type(random_st.noncentral_chisquare(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.noncentral_chisquare(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_chisquare(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.normal(0.5, 0.5)) # E: float +reveal_type(random_st.normal(0.5, 0.5, size=None)) # E: float +reveal_type(random_st.normal(0.5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(0.5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_0p5, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(0.5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_like_0p5, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(0.5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_0p5, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_like_0p5, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_0p5, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.normal(D_arr_like_0p5, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.triangular(0.1, 0.5, 0.9)) # E: float +reveal_type(random_st.triangular(0.1, 0.5, 0.9, size=None)) # E: float +reveal_type(random_st.triangular(0.1, 0.5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_0p1, 0.5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(0.1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_0p1, 0.5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(0.1, D_arr_0p5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_like_0p1, 0.5, D_arr_0p9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(0.5, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_0p1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_like_0p1, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.triangular(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.noncentral_f(0.1, 0.5, 0.9)) # E: float +reveal_type(random_st.noncentral_f(0.1, 0.5, 0.9, size=None)) # E: float +reveal_type(random_st.noncentral_f(0.1, 0.5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_0p1, 0.5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(0.1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_0p1, 0.5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(0.1, D_arr_0p5, 0.9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_like_0p1, 0.5, D_arr_0p9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(0.5, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_0p1, D_arr_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, 0.9)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_0p1, D_arr_0p5, D_arr_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.noncentral_f(D_arr_like_0p1, D_arr_like_0p5, D_arr_like_0p9, size=1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.binomial(10, 0.5)) # E: int +reveal_type(random_st.binomial(10, 0.5, size=None)) # E: int +reveal_type(random_st.binomial(10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_like_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_like_10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.binomial(I_arr_like_10, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.negative_binomial(10, 0.5)) # E: int +reveal_type(random_st.negative_binomial(10, 0.5, size=None)) # E: int +reveal_type(random_st.negative_binomial(10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_10, 0.5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_like_10, 0.5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_10, D_arr_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_like_10, D_arr_like_0p5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_10, D_arr_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.negative_binomial(I_arr_like_10, D_arr_like_0p5, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.hypergeometric(20, 20, 10)) # E: int +reveal_type(random_st.hypergeometric(20, 20, 10, size=None)) # E: int +reveal_type(random_st.hypergeometric(20, 20, 10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_20, 20, 10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(20, I_arr_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_20, 20, I_arr_like_10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(20, I_arr_20, 10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_like_20, 20, I_arr_10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(20, I_arr_like_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_20, I_arr_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_like_20, I_arr_like_20, 10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_20, I_arr_20, I_arr_10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.hypergeometric(I_arr_like_20, I_arr_like_20, I_arr_like_10, size=1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.randint(0, 100)) # E: int +reveal_type(random_st.randint(100)) # E: int +reveal_type(random_st.randint([100])) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.randint(0, [100])) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.randint(2, dtype=bool)) # E: builtins.bool +reveal_type(random_st.randint(0, 2, dtype=bool)) # E: builtins.bool +reveal_type(random_st.randint(I_bool_high_open, dtype=bool)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(random_st.randint(I_bool_low, I_bool_high_open, dtype=bool)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(random_st.randint(0, I_bool_high_open, dtype=bool)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] + +reveal_type(random_st.randint(2, dtype=np.bool_)) # E: builtins.bool +reveal_type(random_st.randint(0, 2, dtype=np.bool_)) # E: builtins.bool +reveal_type(random_st.randint(I_bool_high_open, dtype=np.bool_)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(random_st.randint(I_bool_low, I_bool_high_open, dtype=np.bool_)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] +reveal_type(random_st.randint(0, I_bool_high_open, dtype=np.bool_)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_] + +reveal_type(random_st.randint(256, dtype="u1")) # E: int +reveal_type(random_st.randint(0, 256, dtype="u1")) # E: int +reveal_type(random_st.randint(I_u1_high_open, dtype="u1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(I_u1_low, I_u1_high_open, dtype="u1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(0, I_u1_high_open, dtype="u1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] + +reveal_type(random_st.randint(256, dtype="uint8")) # E: int +reveal_type(random_st.randint(0, 256, dtype="uint8")) # E: int +reveal_type(random_st.randint(I_u1_high_open, dtype="uint8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(I_u1_low, I_u1_high_open, dtype="uint8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(0, I_u1_high_open, dtype="uint8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] + +reveal_type(random_st.randint(256, dtype=np.uint8)) # E: int +reveal_type(random_st.randint(0, 256, dtype=np.uint8)) # E: int +reveal_type(random_st.randint(I_u1_high_open, dtype=np.uint8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(I_u1_low, I_u1_high_open, dtype=np.uint8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(0, I_u1_high_open, dtype=np.uint8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._8Bit]]] + +reveal_type(random_st.randint(65536, dtype="u2")) # E: int +reveal_type(random_st.randint(0, 65536, dtype="u2")) # E: int +reveal_type(random_st.randint(I_u2_high_open, dtype="u2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(I_u2_low, I_u2_high_open, dtype="u2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(0, I_u2_high_open, dtype="u2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] + +reveal_type(random_st.randint(65536, dtype="uint16")) # E: int +reveal_type(random_st.randint(0, 65536, dtype="uint16")) # E: int +reveal_type(random_st.randint(I_u2_high_open, dtype="uint16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(I_u2_low, I_u2_high_open, dtype="uint16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(0, I_u2_high_open, dtype="uint16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] + +reveal_type(random_st.randint(65536, dtype=np.uint16)) # E: int +reveal_type(random_st.randint(0, 65536, dtype=np.uint16)) # E: int +reveal_type(random_st.randint(I_u2_high_open, dtype=np.uint16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(I_u2_low, I_u2_high_open, dtype=np.uint16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(0, I_u2_high_open, dtype=np.uint16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._16Bit]]] + +reveal_type(random_st.randint(4294967296, dtype="u4")) # E: int +reveal_type(random_st.randint(0, 4294967296, dtype="u4")) # E: int +reveal_type(random_st.randint(I_u4_high_open, dtype="u4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(I_u4_low, I_u4_high_open, dtype="u4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(0, I_u4_high_open, dtype="u4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] + +reveal_type(random_st.randint(4294967296, dtype="uint32")) # E: int +reveal_type(random_st.randint(0, 4294967296, dtype="uint32")) # E: int +reveal_type(random_st.randint(I_u4_high_open, dtype="uint32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(I_u4_low, I_u4_high_open, dtype="uint32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(0, I_u4_high_open, dtype="uint32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] + +reveal_type(random_st.randint(4294967296, dtype=np.uint32)) # E: int +reveal_type(random_st.randint(0, 4294967296, dtype=np.uint32)) # E: int +reveal_type(random_st.randint(I_u4_high_open, dtype=np.uint32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(I_u4_low, I_u4_high_open, dtype=np.uint32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(0, I_u4_high_open, dtype=np.uint32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]] + +reveal_type(random_st.randint(4294967296, dtype=np.uint)) # E: int +reveal_type(random_st.randint(0, 4294967296, dtype=np.uint)) # E: int +reveal_type(random_st.randint(I_u4_high_open, dtype=np.uint)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(random_st.randint(I_u4_low, I_u4_high_open, dtype=np.uint)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] +reveal_type(random_st.randint(0, I_u4_high_open, dtype=np.uint)) # E: numpy.ndarray[Any, numpy.dtype[{uint}]] + +reveal_type(random_st.randint(18446744073709551616, dtype="u8")) # E: int +reveal_type(random_st.randint(0, 18446744073709551616, dtype="u8")) # E: int +reveal_type(random_st.randint(I_u8_high_open, dtype="u8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(I_u8_low, I_u8_high_open, dtype="u8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(0, I_u8_high_open, dtype="u8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] + +reveal_type(random_st.randint(18446744073709551616, dtype="uint64")) # E: int +reveal_type(random_st.randint(0, 18446744073709551616, dtype="uint64")) # E: int +reveal_type(random_st.randint(I_u8_high_open, dtype="uint64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(I_u8_low, I_u8_high_open, dtype="uint64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(0, I_u8_high_open, dtype="uint64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] + +reveal_type(random_st.randint(18446744073709551616, dtype=np.uint64)) # E: int +reveal_type(random_st.randint(0, 18446744073709551616, dtype=np.uint64)) # E: int +reveal_type(random_st.randint(I_u8_high_open, dtype=np.uint64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(I_u8_low, I_u8_high_open, dtype=np.uint64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(0, I_u8_high_open, dtype=np.uint64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._64Bit]]] + +reveal_type(random_st.randint(128, dtype="i1")) # E: int +reveal_type(random_st.randint(-128, 128, dtype="i1")) # E: int +reveal_type(random_st.randint(I_i1_high_open, dtype="i1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(I_i1_low, I_i1_high_open, dtype="i1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(-128, I_i1_high_open, dtype="i1")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] + +reveal_type(random_st.randint(128, dtype="int8")) # E: int +reveal_type(random_st.randint(-128, 128, dtype="int8")) # E: int +reveal_type(random_st.randint(I_i1_high_open, dtype="int8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(I_i1_low, I_i1_high_open, dtype="int8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(-128, I_i1_high_open, dtype="int8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] + +reveal_type(random_st.randint(128, dtype=np.int8)) # E: int +reveal_type(random_st.randint(-128, 128, dtype=np.int8)) # E: int +reveal_type(random_st.randint(I_i1_high_open, dtype=np.int8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(I_i1_low, I_i1_high_open, dtype=np.int8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] +reveal_type(random_st.randint(-128, I_i1_high_open, dtype=np.int8)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._8Bit]]] + +reveal_type(random_st.randint(32768, dtype="i2")) # E: int +reveal_type(random_st.randint(-32768, 32768, dtype="i2")) # E: int +reveal_type(random_st.randint(I_i2_high_open, dtype="i2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(I_i2_low, I_i2_high_open, dtype="i2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(-32768, I_i2_high_open, dtype="i2")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(32768, dtype="int16")) # E: int +reveal_type(random_st.randint(-32768, 32768, dtype="int16")) # E: int +reveal_type(random_st.randint(I_i2_high_open, dtype="int16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(I_i2_low, I_i2_high_open, dtype="int16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(-32768, I_i2_high_open, dtype="int16")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(32768, dtype=np.int16)) # E: int +reveal_type(random_st.randint(-32768, 32768, dtype=np.int16)) # E: int +reveal_type(random_st.randint(I_i2_high_open, dtype=np.int16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(I_i2_low, I_i2_high_open, dtype=np.int16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] +reveal_type(random_st.randint(-32768, I_i2_high_open, dtype=np.int16)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._16Bit]]] + +reveal_type(random_st.randint(2147483648, dtype="i4")) # E: int +reveal_type(random_st.randint(-2147483648, 2147483648, dtype="i4")) # E: int +reveal_type(random_st.randint(I_i4_high_open, dtype="i4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(I_i4_low, I_i4_high_open, dtype="i4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(-2147483648, I_i4_high_open, dtype="i4")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] + +reveal_type(random_st.randint(2147483648, dtype="int32")) # E: int +reveal_type(random_st.randint(-2147483648, 2147483648, dtype="int32")) # E: int +reveal_type(random_st.randint(I_i4_high_open, dtype="int32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(I_i4_low, I_i4_high_open, dtype="int32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(-2147483648, I_i4_high_open, dtype="int32")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] + +reveal_type(random_st.randint(2147483648, dtype=np.int32)) # E: int +reveal_type(random_st.randint(-2147483648, 2147483648, dtype=np.int32)) # E: int +reveal_type(random_st.randint(I_i4_high_open, dtype=np.int32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(I_i4_low, I_i4_high_open, dtype=np.int32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] +reveal_type(random_st.randint(-2147483648, I_i4_high_open, dtype=np.int32)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._32Bit]]] + +reveal_type(random_st.randint(2147483648, dtype=np.int_)) # E: int +reveal_type(random_st.randint(-2147483648, 2147483648, dtype=np.int_)) # E: int +reveal_type(random_st.randint(I_i4_high_open, dtype=np.int_)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.randint(I_i4_low, I_i4_high_open, dtype=np.int_)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.randint(-2147483648, I_i4_high_open, dtype=np.int_)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.randint(9223372036854775808, dtype="i8")) # E: int +reveal_type(random_st.randint(-9223372036854775808, 9223372036854775808, dtype="i8")) # E: int +reveal_type(random_st.randint(I_i8_high_open, dtype="i8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(I_i8_low, I_i8_high_open, dtype="i8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(-9223372036854775808, I_i8_high_open, dtype="i8")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(random_st.randint(9223372036854775808, dtype="int64")) # E: int +reveal_type(random_st.randint(-9223372036854775808, 9223372036854775808, dtype="int64")) # E: int +reveal_type(random_st.randint(I_i8_high_open, dtype="int64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(I_i8_low, I_i8_high_open, dtype="int64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(-9223372036854775808, I_i8_high_open, dtype="int64")) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(random_st.randint(9223372036854775808, dtype=np.int64)) # E: int +reveal_type(random_st.randint(-9223372036854775808, 9223372036854775808, dtype=np.int64)) # E: int +reveal_type(random_st.randint(I_i8_high_open, dtype=np.int64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(I_i8_low, I_i8_high_open, dtype=np.int64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] +reveal_type(random_st.randint(-9223372036854775808, I_i8_high_open, dtype=np.int64)) # E: numpy.ndarray[Any, numpy.dtype[numpy.signedinteger[numpy.typing._64Bit]]] + +reveal_type(random_st._bit_generator) # E: BitGenerator + +reveal_type(random_st.bytes(2)) # E: bytes + +reveal_type(random_st.choice(5)) # E: int +reveal_type(random_st.choice(5, 3)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.choice(5, 3, replace=True)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.choice(5, 3, p=[1 / 5] * 5)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.choice(5, 3, p=[1 / 5] * 5, replace=False)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.choice(["pooh", "rabbit", "piglet", "Christopher"])) # E: Any +reveal_type(random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3)) # E: numpy.ndarray[Any, Any] +reveal_type(random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, p=[1 / 4] * 4)) # E: numpy.ndarray[Any, Any] +reveal_type(random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=True)) # E: numpy.ndarray[Any, Any] +reveal_type(random_st.choice(["pooh", "rabbit", "piglet", "Christopher"], 3, replace=False, p=np.array([1 / 8, 1 / 8, 1 / 2, 1 / 4]))) # E: numpy.ndarray[Any, Any] + +reveal_type(random_st.dirichlet([0.5, 0.5])) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.dirichlet(np.array([0.5, 0.5]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.dirichlet(np.array([0.5, 0.5]), size=3)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.multinomial(20, [1 / 6.0] * 6)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.multinomial(20, np.array([0.5, 0.5]))) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.multinomial(20, [1 / 6.0] * 6, size=2)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] + +reveal_type(random_st.multivariate_normal([0.0], [[1.0]])) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.multivariate_normal([0.0], np.array([[1.0]]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.multivariate_normal(np.array([0.0]), [[1.0]])) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.multivariate_normal([0.0], np.array([[1.0]]))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.permutation(10)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.permutation([1, 2, 3, 4])) # E: numpy.ndarray[Any, Any] +reveal_type(random_st.permutation(np.array([1, 2, 3, 4]))) # E: numpy.ndarray[Any, Any] +reveal_type(random_st.permutation(D_2D)) # E: numpy.ndarray[Any, Any] + +reveal_type(random_st.shuffle(np.arange(10))) # E: None +reveal_type(random_st.shuffle([1, 2, 3, 4, 5])) # E: None +reveal_type(random_st.shuffle(D_2D)) # E: None + +reveal_type(np.random.RandomState(pcg64)) # E: RandomState +reveal_type(np.random.RandomState(0)) # E: RandomState +reveal_type(np.random.RandomState([0, 1, 2])) # E: RandomState +reveal_type(random_st.__str__()) # E: str +reveal_type(random_st.__repr__()) # E: str +random_st_state = random_st.__getstate__() +reveal_type(random_st_state) # E: builtins.dict[builtins.str, Any] +reveal_type(random_st.__setstate__(random_st_state)) # E: None +reveal_type(random_st.seed()) # E: None +reveal_type(random_st.seed(1)) # E: None +reveal_type(random_st.seed([0, 1])) # E: None +random_st_get_state = random_st.get_state() +reveal_type(random_st_state) # E: builtins.dict[builtins.str, Any] +random_st_get_state_legacy = random_st.get_state(legacy=True) +reveal_type(random_st_get_state_legacy) # E: Union[builtins.dict[builtins.str, Any], Tuple[builtins.str, numpy.ndarray[Any, numpy.dtype[numpy.unsignedinteger[numpy.typing._32Bit]]], builtins.int, builtins.int, builtins.float]] +reveal_type(random_st.set_state(random_st_get_state)) # E: None + +reveal_type(random_st.rand()) # E: float +reveal_type(random_st.rand(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.rand(1, 2)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.randn()) # E: float +reveal_type(random_st.randn(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.randn(1, 2)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.random_sample()) # E: float +reveal_type(random_st.random_sample(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] +reveal_type(random_st.random_sample(size=(1, 2))) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[numpy.typing._64Bit]] + +reveal_type(random_st.tomaxint()) # E: int +reveal_type(random_st.tomaxint(1)) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] +reveal_type(random_st.tomaxint((1,))) # E: numpy.ndarray[Any, numpy.dtype[{int_}]] diff --git a/numpy/typing/tests/data/reveal/scalars.py b/numpy/typing/tests/data/reveal/scalars.py index e887e302d4f4..d98388422e07 100644 --- a/numpy/typing/tests/data/reveal/scalars.py +++ b/numpy/typing/tests/data/reveal/scalars.py @@ -1,28 +1,116 @@ import numpy as np -x = np.complex64(3 + 2j) +b: np.bool_ +u8: np.uint64 +i8: np.int64 +f8: np.float64 +c8: np.complex64 +c16: np.complex128 +U: np.str_ +S: np.bytes_ -reveal_type(x.real) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(x.imag) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(c8.real) # E: {float32} +reveal_type(c8.imag) # E: {float32} -reveal_type(x.real.real) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(x.real.imag) # E: numpy.floating[numpy.typing._32Bit] +reveal_type(c8.real.real) # E: {float32} +reveal_type(c8.real.imag) # E: {float32} -reveal_type(x.itemsize) # E: int -reveal_type(x.shape) # E: Tuple[] -reveal_type(x.strides) # E: Tuple[] +reveal_type(c8.itemsize) # E: int +reveal_type(c8.shape) # E: Tuple[] +reveal_type(c8.strides) # E: Tuple[] -reveal_type(x.ndim) # E: Literal[0] -reveal_type(x.size) # E: Literal[1] +reveal_type(c8.ndim) # E: Literal[0] +reveal_type(c8.size) # E: Literal[1] -reveal_type(x.squeeze()) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(x.byteswap()) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] -reveal_type(x.transpose()) # E: numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit] +reveal_type(c8.squeeze()) # E: {complex64} +reveal_type(c8.byteswap()) # E: {complex64} +reveal_type(c8.transpose()) # E: {complex64} -reveal_type(x.dtype) # E: numpy.dtype[numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit]] +reveal_type(c8.dtype) # E: numpy.dtype[{complex64}] -reveal_type(np.complex64().real) # E: numpy.floating[numpy.typing._32Bit] -reveal_type(np.complex128().imag) # E: numpy.floating[numpy.typing._64Bit] +reveal_type(c8.real) # E: {float32} +reveal_type(c16.imag) # E: {float64} reveal_type(np.unicode_('foo')) # E: numpy.str_ reveal_type(np.str0('foo')) # E: numpy.str_ + +# Aliases +reveal_type(np.unicode_()) # E: numpy.str_ +reveal_type(np.str0()) # E: numpy.str_ +reveal_type(np.bool8()) # E: numpy.bool_ +reveal_type(np.bytes0()) # E: numpy.bytes_ +reveal_type(np.string_()) # E: numpy.bytes_ +reveal_type(np.object0()) # E: numpy.object_ +reveal_type(np.void0(0)) # E: numpy.void + +reveal_type(np.byte()) # E: {byte} +reveal_type(np.short()) # E: {short} +reveal_type(np.intc()) # E: {intc} +reveal_type(np.intp()) # E: {intp} +reveal_type(np.int0()) # E: {intp} +reveal_type(np.int_()) # E: {int_} +reveal_type(np.longlong()) # E: {longlong} + +reveal_type(np.ubyte()) # E: {ubyte} +reveal_type(np.ushort()) # E: {ushort} +reveal_type(np.uintc()) # E: {uintc} +reveal_type(np.uintp()) # E: {uintp} +reveal_type(np.uint0()) # E: {uintp} +reveal_type(np.uint()) # E: {uint} +reveal_type(np.ulonglong()) # E: {ulonglong} + +reveal_type(np.half()) # E: {half} +reveal_type(np.single()) # E: {single} +reveal_type(np.double()) # E: {double} +reveal_type(np.float_()) # E: {double} +reveal_type(np.longdouble()) # E: {longdouble} +reveal_type(np.longfloat()) # E: {longdouble} + +reveal_type(np.csingle()) # E: {csingle} +reveal_type(np.singlecomplex()) # E: {csingle} +reveal_type(np.cdouble()) # E: {cdouble} +reveal_type(np.complex_()) # E: {cdouble} +reveal_type(np.cfloat()) # E: {cdouble} +reveal_type(np.clongdouble()) # E: {clongdouble} +reveal_type(np.clongfloat()) # E: {clongdouble} +reveal_type(np.longcomplex()) # E: {clongdouble} + +reveal_type(b.item()) # E: bool +reveal_type(i8.item()) # E: int +reveal_type(u8.item()) # E: int +reveal_type(f8.item()) # E: float +reveal_type(c16.item()) # E: complex +reveal_type(U.item()) # E: str +reveal_type(S.item()) # E: bytes + +reveal_type(b.tolist()) # E: bool +reveal_type(i8.tolist()) # E: int +reveal_type(u8.tolist()) # E: int +reveal_type(f8.tolist()) # E: float +reveal_type(c16.tolist()) # E: complex +reveal_type(U.tolist()) # E: str +reveal_type(S.tolist()) # E: bytes + +reveal_type(b.ravel()) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(i8.ravel()) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] +reveal_type(u8.ravel()) # E: numpy.ndarray[Any, numpy.dtype[{uint64}]] +reveal_type(f8.ravel()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(c16.ravel()) # E: numpy.ndarray[Any, numpy.dtype[{complex128}]] +reveal_type(U.ravel()) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(S.ravel()) # E: numpy.ndarray[Any, numpy.dtype[numpy.bytes_]] + +reveal_type(b.flatten()) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(i8.flatten()) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] +reveal_type(u8.flatten()) # E: numpy.ndarray[Any, numpy.dtype[{uint64}]] +reveal_type(f8.flatten()) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(c16.flatten()) # E: numpy.ndarray[Any, numpy.dtype[{complex128}]] +reveal_type(U.flatten()) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(S.flatten()) # E: numpy.ndarray[Any, numpy.dtype[numpy.bytes_]] + +reveal_type(b.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(i8.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[{int64}]] +reveal_type(u8.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[{uint64}]] +reveal_type(f8.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[{float64}]] +reveal_type(c16.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[{complex128}]] +reveal_type(U.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] +reveal_type(S.reshape(1)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bytes_]] diff --git a/numpy/typing/tests/data/reveal/ufunclike.py b/numpy/typing/tests/data/reveal/ufunclike.py new file mode 100644 index 000000000000..8b3aea7ceeb9 --- /dev/null +++ b/numpy/typing/tests/data/reveal/ufunclike.py @@ -0,0 +1,29 @@ +from typing import List, Any +import numpy as np + +AR_LIKE_b: List[bool] +AR_LIKE_u: List[np.uint32] +AR_LIKE_i: List[int] +AR_LIKE_f: List[float] +AR_LIKE_O: List[np.object_] + +AR_U: np.ndarray[Any, np.dtype[np.str_]] + +reveal_type(np.fix(AR_LIKE_b)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(np.fix(AR_LIKE_u)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(np.fix(AR_LIKE_i)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(np.fix(AR_LIKE_f)) # E: numpy.ndarray[Any, numpy.dtype[numpy.floating[Any]]] +reveal_type(np.fix(AR_LIKE_O)) # E: Any +reveal_type(np.fix(AR_LIKE_f, out=AR_U)) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] + +reveal_type(np.isposinf(AR_LIKE_b)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isposinf(AR_LIKE_u)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isposinf(AR_LIKE_i)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isposinf(AR_LIKE_f)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isposinf(AR_LIKE_f, out=AR_U)) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] + +reveal_type(np.isneginf(AR_LIKE_b)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isneginf(AR_LIKE_u)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isneginf(AR_LIKE_i)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isneginf(AR_LIKE_f)) # E: numpy.ndarray[Any, numpy.dtype[numpy.bool_]] +reveal_type(np.isneginf(AR_LIKE_f, out=AR_U)) # E: numpy.ndarray[Any, numpy.dtype[numpy.str_]] diff --git a/numpy/typing/tests/data/reveal/ufuncs.py b/numpy/typing/tests/data/reveal/ufuncs.py new file mode 100644 index 000000000000..ade45577c9a3 --- /dev/null +++ b/numpy/typing/tests/data/reveal/ufuncs.py @@ -0,0 +1,68 @@ +import numpy as np +import numpy.typing as npt + +f8: np.float64 +AR_f8: npt.NDArray[np.float64] +AR_i8: npt.NDArray[np.int64] + +reveal_type(np.absolute.__doc__) # E: str +reveal_type(np.absolute.types) # E: builtins.list[builtins.str] + +reveal_type(np.absolute.__name__) # E: Literal['absolute'] +reveal_type(np.absolute.ntypes) # E: Literal[20] +reveal_type(np.absolute.identity) # E: None +reveal_type(np.absolute.nin) # E: Literal[1] +reveal_type(np.absolute.nin) # E: Literal[1] +reveal_type(np.absolute.nout) # E: Literal[1] +reveal_type(np.absolute.nargs) # E: Literal[2] +reveal_type(np.absolute.signature) # E: None +reveal_type(np.absolute(f8)) # E: Any +reveal_type(np.absolute(AR_f8)) # E: numpy.ndarray +reveal_type(np.absolute.at(AR_f8, AR_i8)) # E: None + +reveal_type(np.add.__name__) # E: Literal['add'] +reveal_type(np.add.ntypes) # E: Literal[22] +reveal_type(np.add.identity) # E: Literal[0] +reveal_type(np.add.nin) # E: Literal[2] +reveal_type(np.add.nout) # E: Literal[1] +reveal_type(np.add.nargs) # E: Literal[3] +reveal_type(np.add.signature) # E: None +reveal_type(np.add(f8, f8)) # E: Any +reveal_type(np.add(AR_f8, f8)) # E: numpy.ndarray +reveal_type(np.add.at(AR_f8, AR_i8, f8)) # E: None +reveal_type(np.add.reduce(AR_f8, axis=0)) # E: Any +reveal_type(np.add.accumulate(AR_f8)) # E: numpy.ndarray +reveal_type(np.add.reduceat(AR_f8, AR_i8)) # E: numpy.ndarray +reveal_type(np.add.outer(f8, f8)) # E: Any +reveal_type(np.add.outer(AR_f8, f8)) # E: numpy.ndarray + +reveal_type(np.frexp.__name__) # E: Literal['frexp'] +reveal_type(np.frexp.ntypes) # E: Literal[4] +reveal_type(np.frexp.identity) # E: None +reveal_type(np.frexp.nin) # E: Literal[1] +reveal_type(np.frexp.nout) # E: Literal[2] +reveal_type(np.frexp.nargs) # E: Literal[3] +reveal_type(np.frexp.signature) # E: None +reveal_type(np.frexp(f8)) # E: Tuple[Any, Any] +reveal_type(np.frexp(AR_f8)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[Any]], numpy.ndarray[Any, numpy.dtype[Any]]] + +reveal_type(np.divmod.__name__) # E: Literal['divmod'] +reveal_type(np.divmod.ntypes) # E: Literal[15] +reveal_type(np.divmod.identity) # E: None +reveal_type(np.divmod.nin) # E: Literal[2] +reveal_type(np.divmod.nout) # E: Literal[2] +reveal_type(np.divmod.nargs) # E: Literal[4] +reveal_type(np.divmod.signature) # E: None +reveal_type(np.divmod(f8, f8)) # E: Tuple[Any, Any] +reveal_type(np.divmod(AR_f8, f8)) # E: Tuple[numpy.ndarray[Any, numpy.dtype[Any]], numpy.ndarray[Any, numpy.dtype[Any]]] + +reveal_type(np.matmul.__name__) # E: Literal['matmul'] +reveal_type(np.matmul.ntypes) # E: Literal[19] +reveal_type(np.matmul.identity) # E: None +reveal_type(np.matmul.nin) # E: Literal[2] +reveal_type(np.matmul.nout) # E: Literal[1] +reveal_type(np.matmul.nargs) # E: Literal[3] +reveal_type(np.matmul.signature) # E: Literal['(n?,k),(k,m?)->(n?,m?)'] +reveal_type(np.matmul.identity) # E: None +reveal_type(np.matmul(AR_f8, AR_f8)) # E: Any +reveal_type(np.matmul(AR_f8, AR_f8, axes=[(0, 1), (0, 1), (0, 1)])) # E: Any diff --git a/numpy/typing/tests/test_generic_alias.py b/numpy/typing/tests/test_generic_alias.py new file mode 100644 index 000000000000..0b99174392f0 --- /dev/null +++ b/numpy/typing/tests/test_generic_alias.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import sys +import types +import pickle +import weakref +from typing import TypeVar, Any, Callable, Tuple, Type, Union + +import pytest +import numpy as np +from numpy.typing._generic_alias import _GenericAlias + +ScalarType = TypeVar("ScalarType", bound=np.generic, covariant=True) +T1 = TypeVar("T1") +T2 = TypeVar("T2") +DType = _GenericAlias(np.dtype, (ScalarType,)) +NDArray = _GenericAlias(np.ndarray, (Any, DType)) + +if sys.version_info >= (3, 9): + DType_ref = types.GenericAlias(np.dtype, (ScalarType,)) + NDArray_ref = types.GenericAlias(np.ndarray, (Any, DType_ref)) + FuncType = Callable[[Union[_GenericAlias, types.GenericAlias]], Any] +else: + DType_ref = NotImplemented + NDArray_ref = NotImplemented + FuncType = Callable[[_GenericAlias], Any] + +GETATTR_NAMES = sorted(set(dir(np.ndarray)) - _GenericAlias._ATTR_EXCEPTIONS) + +BUFFER = np.array([1], dtype=np.int64) +BUFFER.setflags(write=False) + +def _get_subclass_mro(base: type) -> Tuple[type, ...]: + class Subclass(base): # type: ignore[misc,valid-type] + pass + return Subclass.__mro__[1:] + + +class TestGenericAlias: + """Tests for `numpy.typing._generic_alias._GenericAlias`.""" + + @pytest.mark.parametrize("name,func", [ + ("__init__", lambda n: n), + ("__origin__", lambda n: n.__origin__), + ("__args__", lambda n: n.__args__), + ("__parameters__", lambda n: n.__parameters__), + ("__reduce__", lambda n: n.__reduce__()[1:]), + ("__reduce_ex__", lambda n: n.__reduce_ex__(1)[1:]), + ("__mro_entries__", lambda n: n.__mro_entries__([object])), + ("__hash__", lambda n: hash(n)), + ("__repr__", lambda n: repr(n)), + ("__getitem__", lambda n: n[np.float64]), + ("__getitem__", lambda n: n[ScalarType][np.float64]), + ("__getitem__", lambda n: n[Union[np.int64, ScalarType]][np.float64]), + ("__getitem__", lambda n: n[Union[T1, T2]][np.float32, np.float64]), + ("__eq__", lambda n: n == n), + ("__ne__", lambda n: n != np.ndarray), + ("__dir__", lambda n: dir(n)), + ("__call__", lambda n: n((1,), np.int64, BUFFER)), + ("__call__", lambda n: n(shape=(1,), dtype=np.int64, buffer=BUFFER)), + ("subclassing", lambda n: _get_subclass_mro(n)), + ("pickle", lambda n: n == pickle.loads(pickle.dumps(n))), + ("__weakref__", lambda n: n == weakref.ref(n)()), + ]) + def test_pass(self, name: str, func: FuncType) -> None: + """Compare `types.GenericAlias` with its numpy-based backport. + + Checker whether ``func`` runs as intended and that both `GenericAlias` + and `_GenericAlias` return the same result. + + """ + value = func(NDArray) + + if sys.version_info >= (3, 9): + value_ref = func(NDArray_ref) + assert value == value_ref + + @pytest.mark.parametrize("name", GETATTR_NAMES) + def test_getattr(self, name: str) -> None: + """Test that `getattr` wraps around the underlying type, + aka ``__origin__``. + + """ + value = getattr(NDArray, name) + value_ref1 = getattr(np.ndarray, name) + + if sys.version_info >= (3, 9): + value_ref2 = getattr(NDArray_ref, name) + assert value == value_ref1 == value_ref2 + else: + assert value == value_ref1 + + @pytest.mark.parametrize("name,exc_type,func", [ + ("__getitem__", TypeError, lambda n: n[()]), + ("__getitem__", TypeError, lambda n: n[Any, Any]), + ("__getitem__", TypeError, lambda n: n[Any][Any]), + ("isinstance", TypeError, lambda n: isinstance(np.array(1), n)), + ("issublass", TypeError, lambda n: issubclass(np.ndarray, n)), + ("setattr", AttributeError, lambda n: setattr(n, "__origin__", int)), + ("setattr", AttributeError, lambda n: setattr(n, "test", int)), + ("getattr", AttributeError, lambda n: getattr(n, "test")), + ]) + def test_raise( + self, + name: str, + exc_type: Type[BaseException], + func: FuncType, + ) -> None: + """Test operations that are supposed to raise.""" + with pytest.raises(exc_type): + func(NDArray) + + if sys.version_info >= (3, 9): + with pytest.raises(exc_type): + func(NDArray_ref) diff --git a/numpy/typing/tests/test_isfile.py b/numpy/typing/tests/test_isfile.py index 569f05435c3d..b617b3873b64 100644 --- a/numpy/typing/tests/test_isfile.py +++ b/numpy/typing/tests/test_isfile.py @@ -10,7 +10,6 @@ ROOT / "__init__.pyi", ROOT / "char.pyi", ROOT / "ctypeslib.pyi", - ROOT / "emath.pyi", ROOT / "rec.pyi", ROOT / "core" / "__init__.pyi", ROOT / "distutils" / "__init__.pyi", diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index 90de4fd6dc41..be08c1359daa 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -2,10 +2,14 @@ import itertools import os import re +import shutil from collections import defaultdict -from typing import Optional +from typing import Optional, IO, Dict, List import pytest +import numpy as np +from numpy.typing.mypy_plugin import _PRECISION_DICT, _EXTENDED_PRECISION_LIST + try: from mypy import api except ImportError: @@ -18,9 +22,64 @@ PASS_DIR = os.path.join(DATA_DIR, "pass") FAIL_DIR = os.path.join(DATA_DIR, "fail") REVEAL_DIR = os.path.join(DATA_DIR, "reveal") +MISC_DIR = os.path.join(DATA_DIR, "misc") MYPY_INI = os.path.join(DATA_DIR, "mypy.ini") CACHE_DIR = os.path.join(DATA_DIR, ".mypy_cache") +#: A dictionary with file names as keys and lists of the mypy stdout as values. +#: To-be populated by `run_mypy`. +OUTPUT_MYPY: Dict[str, List[str]] = {} + + +def _key_func(key: str) -> str: + """Split at the first occurance of the ``:`` character. + + Windows drive-letters (*e.g.* ``C:``) are ignored herein. + """ + drive, tail = os.path.splitdrive(key) + return os.path.join(drive, tail.split(":", 1)[0]) + + +def _strip_filename(msg: str) -> str: + """Strip the filename from a mypy message.""" + _, tail = os.path.splitdrive(msg) + return tail.split(":", 1)[-1] + + +@pytest.mark.slow +@pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") +@pytest.fixture(scope="module", autouse=True) +def run_mypy() -> None: + """Clears the cache and run mypy before running any of the typing tests. + + The mypy results are cached in `OUTPUT_MYPY` for further use. + + The cache refresh can be skipped using + + NUMPY_TYPING_TEST_CLEAR_CACHE=0 pytest numpy/typing/tests + """ + if os.path.isdir(CACHE_DIR) and bool(os.environ.get("NUMPY_TYPING_TEST_CLEAR_CACHE", True)): + shutil.rmtree(CACHE_DIR) + + for directory in (PASS_DIR, REVEAL_DIR, FAIL_DIR, MISC_DIR): + # Run mypy + stdout, stderr, exit_code = api.run([ + "--config-file", + MYPY_INI, + "--cache-dir", + CACHE_DIR, + directory, + ]) + if stderr: + pytest.fail(f"Unexpected mypy standard error\n\n{stderr}") + elif exit_code not in {0, 1}: + pytest.fail(f"Unexpected mypy exit code: {exit_code}\n\n{stdout}") + stdout = stdout.replace('*', '') + + # Parse the output + iterator = itertools.groupby(stdout.split("\n"), key=_key_func) + OUTPUT_MYPY.update((k, list(v)) for k, v in iterator if k) + def get_test_cases(directory): for root, _, files in os.walk(directory): @@ -41,15 +100,12 @@ def get_test_cases(directory): @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(PASS_DIR)) def test_success(path): - stdout, stderr, exitcode = api.run([ - "--config-file", - MYPY_INI, - "--cache-dir", - CACHE_DIR, - path, - ]) - assert exitcode == 0, stdout - assert re.match(r"Success: no issues found in \d+ source files?", stdout.strip()) + # Alias `OUTPUT_MYPY` so that it appears in the local namespace + output_mypy = OUTPUT_MYPY + if path in output_mypy: + msg = "Unexpected mypy output\n\n" + msg += "\n".join(_strip_filename(v) for v in output_mypy[path]) + raise AssertionError(msg) @pytest.mark.slow @@ -58,37 +114,23 @@ def test_success(path): def test_fail(path): __tracebackhide__ = True - stdout, stderr, exitcode = api.run([ - "--config-file", - MYPY_INI, - "--cache-dir", - CACHE_DIR, - path, - ]) - assert exitcode != 0 - with open(path) as fin: lines = fin.readlines() errors = defaultdict(lambda: "") - error_lines = stdout.rstrip("\n").split("\n") - assert re.match( - r"Found \d+ errors? in \d+ files? \(checked \d+ source files?\)", - error_lines[-1].strip(), - ) - for error_line in error_lines[:-1]: - error_line = error_line.strip() - if not error_line: - continue + output_mypy = OUTPUT_MYPY + assert path in output_mypy + for error_line in output_mypy[path]: + error_line = _strip_filename(error_line) match = re.match( - r"^.+\.py:(?P\d+): (error|note): .+$", + r"(?P\d+): (error|note): .+$", error_line, ) if match is None: raise ValueError(f"Unexpected error line format: {error_line}") lineno = int(match.group('lineno')) - errors[lineno] += error_line + errors[lineno] += f'{error_line}\n' for i, line in enumerate(lines): lineno = i + 1 @@ -101,7 +143,7 @@ def test_fail(path): expected_error = errors.get(lineno) _test_fail(path, marker, expected_error, lineno) else: - pytest.fail(f"Error {repr(errors[lineno])} not found") + pytest.fail(f"Unexpected mypy output\n\n{errors[lineno]}") _FAIL_MSG1 = """Extra error at line {} @@ -123,31 +165,106 @@ def _test_fail(path: str, error: str, expected_error: Optional[str], lineno: int raise AssertionError(_FAIL_MSG2.format(lineno, expected_error, error)) +def _construct_format_dict(): + dct = {k.split(".")[-1]: v.replace("numpy", "numpy.typing") for + k, v in _PRECISION_DICT.items()} + + return { + "uint8": "numpy.unsignedinteger[numpy.typing._8Bit]", + "uint16": "numpy.unsignedinteger[numpy.typing._16Bit]", + "uint32": "numpy.unsignedinteger[numpy.typing._32Bit]", + "uint64": "numpy.unsignedinteger[numpy.typing._64Bit]", + "uint128": "numpy.unsignedinteger[numpy.typing._128Bit]", + "uint256": "numpy.unsignedinteger[numpy.typing._256Bit]", + "int8": "numpy.signedinteger[numpy.typing._8Bit]", + "int16": "numpy.signedinteger[numpy.typing._16Bit]", + "int32": "numpy.signedinteger[numpy.typing._32Bit]", + "int64": "numpy.signedinteger[numpy.typing._64Bit]", + "int128": "numpy.signedinteger[numpy.typing._128Bit]", + "int256": "numpy.signedinteger[numpy.typing._256Bit]", + "float16": "numpy.floating[numpy.typing._16Bit]", + "float32": "numpy.floating[numpy.typing._32Bit]", + "float64": "numpy.floating[numpy.typing._64Bit]", + "float80": "numpy.floating[numpy.typing._80Bit]", + "float96": "numpy.floating[numpy.typing._96Bit]", + "float128": "numpy.floating[numpy.typing._128Bit]", + "float256": "numpy.floating[numpy.typing._256Bit]", + "complex64": "numpy.complexfloating[numpy.typing._32Bit, numpy.typing._32Bit]", + "complex128": "numpy.complexfloating[numpy.typing._64Bit, numpy.typing._64Bit]", + "complex160": "numpy.complexfloating[numpy.typing._80Bit, numpy.typing._80Bit]", + "complex192": "numpy.complexfloating[numpy.typing._96Bit, numpy.typing._96Bit]", + "complex256": "numpy.complexfloating[numpy.typing._128Bit, numpy.typing._128Bit]", + "complex512": "numpy.complexfloating[numpy.typing._256Bit, numpy.typing._256Bit]", + + "ubyte": f"numpy.unsignedinteger[{dct['_NBitByte']}]", + "ushort": f"numpy.unsignedinteger[{dct['_NBitShort']}]", + "uintc": f"numpy.unsignedinteger[{dct['_NBitIntC']}]", + "uintp": f"numpy.unsignedinteger[{dct['_NBitIntP']}]", + "uint": f"numpy.unsignedinteger[{dct['_NBitInt']}]", + "ulonglong": f"numpy.unsignedinteger[{dct['_NBitLongLong']}]", + "byte": f"numpy.signedinteger[{dct['_NBitByte']}]", + "short": f"numpy.signedinteger[{dct['_NBitShort']}]", + "intc": f"numpy.signedinteger[{dct['_NBitIntC']}]", + "intp": f"numpy.signedinteger[{dct['_NBitIntP']}]", + "int_": f"numpy.signedinteger[{dct['_NBitInt']}]", + "longlong": f"numpy.signedinteger[{dct['_NBitLongLong']}]", + + "half": f"numpy.floating[{dct['_NBitHalf']}]", + "single": f"numpy.floating[{dct['_NBitSingle']}]", + "double": f"numpy.floating[{dct['_NBitDouble']}]", + "longdouble": f"numpy.floating[{dct['_NBitLongDouble']}]", + "csingle": f"numpy.complexfloating[{dct['_NBitSingle']}, {dct['_NBitSingle']}]", + "cdouble": f"numpy.complexfloating[{dct['_NBitDouble']}, {dct['_NBitDouble']}]", + "clongdouble": f"numpy.complexfloating[{dct['_NBitLongDouble']}, {dct['_NBitLongDouble']}]", + + # numpy.typing + "_NBitInt": dct['_NBitInt'], + } + + +#: A dictionary with all supported format keys (as keys) +#: and matching values +FORMAT_DICT: Dict[str, str] = _construct_format_dict() + + +def _parse_reveals(file: IO[str]) -> List[str]: + """Extract and parse all ``" # E: "`` comments from the passed file-like object. + + All format keys will be substituted for their respective value from `FORMAT_DICT`, + *e.g.* ``"{float64}"`` becomes ``"numpy.floating[numpy.typing._64Bit]"``. + """ + string = file.read().replace("*", "") + + # Grab all `# E:`-based comments + comments_array = np.char.partition(string.split("\n"), sep=" # E: ")[:, 2] + comments = "/n".join(comments_array) + + # Only search for the `{*}` pattern within comments, + # otherwise there is the risk of accidently grabbing dictionaries and sets + key_set = set(re.findall(r"\{(.*?)\}", comments)) + kwargs = { + k: FORMAT_DICT.get(k, f"") for k in key_set + } + fmt_str = comments.format(**kwargs) + + return fmt_str.split("/n") + + @pytest.mark.slow @pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") @pytest.mark.parametrize("path", get_test_cases(REVEAL_DIR)) def test_reveal(path): __tracebackhide__ = True - stdout, stderr, exitcode = api.run([ - "--config-file", - MYPY_INI, - "--cache-dir", - CACHE_DIR, - path, - ]) - with open(path) as fin: - lines = fin.read().replace('*', '').split("\n") - - stdout_list = stdout.replace('*', '').split("\n") - for error_line in stdout_list: - error_line = error_line.strip() - if not error_line: - continue + lines = _parse_reveals(fin) + output_mypy = OUTPUT_MYPY + assert path in output_mypy + for error_line in output_mypy[path]: + error_line = _strip_filename(error_line) match = re.match( - r"^.+\.py:(?P\d+): note: .+$", + r"(?P\d+): note: .+$", error_line, ) if match is None: @@ -155,7 +272,7 @@ def test_reveal(path): lineno = int(match.group('lineno')) - 1 assert "Revealed type is" in error_line - marker = lines[lineno].split("# E:")[-1].strip() + marker = lines[lineno] _test_reveal(path, marker, error_line, 1 + lineno) @@ -180,3 +297,46 @@ def test_code_runs(path): spec = importlib.util.spec_from_file_location(f"{dirname}.{filename}", path) test_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(test_module) + + +LINENO_MAPPING = { + 3: "uint128", + 4: "uint256", + 6: "int128", + 7: "int256", + 9: "float80", + 10: "float96", + 11: "float128", + 12: "float256", + 14: "complex160", + 15: "complex192", + 16: "complex256", + 17: "complex512", +} + + +@pytest.mark.slow +@pytest.mark.skipif(NO_MYPY, reason="Mypy is not installed") +def test_extended_precision() -> None: + path = os.path.join(MISC_DIR, "extended_precision.py") + output_mypy = OUTPUT_MYPY + assert path in output_mypy + + for _msg in output_mypy[path]: + *_, _lineno, msg_typ, msg = _msg.split(":") + + msg = _strip_filename(msg) + lineno = int(_lineno) + msg_typ = msg_typ.strip() + assert msg_typ in {"error", "note"} + + if LINENO_MAPPING[lineno] in _EXTENDED_PRECISION_LIST: + if msg_typ == "error": + raise ValueError(f"Unexpected reveal line format: {lineno}") + else: + marker = FORMAT_DICT[LINENO_MAPPING[lineno]] + _test_reveal(path, marker, msg, lineno) + else: + if msg_typ == "error": + marker = "Module has no attribute" + _test_fail(path, marker, msg, lineno) diff --git a/numpy/version.py b/numpy/version.py new file mode 100644 index 000000000000..4159a1c0eb82 --- /dev/null +++ b/numpy/version.py @@ -0,0 +1,12 @@ +from ._version import get_versions + +__ALL__ = ['version', 'full_version', 'git_revision', 'release'] + +vinfo = get_versions() +version: str = vinfo["version"] +full_version: str = vinfo['version'] +git_revision: str = vinfo['full-revisionid'] +release = 'dev0' not in version and '+' not in version +short_version: str = vinfo['version'].split("+")[0] + +del get_versions, vinfo diff --git a/pavement.py b/pavement.py index ca1d307f7db3..66c2cf953eaf 100644 --- a/pavement.py +++ b/pavement.py @@ -26,6 +26,7 @@ import sys import shutil import hashlib +import textwrap # The paver package needs to be installed to run tasks import paver @@ -37,7 +38,7 @@ #----------------------------------- # Path to the release notes -RELEASE_NOTES = 'doc/source/release/1.18.0-notes.rst' +RELEASE_NOTES = 'doc/source/release/1.21.0-notes.rst' #------------------------------------------------------- @@ -49,25 +50,13 @@ installersdir=os.path.join("release", "installers")),) -#----------------------------- -# Generate the release version -#----------------------------- +#------------------------ +# Get the release version +#------------------------ sys.path.insert(0, os.path.dirname(__file__)) try: - setup_py = __import__("setup") - FULLVERSION = setup_py.VERSION - # This is duplicated from setup.py - if os.path.exists('.git'): - GIT_REVISION = setup_py.git_version() - elif os.path.exists('numpy/version.py'): - # must be a source distribution, use existing version file - from numpy.version import git_revision as GIT_REVISION - else: - GIT_REVISION = "Unknown" - - if not setup_py.ISRELEASED: - FULLVERSION += '.dev0+' + GIT_REVISION[:7] + from setup import FULLVERSION finally: sys.path.pop(0) @@ -210,22 +199,25 @@ def write_release_task(options, filename='README'): with open(notes) as fnotes: freadme.write(fnotes.read()) - freadme.writelines(""" -Checksums -========= + freadme.writelines(textwrap.dedent( + """ + Checksums + ========= -MD5 ---- -:: + MD5 + --- + :: -""") + """)) freadme.writelines([f' {c}\n' for c in compute_md5(idirs)]) - freadme.writelines(""" -SHA256 ------- -:: -""") + freadme.writelines(textwrap.dedent( + """ + SHA256 + ------ + :: + + """)) freadme.writelines([f' {c}\n' for c in compute_sha256(idirs)]) # generate md file using pandoc before signing diff --git a/pyproject.toml b/pyproject.toml index 2526842fa24b..14f275e97bbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,9 @@ [build-system] # Minimum requirements for the build system to execute. requires = [ + "packaging==20.5; platform_machine=='arm64'", # macos M1 "setuptools<49.2.0", - "wheel<=0.35.1", + "wheel==0.36.2", "Cython>=0.29.21,<3.0", # Note: keep in sync with tools/cythonize.py ] @@ -73,4 +74,3 @@ requires = [ directory = "change" name = "Changes" showcontent = true - diff --git a/pytest.ini b/pytest.ini index 149af04b88c6..dfad538c2327 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,4 +14,5 @@ filterwarnings = # Matrix PendingDeprecationWarning. ignore:the matrix subclass is not ignore:Importing from numpy.matlib is - +# pytest warning when using PYTHONOPTIMIZE + ignore:assertions not in test modules or plugins:pytest.PytestConfigWarning diff --git a/runtests.py b/runtests.py index 87e26768b351..452ccbc64c5a 100755 --- a/runtests.py +++ b/runtests.py @@ -8,7 +8,10 @@ $ python runtests.py $ python runtests.py -s {SAMPLE_SUBMODULE} + $ # Run a standalone test function: $ python runtests.py -t {SAMPLE_TEST} + $ # Run a test defined as a method of a TestXXX class: + $ python runtests.py -t {SAMPLE_TEST2} $ python runtests.py --ipython $ python runtests.py --python somescript.py $ python runtests.py --bench @@ -28,6 +31,12 @@ $ python runtests.py --gcov [...other args...] $ python runtests.py --lcov-html +Run lint checks. +Provide target branch name or `uncommitted` to check before committing: + + $ python runtests.py --lint main + $ python runtests.py --lint uncommitted + """ # # This is a generic test runner script for projects using NumPy's test @@ -37,6 +46,7 @@ PROJECT_MODULE = "numpy" PROJECT_ROOT_FILES = ['numpy', 'LICENSE.txt', 'setup.py'] SAMPLE_TEST = "numpy/linalg/tests/test_linalg.py::test_byteorder_check" +SAMPLE_TEST2 = "numpy/core/tests/test_memmap.py::TestMemmap::test_open_with_filename" SAMPLE_SUBMODULE = "linalg" EXTRA_PATH = ['/usr/lib/ccache', '/usr/lib/f90cache', @@ -68,36 +78,44 @@ def main(argv): parser = ArgumentParser(usage=__doc__.lstrip()) parser.add_argument("--verbose", "-v", action="count", default=1, - help="more verbosity") + help="Add one verbosity level to pytest. Default is 0") parser.add_argument("--debug-info", action="store_true", - help=("add --verbose-cfg to build_src to show compiler " - "configuration output while creating " + help=("Add --verbose-cfg to build_src to show " + "compiler configuration output while creating " "_numpyconfig.h and config.h")) parser.add_argument("--no-build", "-n", action="store_true", default=False, - help="do not build the project (use system installed version)") - parser.add_argument("--build-only", "-b", action="store_true", default=False, - help="just build, do not run any tests") + help="Do not build the project (use system installed " + "version)") + parser.add_argument("--build-only", "-b", action="store_true", + default=False, help="Just build, do not run any tests") parser.add_argument("--doctests", action="store_true", default=False, help="Run doctests in module") parser.add_argument("--refguide-check", action="store_true", default=False, - help="Run refguide (doctest) check (do not run regular tests.)") + help="Run refguide (doctest) check (do not run " + "regular tests.)") parser.add_argument("--coverage", action="store_true", default=False, - help=("report coverage of project code. HTML output goes " - "under build/coverage")) + help=("Report coverage of project code. HTML output " + "goes under build/coverage")) + parser.add_argument("--lint", default=None, + help="'' or 'uncommitted', passed to " + "tools/linter.py [--branch BRANCH] " + "[--uncommitted]") parser.add_argument("--durations", action="store", default=-1, type=int, - help=("Time N slowest tests, time all if 0, time none if < 0")) + help=("Time N slowest tests, time all if 0, time none " + "if < 0")) parser.add_argument("--gcov", action="store_true", default=False, - help=("enable C code coverage via gcov (requires GCC). " - "gcov output goes to build/**/*.gc*")) + help=("Enable C code coverage via gcov (requires " + "GCC). gcov output goes to build/**/*.gc*")) parser.add_argument("--lcov-html", action="store_true", default=False, - help=("produce HTML for C code coverage information " + help=("Produce HTML for C code coverage information " "from a previous run with --gcov. " "HTML output goes to build/lcov/")) parser.add_argument("--mode", "-m", default="fast", help="'fast', 'full', or something that could be " "passed to nosetests -A [default: fast]") parser.add_argument("--submodule", "-s", default=None, - help="Submodule whose tests to run (cluster, constants, ...)") + help="Submodule whose tests to run (cluster, " + "constants, ...)") parser.add_argument("--pythonpath", "-p", default=None, help="Paths to prepend to PYTHONPATH") parser.add_argument("--tests", "-t", action='append', @@ -115,13 +133,16 @@ def main(argv): parser.add_argument("--parallel", "-j", type=int, default=0, help="Number of parallel jobs during build") parser.add_argument("--warn-error", action="store_true", - help="Set -Werror to convert all compiler warnings to errors") + help="Set -Werror to convert all compiler warnings to " + "errors") parser.add_argument("--cpu-baseline", default=None, - help="Specify a list of enabled baseline CPU optimizations"), + help="Specify a list of enabled baseline CPU " + "optimizations"), parser.add_argument("--cpu-dispatch", default=None, help="Specify a list of dispatched CPU optimizations"), parser.add_argument("--disable-optimization", action="store_true", - help="Disable CPU optimized code(dispatch,simd,fast...)"), + help="Disable CPU optimized code (dispatch, simd, " + "fast, ...)"), parser.add_argument("--simd-test", default=None, help="Specify a list of CPU optimizations to be " "tested against NumPy SIMD interface"), @@ -136,7 +157,8 @@ def main(argv): "COMMIT. Note that you need to commit your " "changes first!")) parser.add_argument("args", metavar="ARGS", default=[], nargs=REMAINDER, - help="Arguments to pass to pytest, asv, mypy, Python or shell") + help="Arguments to pass to pytest, asv, mypy, Python " + "or shell") args = parser.parse_args(argv) if args.durations < 0: @@ -162,6 +184,9 @@ def main(argv): print("*** Benchmarks should not be run against debug " "version; remove -g flag ***") + if args.lint: + check_lint(args.lint) + if not args.no_build: # we need the noarch path in case the package is pure python. site_dir, site_dir_noarch = build_project(args) @@ -432,8 +457,6 @@ def build_project(args): cmd += ["build"] if args.parallel > 1: cmd += ["-j", str(args.parallel)] - if args.debug_info: - cmd += ["build_src", "--verbose-cfg"] if args.warn_error: cmd += ["--warn-error"] if args.cpu_baseline: @@ -444,6 +467,8 @@ def build_project(args): cmd += ["--disable-optimization"] if args.simd_test is not None: cmd += ["--simd-test", args.simd_test] + if args.debug_info: + cmd += ["build_src", "--verbose-cfg"] # Install; avoid producing eggs so numpy can be imported from dst_dir. cmd += ['install', '--prefix=' + dst_dir, '--single-version-externally-managed', @@ -524,6 +549,7 @@ def asv_compare_config(bench_path, args, h_commits): is_cached = asv_substitute_config(conf_path, nconf_path, numpy_build_options = ' '.join([f'\\"{v}\\"' for v in build]), + numpy_global_options= ' '.join([f'--global-option=\\"{v}\\"' for v in ["build"] + build]) ) if not is_cached: asv_clear_cache(bench_path, h_commits) @@ -538,7 +564,7 @@ def asv_clear_cache(bench_path, h_commits, env_dir="env"): for asv_build_cache in glob.glob(asv_build_pattern, recursive=True): for c in h_commits: try: shutil.rmtree(os.path.join(asv_build_cache, c)) - except OSError: pass + except OSError: pass def asv_substitute_config(in_config, out_config, **custom_vars): """ @@ -636,6 +662,24 @@ def lcov_generate(): else: print("HTML output generated under build/lcov/") +def check_lint(lint_args): + """ + Adds ROOT_DIR to path and performs lint checks. + This functions exits the program with status code of lint check. + """ + sys.path.append(ROOT_DIR) + try: + from tools.linter import DiffLinter + except ModuleNotFoundError as e: + print(f"Error: {e.msg}. " + "Install using linter_requirements.txt.") + sys.exit(1) + + uncommitted = lint_args == "uncommitted" + branch = "main" if uncommitted else lint_args + + DiffLinter(branch).run_lint(uncommitted) + if __name__ == "__main__": main(argv=sys.argv[1:]) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000000..5bca14ba00ee --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +VCS = git +style = pep440 +versionfile_source = numpy/_version.py +versionfile_build = numpy/_version.py +tag_prefix = v +parentdir_prefix = numpy- diff --git a/setup.py b/setup.py index 93db8c8fa10d..82661046637f 100755 --- a/setup.py +++ b/setup.py @@ -24,13 +24,63 @@ import subprocess import textwrap import warnings +import builtins +import re -if sys.version_info[:2] < (3, 6): - raise RuntimeError("Python version >= 3.6 required.") +# Python supported version checks. Keep right after stdlib imports to ensure we +# get a sensible error for older Python versions +if sys.version_info[:2] < (3, 7): + raise RuntimeError("Python version >= 3.7 required.") -import builtins +import versioneer + + +# This is a bit hackish: we are setting a global variable so that the main +# numpy __init__ can detect if it is being loaded by the setup routine, to +# avoid attempting to load components that aren't built yet. While ugly, it's +# a lot more robust than what was previously being used. +builtins.__NUMPY_SETUP__ = True + +# Needed for backwards code compatibility below and in some CI scripts. +# The version components are changed from ints to strings, but only VERSION +# seems to matter outside of this module and it was already a str. +FULLVERSION = versioneer.get_version() + +# Capture the version string: +# 1.22.0.dev0+ ... -> ISRELEASED == False, VERSION == 1.22.0 +# 1.22.0rc1+ ... -> ISRELEASED == False, VERSION == 1.22.0 +# 1.22.0 ... -> ISRELEASED == True, VERSION == 1.22.0 +# 1.22.0rc1 ... -> ISRELEASED == True, VERSION == 1.22.0 +ISRELEASED = re.search(r'(dev|\+)', FULLVERSION) is None +MAJOR, MINOR, MICRO = re.match(r'(\d+)\.(\d+)\.(\d+)', FULLVERSION).groups() +VERSION = '{}.{}.{}'.format(MAJOR, MINOR, MICRO) + +# The first version not in the `Programming Language :: Python :: ...` classifiers above +if sys.version_info >= (3, 10): + fmt = "NumPy {} may not yet support Python {}.{}." + warnings.warn( + fmt.format(VERSION, *sys.version_info[:2]), + RuntimeWarning) + del fmt + +# BEFORE importing setuptools, remove MANIFEST. Otherwise it may not be +# properly updated when the contents of directories change (true for distutils, +# not sure about setuptools). +if os.path.exists('MANIFEST'): + os.remove('MANIFEST') + +# We need to import setuptools here in order for it to persist in sys.modules. +# Its presence/absence is used in subclassing setup in numpy/distutils/core.py. +# However, we need to run the distutils version of sdist, so import that first +# so that it is in sys.modules +import numpy.distutils.command.sdist +import setuptools + +# Initialize cmdclass from versioneer +from numpy.distutils.core import numpy_cmdclass +cmdclass = versioneer.get_cmdclass(numpy_cmdclass) CLASSIFIERS = """\ Development Status :: 5 - Production/Stable @@ -40,7 +90,6 @@ Programming Language :: C Programming Language :: Python Programming Language :: Python :: 3 -Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 @@ -55,111 +104,6 @@ Operating System :: MacOS """ -MAJOR = 1 -MINOR = 20 -MICRO = 0 -ISRELEASED = False -VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) - -# The first version not in the `Programming Language :: Python :: ...` classifiers above -if sys.version_info >= (3, 10): - warnings.warn( - f"NumPy {VERSION} may not yet support Python " - f"{sys.version_info.major}.{sys.version_info.minor}.", - RuntimeWarning, - ) - - -# Return the git revision as a string -def git_version(): - def _minimal_ext_cmd(cmd): - # construct minimal environment - env = {} - for k in ['SYSTEMROOT', 'PATH', 'HOME']: - v = os.environ.get(k) - if v is not None: - env[k] = v - # LANGUAGE is used on win32 - env['LANGUAGE'] = 'C' - env['LANG'] = 'C' - env['LC_ALL'] = 'C' - out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=env) - return out - - try: - out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD']) - GIT_REVISION = out.strip().decode('ascii') - except (subprocess.SubprocessError, OSError): - GIT_REVISION = "Unknown" - - if not GIT_REVISION: - # this shouldn't happen but apparently can (see gh-8512) - GIT_REVISION = "Unknown" - - return GIT_REVISION - - -# BEFORE importing setuptools, remove MANIFEST. Otherwise it may not be -# properly updated when the contents of directories change (true for distutils, -# not sure about setuptools). -if os.path.exists('MANIFEST'): - os.remove('MANIFEST') - -# This is a bit hackish: we are setting a global variable so that the main -# numpy __init__ can detect if it is being loaded by the setup routine, to -# avoid attempting to load components that aren't built yet. While ugly, it's -# a lot more robust than what was previously being used. -builtins.__NUMPY_SETUP__ = True - - -def get_version_info(): - # Adding the git rev number needs to be done inside write_version_py(), - # otherwise the import of numpy.version messes up the build under Python 3. - FULLVERSION = VERSION - if os.path.exists('.git'): - GIT_REVISION = git_version() - elif os.path.exists('numpy/version.py'): - # must be a source distribution, use existing version file - try: - from numpy.version import git_revision as GIT_REVISION - except ImportError: - raise ImportError("Unable to import git_revision. Try removing " - "numpy/version.py and the build directory " - "before building.") - else: - GIT_REVISION = "Unknown" - - if not ISRELEASED: - FULLVERSION += '.dev0+' + GIT_REVISION[:7] - - return FULLVERSION, GIT_REVISION - - -def write_version_py(filename='numpy/version.py'): - cnt = """ -# THIS FILE IS GENERATED FROM NUMPY SETUP.PY -# -# To compare versions robustly, use `numpy.lib.NumpyVersion` -short_version: str = '%(version)s' -version: str = '%(version)s' -full_version: str = '%(full_version)s' -git_revision: str = '%(git_revision)s' -release: bool = %(isrelease)s - -if not release: - version = full_version -""" - FULLVERSION, GIT_REVISION = get_version_info() - - a = open(filename, 'w') - try: - a.write(cnt % {'version': VERSION, - 'full_version': FULLVERSION, - 'git_revision': GIT_REVISION, - 'isrelease': str(ISRELEASED)}) - finally: - a.close() - def configuration(parent_package='', top_path=None): from numpy.distutils.misc_util import Configuration @@ -228,13 +172,14 @@ def __exit__(self, exception_type, exception_value, traceback): f.write(self.bsd_text) -from distutils.command.sdist import sdist -class sdist_checked(sdist): +# Need to inherit from versioneer version of sdist to get the encoded +# version information. +class sdist_checked(cmdclass['sdist']): """ check submodules on sdist to prevent incomplete tarballs """ def run(self): check_submodules() with concat_license_files(): - sdist.run(self) + super().run() def get_build_overrides(): @@ -307,7 +252,8 @@ def parse_setuppy_commands(): '--maintainer', '--maintainer-email', '--contact', '--contact-email', '--url', '--license', '--description', '--long-description', '--platforms', '--classifiers', - '--keywords', '--provides', '--requires', '--obsoletes'] + '--keywords', '--provides', '--requires', '--obsoletes', + 'version',] for command in info_commands: if command in args: @@ -318,7 +264,8 @@ def parse_setuppy_commands(): # below and not standalone. Hence they're not added to good_commands. good_commands = ('develop', 'sdist', 'build', 'build_ext', 'build_py', 'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm', - 'bdist_wininst', 'bdist_msi', 'bdist_mpkg', 'build_src') + 'bdist_wininst', 'bdist_msi', 'bdist_mpkg', 'build_src', + 'bdist_egg') for command in good_commands: if command in args: @@ -373,8 +320,6 @@ def parse_setuppy_commands(): Instead, build what you want to upload and upload those files with `twine upload -s ` instead. """, - upload_docs="`setup.py upload_docs` is not supported", - easy_install="`setup.py easy_install` is not supported", clean=""" `setup.py clean` is not supported, use one of the following instead: @@ -382,10 +327,6 @@ def parse_setuppy_commands(): - `git clean -Xdf` (cleans all versioned files, doesn't touch files that aren't checked into the git repo) """, - check="`setup.py check` is not supported", - register="`setup.py register` is not supported", - bdist_dumb="`setup.py bdist_dumb` is not supported", - bdist="`setup.py bdist` is not supported", build_sphinx=""" `setup.py build_sphinx` is not supported, use the Makefile under doc/""", @@ -406,21 +347,17 @@ def parse_setuppy_commands(): # Commands that do more than print info, but also don't need Cython and # template parsing. - other_commands = ['egg_info', 'install_egg_info', 'rotate'] + other_commands = ['egg_info', 'install_egg_info', 'rotate', 'dist_info'] for command in other_commands: if command in args: return False # If we got here, we didn't detect what setup.py command was given - import warnings - warnings.warn("Unrecognized setuptools command, proceeding with " - "generating Cython sources and expanding templates", - stacklevel=2) - return True + raise RuntimeError("Unrecognized setuptools command: {}".format(args)) def get_docs_url(): - if not ISRELEASED: + if 'dev' in VERSION: return "https://numpy.org/devdocs" else: # For releases, this URL ends up on pypi. @@ -435,9 +372,6 @@ def setup_package(): os.chdir(src_path) sys.path.insert(0, src_path) - # Rewrite the version file every time - write_version_py() - # The f2py scripts that will be installed if sys.platform == 'win32': f2py_cmds = [ @@ -450,7 +384,7 @@ def setup_package(): 'f2py%s.%s = numpy.f2py.f2py2e:main' % sys.version_info[:2], ] - cmdclass = {"sdist": sdist_checked, } + cmdclass["sdist"] = sdist_checked metadata = dict( name='numpy', maintainer="NumPy Developers", @@ -469,8 +403,9 @@ def setup_package(): classifiers=[_f for _f in CLASSIFIERS.split('\n') if _f], platforms=["Windows", "Linux", "Solaris", "Mac OS-X", "Unix"], test_suite='pytest', + version=versioneer.get_version(), cmdclass=cmdclass, - python_requires='>=3.6', + python_requires='>=3.7', zip_safe=False, entry_points={ 'console_scripts': f2py_cmds @@ -486,8 +421,9 @@ def setup_package(): if run_build: # patches distutils, even though we don't use it - import setuptools # noqa: F401 + #from setuptools import setup from numpy.distutils.core import setup + if 'sdist' not in sys.argv: # Generate Cython sources, unless we're generating an sdist generate_cython() @@ -496,10 +432,8 @@ def setup_package(): # Customize extension building cmdclass['build_clib'], cmdclass['build_ext'] = get_build_overrides() else: + #from numpy.distutils.core import setup from setuptools import setup - # Version number is added to metadata inside configuration() if build - # is run. - metadata['version'] = get_version_info()[0] try: setup(**metadata) diff --git a/shippable.yml b/shippable.yml deleted file mode 100644 index 2843377e26e2..000000000000 --- a/shippable.yml +++ /dev/null @@ -1,67 +0,0 @@ -branches: - only: - - master - - maintenance/* - -language: python - -python: - # use versions available for job image - # aarch64_u16pytall:v6.7.4 - # (what we currently have access to by default) - # this is a bit restrictive in terms - # of version availability / control, - # but it is convenient - - 3.7 - -runtime: - # use the free open source pool of nodes - # only for ARM platform - nodePool: shippable_shared_aarch64 - -build: - ci: - # install dependencies and newer toolchain for gfortran5 - - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - - sudo apt-get update - - sudo apt-get install gcc gfortran libgfortran5 - - target=$(python tools/openblas_support.py) - - ls -lR "${target}" - - sudo cp -r "${target}"/lib/* /usr/lib - - sudo cp "${target}"/include/* /usr/include - - python -m pip install --upgrade pip - - # we will pay the ~13 minute cost of compiling Cython only when a new - # version is scraped in by pip; otherwise, use the cached - # wheel shippable places on Amazon S3 after we build it once - - python -m pip install -r test_requirements.txt --cache-dir=/root/.cache/pip/wheels/$SHIPPABLE_PYTHON_VERSION - # install pytest-xdist to leverage a second core - # for unit tests - - python -m pip install pytest-xdist - - # build and test numpy - - export PATH=$PATH:$SHIPPABLE_REPO_DIR - # build first and adjust PATH so f2py is found in scripts dir - # use > 1 core for build sometimes slows down a fair bit, - # other times modestly speeds up, so avoid for now - - python -m pip install . - - extra_directories=($SHIPPABLE_REPO_DIR/build/*scripts*) - - extra_path=$(printf "%s:" "${extra_directories[@]}") - - export PATH="${extra_path}${PATH}" - # check OpenBLAS version - - python tools/openblas_support.py --check_version - # run the test suite - - python runtests.py -n --debug-info --show-build-log -- -rsx --junit-xml=$SHIPPABLE_REPO_DIR/shippable/testresults/tests.xml -n 2 --durations=10 - - cache: false - -# disable email notification -# of CI job result -integrations: - notifications: - - integrationName: email - type: email - on_success: never - on_failure: never - on_cancel: never - on_pull_request: never diff --git a/site.cfg.example b/site.cfg.example index c809303a205c..1a6b36d2c6eb 100644 --- a/site.cfg.example +++ b/site.cfg.example @@ -28,7 +28,7 @@ # extensions with this dependency. Use the character given by os.pathsep # to separate the items in the list. Note that this character is known to # vary on some unix-like systems; if a colon does not work, try a comma. -# This also applies to include_dirs and src_dirs (see below). +# This also applies to include_dirs. # On UN*X-type systems (OS X, most BSD and Linux systems): # library_dirs = /usr/lib:/usr/local/lib # On Windows: @@ -40,15 +40,6 @@ # List of directories to add to the header file search path. # include_dirs = /usr/include:/usr/local/include # -# src_dirs -# List of directories that contain extracted source code for the -# dependency. For some dependencies, numpy.distutils will be able to build -# them from source if binaries cannot be found. The FORTRAN BLAS and -# LAPACK libraries are one example. However, most dependencies are more -# complicated and require actual installation that you need to do -# yourself. -# src_dirs = /home/username/src/BLAS_SRC:/home/username/src/LAPACK_SRC -# # search_static_first # Boolean (one of (0, false, no, off) for False or (1, true, yes, on) for # True) to tell numpy.distutils to prefer static libraries (.a) over diff --git a/test_requirements.txt b/test_requirements.txt index 33d87b3a987e..e23ec0333299 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,17 +1,15 @@ -cython==0.29.21 -wheel +cython==0.29.23 +wheel<0.36.3 setuptools<49.2.0 -hypothesis==5.41.3 -pytest==6.0.2 -pytz==2020.4 -pytest-cov==2.10.1 -pickle5; python_version == '3.7' -pickle5; python_version == '3.6' and platform_python_implementation != 'PyPy' +hypothesis==6.12.0 +pytest==6.2.4 +pytz==2021.1 +pytest-cov==2.12.0 +pickle5; python_version == '3.7' and platform_python_implementation != 'PyPy' # for numpy.random.test.test_extending cffi # For testing types. Notes on the restrictions: # - Mypy relies on C API features not present in PyPy -# - Mypy doesn't currently work on Python 3.9 -# - Python 3.6 doesn't work because it doesn't understand py.typed -mypy==0.790; platform_python_implementation != "PyPy" and python_version > "3.6" -typing_extensions +# - There is no point in installing typing_extensions without mypy +mypy==0.812; platform_python_implementation != "PyPy" +typing_extensions==3.10.0.0; platform_python_implementation != "PyPy" diff --git a/tools/changelog.py b/tools/changelog.py index 920f5b87faa3..2bd7cde08d83 100755 --- a/tools/changelog.py +++ b/tools/changelog.py @@ -22,6 +22,7 @@ - gitpython - pygithub +- git >= 2.29.0 Some code was copied from scipy `tools/gh_list.py` and `tools/authors.py`. @@ -57,21 +58,29 @@ def get_authors(revision_range): - pat = u'^.*\\t(.*)$' lst_release, cur_release = [r.strip() for r in revision_range.split('..')] + authors_pat = r'^.*\t(.*)$' - # authors, in current release and previous to current release. - cur = set(re.findall(pat, this_repo.git.shortlog('-s', revision_range), - re.M)) - pre = set(re.findall(pat, this_repo.git.shortlog('-s', lst_release), - re.M)) + # authors and co-authors in current and previous releases. + grp1 = '--group=author' + grp2 = '--group=trailer:co-authored-by' + cur = this_repo.git.shortlog('-s', grp1, grp2, revision_range) + pre = this_repo.git.shortlog('-s', grp1, grp2, lst_release) + authors_cur = set(re.findall(authors_pat, cur, re.M)) + authors_pre = set(re.findall(authors_pat, pre, re.M)) - # Homu is the author of auto merges, clean him out. - cur.discard('Homu') - pre.discard('Homu') + # Ignore the bot Homu. + authors_cur.discard('Homu') + authors_pre.discard('Homu') + + # Ignore the bot dependabot-preview + authors_cur.discard('dependabot-preview') + authors_pre.discard('dependabot-preview') # Append '+' to new authors. - authors = [s + u' +' for s in cur - pre] + [s for s in cur & pre] + authors_new = [s + ' +' for s in authors_cur - authors_pre] + authors_old = [s for s in authors_cur & authors_pre] + authors = authors_new + authors_old authors.sort() return authors @@ -82,18 +91,18 @@ def get_pull_requests(repo, revision_range): # From regular merges merges = this_repo.git.log( '--oneline', '--merges', revision_range) - issues = re.findall(u"Merge pull request \\#(\\d*)", merges) + issues = re.findall(r"Merge pull request \#(\d*)", merges) prnums.extend(int(s) for s in issues) # From Homu merges (Auto merges) - issues = re. findall(u"Auto merge of \\#(\\d*)", merges) + issues = re. findall(r"Auto merge of \#(\d*)", merges) prnums.extend(int(s) for s in issues) # From fast forward squash-merges commits = this_repo.git.log( '--oneline', '--no-merges', '--first-parent', revision_range) - issues = re.findall(u'^.*\\(\\#(\\d+)\\)$', commits, re.M) - prnums.extend(int(s) for s in issues) + issues = re.findall(r'^.*\((\#|gh-|gh-\#)(\d+)\)$', commits, re.M) + prnums.extend(int(s[1]) for s in issues) # get PR data from github repo prnums.sort() @@ -109,31 +118,31 @@ def main(token, revision_range): # document authors authors = get_authors(revision_range) - heading = u"Contributors" + heading = "Contributors" print() print(heading) - print(u"="*len(heading)) + print("="*len(heading)) print(author_msg % len(authors)) for s in authors: - print(u'* ' + s) + print('* ' + s) # document pull requests pull_requests = get_pull_requests(github_repo, revision_range) - heading = u"Pull requests merged" - pull_msg = u"* `#{0} <{1}>`__: {2}" + heading = "Pull requests merged" + pull_msg = "* `#{0} <{1}>`__: {2}" print() print(heading) - print(u"="*len(heading)) + print("="*len(heading)) print(pull_request_msg % len(pull_requests)) for pull in pull_requests: - title = re.sub(u"\\s+", u" ", pull.title.strip()) + title = re.sub(r"\s+", " ", pull.title.strip()) if len(title) > 60: - remainder = re.sub(u"\\s.*$", u"...", title[60:]) + remainder = re.sub(r"\s.*$", "...", title[60:]) if len(remainder) > 20: - remainder = title[:80] + u"..." + remainder = title[:80] + "..." else: title = title[:60] + remainder print(pull_msg.format(pull.number, pull.html_url, title)) diff --git a/tools/ci/push_docs_to_repo.py b/tools/ci/push_docs_to_repo.py index ae530548459b..058f748ec1af 100755 --- a/tools/ci/push_docs_to_repo.py +++ b/tools/ci/push_docs_to_repo.py @@ -45,6 +45,9 @@ def run(cmd, stdout=True): os.chdir(workdir) run(['git', 'init']) +# ensure the working branch is called "main" +# (`--initial-branch=main` appared to have failed on older git versions): +run(['git', 'checkout', '-b', 'main']) run(['git', 'remote', 'add', 'origin', args.remote]) run(['git', 'config', '--local', 'user.name', args.committer]) run(['git', 'config', '--local', 'user.email', args.email]) @@ -56,7 +59,7 @@ def run(cmd, stdout=True): print('- uploading as %s <%s>' % (args.committer, args.email)) if args.force: - run(['git', 'push', 'origin', 'master', '--force']) + run(['git', 'push', 'origin', 'main', '--force']) else: print('\n!! No `--force` argument specified; aborting') print('!! Before enabling that flag, make sure you know what it does\n') diff --git a/tools/ci/test_all_newsfragments_used.py b/tools/ci/test_all_newsfragments_used.py index c2e03154945d..62c9a05f95da 100755 --- a/tools/ci/test_all_newsfragments_used.py +++ b/tools/ci/test_all_newsfragments_used.py @@ -12,5 +12,5 @@ if fragments: print("The following files were not found by towncrier:") - print(" " + " \n".join(fragments)) + print(" " + "\n ".join(fragments)) sys.exit(1) diff --git a/tools/commitstats.py b/tools/commitstats.py index 14c37d4d2b97..534f0a1b8416 100644 --- a/tools/commitstats.py +++ b/tools/commitstats.py @@ -4,7 +4,7 @@ import numpy as np import os -names = re.compile(r'r\d+\s[|]\s(.*)\s[|]\s200') +names = re.compile(r'r\d+\s\|\s(.*)\s\|\s200') def get_count(filename, repo): mystr = open(filename).read() diff --git a/tools/cythonize.py b/tools/cythonize.py index 6cebf0f729ea..06cf54c9a41c 100755 --- a/tools/cythonize.py +++ b/tools/cythonize.py @@ -56,12 +56,14 @@ def process_pyx(fromfile, tofile): try: # try the cython in the installed python first (somewhat related to scipy/scipy#2397) + import Cython from Cython.Compiler.Version import version as cython_version - except ImportError: + except ImportError as e: # The `cython` command need not point to the version installed in the # Python running this script, so raise an error to avoid the chance of # using the wrong version of Cython. - raise OSError('Cython needs to be installed in Python as a module') + msg = 'Cython needs to be installed in Python as a module' + raise OSError(msg) from e else: # check the version, and invoke through python from distutils.version import LooseVersion @@ -73,7 +75,9 @@ def process_pyx(fromfile, tofile): required_version = LooseVersion('0.29.21') if LooseVersion(cython_version) < required_version: - raise RuntimeError(f'Building {VENDOR} requires Cython >= {required_version}') + cython_path = Cython.__file__ + raise RuntimeError(f'Building {VENDOR} requires Cython >= {required_version}' + f', found {cython_version} at {cython_path}') subprocess.check_call( [sys.executable, '-m', 'cython'] + flags + ["-o", tofile, fromfile]) diff --git a/tools/functions_missing_types.py b/tools/functions_missing_types.py index c2fe156f0716..0461aabd3634 100755 --- a/tools/functions_missing_types.py +++ b/tools/functions_missing_types.py @@ -41,10 +41,12 @@ "fastCopyAndTranspose", "get_array_wrap", "int_asbuffer", + "numarray", "oldnumeric", "safe_eval", "set_numeric_ops", "test", + "typeDict", # Builtins "bool", "complex", diff --git a/tools/gitpod/Dockerfile b/tools/gitpod/Dockerfile new file mode 100644 index 000000000000..e2e0e1bc9571 --- /dev/null +++ b/tools/gitpod/Dockerfile @@ -0,0 +1,101 @@ +# +# Dockerfile for NumPy development +# +# Usage: +# ------- +# +# To make a local build of the container, from the 'Docker-dev' directory: +# docker build --rm -f "Dockerfile" -t "." +# +# To use the container use the following command. It assumes that you are in +# the root folder of the NumPy git repository, making it available as +# /home/numpy in the container. Whatever changes you make to that directory +# are visible in the host and container. +# The docker image is retrieved from the NumPy dockerhub repository +# +# docker run --rm -it -v $(pwd):/home/numpy numpy/numpy-dev: +# +# By default the container will activate the conda environment numpy-dev +# which contains all the dependencies needed for NumPy development +# +# To build NumPy run: python setup.py build_ext --inplace +# +# To run the tests use: python runtests.py +# +# This image is based on: Ubuntu 20.04 (focal) +# https://hub.docker.com/_/ubuntu/?tab=tags&name=focal +# OS/ARCH: linux/amd64 +FROM gitpod/workspace-base:latest + +ARG MAMBAFORGE_VERSION="4.10.0-0" +ARG CONDA_ENV=numpy-dev + + +# ---- Configure environment ---- +ENV CONDA_DIR=/home/gitpod/mambaforge3 \ + SHELL=/bin/bash +ENV PATH=${CONDA_DIR}/bin:$PATH \ + WORKSPACE=/workspace/numpy + + +# ----------------------------------------------------------------------------- +# ---- Creating as root - note: make sure to change to gitpod in the end ---- +USER root + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -yq --no-install-recommends \ + ca-certificates \ + dirmngr \ + dvisvgm \ + gnupg \ + gpg-agent \ + texlive-latex-extra \ + vim && \ + # this needs to be done after installing dirmngr + apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \ + apt-add-repository https://cli.github.com/packages && \ + apt-get install -yq --no-install-recommends \ + gh && \ + locale-gen en_US.UTF-8 && \ + apt-get clean && \ + rm -rf /var/cache/apt/* &&\ + rm -rf /var/lib/apt/lists/* &&\ + rm -rf /tmp/* + +# Allows this Dockerfile to activate conda environments +SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] + +# ----------------------------------------------------------------------------- +# ---- Installing mamba ---- +RUN wget -q -O mambaforge3.sh \ + "https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-$MAMBAFORGE_VERSION-Linux-x86_64.sh" && \ + bash mambaforge3.sh -p ${CONDA_DIR} -b && \ + rm mambaforge3.sh + +# ----------------------------------------------------------------------------- +# ---- Copy needed files ---- +# basic workspace configurations +COPY ./tools/gitpod/workspace_config /usr/local/bin/workspace_config + +RUN chmod a+rx /usr/local/bin/workspace_config && \ + workspace_config + +# Copy conda environment file into the container - this needs to exists inside +# the container to create a conda environment from it +COPY environment.yml /tmp/environment.yml + +# ----------------------------------------------------------------------------- +# ---- Create conda environment ---- +# Install NumPy dependencies +RUN mamba env create -f /tmp/environment.yml && \ + conda activate ${CONDA_ENV} && \ + mamba install ccache -y && \ + # needed for docs rendering later on + python -m pip install --no-cache-dir sphinx-autobuild && \ + conda clean --all -f -y && \ + rm -rf /tmp/* + +# ----------------------------------------------------------------------------- +# Always make sure we are not root +USER gitpod \ No newline at end of file diff --git a/tools/gitpod/gitpod.Dockerfile b/tools/gitpod/gitpod.Dockerfile new file mode 100644 index 000000000000..ad731fd63d01 --- /dev/null +++ b/tools/gitpod/gitpod.Dockerfile @@ -0,0 +1,45 @@ +# Doing a local shallow clone - keeps the container secure +# and much slimmer than using COPY directly or making a +# remote clone +ARG BASE_CONTAINER="numpy/numpy-dev:latest" +FROM gitpod/workspace-base:latest as clone + +COPY --chown=gitpod . /tmp/numpy_repo +RUN git clone --depth 1 file:////tmp/numpy_repo /tmp/numpy + +# ----------------------------------------------------------------------------- +# Using the numpy-dev Docker image as a base +# This way, we ensure we have all the needed compilers and dependencies +# while reducing the build time +FROM ${BASE_CONTAINER} as build + +# ----------------------------------------------------------------------------- +USER root + +# ----------------------------------------------------------------------------- +# ---- ENV variables ---- +# ---- Directories needed ---- +ENV WORKSPACE=/workspace/numpy/ \ + CONDA_ENV=numpy-dev + +# Allows this Dockerfile to activate conda environments +SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] + +# Copy over the shallow clone +COPY --from=clone --chown=gitpod /tmp/numpy ${WORKSPACE} + +# Everything happens in the /workspace/numpy directory +WORKDIR ${WORKSPACE} + +# Build numpy to populate the cache used by ccache +RUN conda activate ${CONDA_ENV} && \ + python setup.py build_ext --inplace && \ + ccache -s + +# Gitpod will load the repository into /workspace/numpy. We remove the +# directoy from the image to prevent conflicts +RUN rm -rf ${WORKSPACE} + +# ----------------------------------------------------------------------------- +# Always return to non privileged user +USER gitpod diff --git a/tools/gitpod/settings.json b/tools/gitpod/settings.json new file mode 100644 index 000000000000..8f070c04c05a --- /dev/null +++ b/tools/gitpod/settings.json @@ -0,0 +1,9 @@ +{ + "restructuredtext.languageServer.disabled": true, + "restructuredtext.builtDocumentationPath": "${workspaceRoot}/doc/build/html", + "restructuredtext.confPath": "", + "restructuredtext.updateOnTextChanged": "true", + "restructuredtext.updateDelay": 300, + "restructuredtext.linter.disabled": true, + "python.pythonPath": "/home/gitpod/mambaforge3/envs/numpy-dev/bin/python" +} \ No newline at end of file diff --git a/tools/gitpod/workspace_config b/tools/gitpod/workspace_config new file mode 100644 index 000000000000..aa859c9be4d0 --- /dev/null +++ b/tools/gitpod/workspace_config @@ -0,0 +1,58 @@ +#!/bin/bash +# Basic configurations for the workspace + +set -e + +# gitpod/workspace-base needs at least one file here +touch /home/gitpod/.bashrc.d/empty + +# Add git aliases +git config --global alias.co checkout +git config --global alias.ci commit +git config --global alias.st status +git config --global alias.br branch +git config --global alias.hist "log --pretty=format:'%h %ad | %s%d [%an]' --graph --date=short" +git config --global alias.type 'cat-file -t' +git config --global alias.dump 'cat-file -p' + +# Enable basic vim defaults in ~/.vimrc +echo "filetype plugin indent on" >>~/.vimrc +echo "set colorcolumn=80" >>~/.vimrc +echo "set number" >>~/.vimrc +echo "syntax enable" >>~/.vimrc + +# Vanity custom bash prompt - makes it more legible +echo "PS1='\[\e]0;\u \w\a\]\[\033[01;36m\]\u\[\033[m\] > \[\033[38;5;141m\]\w\[\033[m\] \\$ '" >>~/.bashrc + +# Enable prompt color in the skeleton .bashrc +# hadolint ignore=SC2016 +sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc + +# .gitpod.yml is configured to install NumPy from /workspace/numpy +echo "export PYTHONPATH=${WORKSPACE}" >>~/.bashrc + +# make conda activate command available from /bin/bash (login and interactive) +if [[ ! -f "/etc/profile.d/conda.sh" ]]; then + ln -s ${CONDA_DIR}/etc/profile.d/conda.sh /etc/profile.d/conda.sh +fi +echo ". ${CONDA_DIR}/etc/profile.d/conda.sh" >>~/.bashrc +echo "conda activate numpy-dev" >>~/.bashrc + +# Enable prompt color in the skeleton .bashrc +# hadolint ignore=SC2016 +sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc + +# .gitpod.yml is configured to install numpy from /workspace/numpy +echo "export PYTHONPATH=/workspace/numpy" >>~/.bashrc + +# Set up ccache for compilers for this Dockerfile +# REF: https://github.com/conda-forge/compilers-feedstock/issues/31 +echo "conda activate numpy-dev" >>~/.startuprc +echo "export CC=\"ccache \$CC\"" >>~/.startuprc +echo "export CXX=\"ccache \$CXX\"" >>~/.startuprc +echo "export F77=\"ccache \$F77\"" >>~/.startuprc +echo "export F90=\"ccache \$F90\"" >>~/.startuprc +echo "export GFORTRAN=\"ccache \$GFORTRAN\"" >>~/.startuprc +echo "export FC=\"ccache \$FC\"" >>~/.startuprc +echo "source ~/.startuprc" >>~/.profile +echo "source ~/.startuprc" >>~/.bashrc diff --git a/tools/lint_diff.ini b/tools/lint_diff.ini new file mode 100644 index 000000000000..3b66d3c3e900 --- /dev/null +++ b/tools/lint_diff.ini @@ -0,0 +1,4 @@ +[pycodestyle] +max_line_length = 79 +statistics = True +ignore = E121,E122,E123,E125,E126,E127,E128,E226,E251,E265,E266,E302,E402,E704,E712,E721,E731,E741,W291,W293,W391,W503,W504 diff --git a/tools/linter.py b/tools/linter.py new file mode 100644 index 000000000000..fd229dbef8f9 --- /dev/null +++ b/tools/linter.py @@ -0,0 +1,83 @@ +import os +import sys +import subprocess +from argparse import ArgumentParser +from git import Repo, exc + +CONFIG = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'lint_diff.ini', +) + +# NOTE: The `diff` and `exclude` options of pycodestyle seem to be +# incompatible, so instead just exclude the necessary files when +# computing the diff itself. +EXCLUDE = ( + "numpy/typing/tests/data/", + "numpy/__config__.py", +) + + +class DiffLinter: + def __init__(self, branch): + self.branch = branch + self.repo = Repo('.') + self.head = self.repo.head.commit + + def get_branch_diff(self, uncommitted = False): + """ + Determine the first common ancestor commit. + Find diff between branch and FCA commit. + Note: if `uncommitted` is set, check only + uncommitted changes + """ + try: + commit = self.repo.merge_base(self.branch, self.head)[0] + except exc.GitCommandError: + print(f"Branch with name `{self.branch}` does not exist") + sys.exit(1) + + exclude = [f':(exclude){i}' for i in EXCLUDE] + if uncommitted: + diff = self.repo.git.diff( + self.head, '--unified=0', '***.py', *exclude + ) + else: + diff = self.repo.git.diff( + commit, self.head, '--unified=0', '***.py', *exclude + ) + return diff + + def run_pycodestyle(self, diff): + """ + Original Author: Josh Wilson (@person142) + Source: + https://github.com/scipy/scipy/blob/main/tools/lint_diff.py + Run pycodestyle on the given diff. + """ + res = subprocess.run( + ['pycodestyle', '--diff', '--config', CONFIG], + input=diff, + stdout=subprocess.PIPE, + encoding='utf-8', + ) + return res.returncode, res.stdout + + def run_lint(self, uncommitted): + diff = self.get_branch_diff(uncommitted) + retcode, errors = self.run_pycodestyle(diff) + + errors and print(errors) + + sys.exit(retcode) + + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument("--branch", type=str, default='main', + help="The branch to diff against") + parser.add_argument("--uncommitted", action='store_true', + help="Check only uncommitted changes") + args = parser.parse_args() + + DiffLinter(args.branch).run_lint(args.uncommitted) diff --git a/tools/npy_tempita/__init__.py b/tools/npy_tempita/__init__.py index 50a99510450d..fedcd91f45b8 100644 --- a/tools/npy_tempita/__init__.py +++ b/tools/npy_tempita/__init__.py @@ -705,7 +705,7 @@ def lex(s, name=None, trim_whitespace=True, line_offset=0, delimiters=None): def trim_lex(tokens): r""" - Takes a lexed set of tokens, and removes whitespace when there is + Takes a lexed list of tokens, and removes whitespace when there is a directive on a line by itself: >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) diff --git a/tools/openblas_support.py b/tools/openblas_support.py index c0e9dddab467..d11ad173befa 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -2,6 +2,7 @@ import hashlib import os import platform +import sysconfig import sys import shutil import tarfile @@ -12,99 +13,36 @@ from urllib.request import urlopen, Request from urllib.error import HTTPError -OPENBLAS_V = '0.3.12' -# Temporary build of OpenBLAS to test a fix for dynamic detection of CPU -OPENBLAS_LONG = 'v0.3.12-buffersize20' +OPENBLAS_V = '0.3.13' +OPENBLAS_LONG = 'v0.3.13-62-gaf2b0d02' BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' -ARCHITECTURES = ['', 'windows', 'darwin', 'aarch64', 'x86_64', - 'i686', 'ppc64le', 's390x'] -sha256_vals = { - "openblas-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": - "7249d68c02e6b6339e06edfeab1fecddf29ee1e67a3afaa77917c320c43de840", - "openblas64_-v0.3.7-527-g79fd006c-win_amd64-gcc_7_1_0.zip": - "6488e0961a5926e47242f63b63b41cfdd661e6f1d267e8e313e397cde4775c17", - "openblas-v0.3.7-527-g79fd006c-win32-gcc_7_1_0.zip": - "5fb0867ca70b1d0fdbf68dd387c0211f26903d74631420e4aabb49e94aa3930d", - "openblas-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "69434bd626bbc495da9ce8c36b005d140c75e3c47f94e88c764a199e820f9259", - "openblas64_-v0.3.7-527-g79fd006c-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "093f6d953e3fa76a86809be67bd1f0b27656671b5a55b233169cfaa43fd63e22", - "openblas-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": - "42676c69dc48cd6e412251b39da6b955a5a0e00323ddd77f9137f7c259d35319", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2014_aarch64.tar.gz": - "5aec167af4052cf5e9e3e416c522d9794efabf03a2aea78b9bb3adc94f0b73d8", - "openblas-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": - "fa67c6cc29d4cc5c70a147c80526243239a6f95fc3feadcf83a78176cd9c526b", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2010_x86_64.tar.gz": - "9ad34e89a5307dcf5823bf5c020580d0559a0c155fe85b44fc219752e61852b0", - "openblas-v0.3.7-527-g79fd006c-manylinux2010_i686.tar.gz": - "0b8595d316c8b7be84ab1f1d5a0c89c1b35f7c987cdaf61d441bcba7ab4c7439", - "openblas-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": - "3e1c7d6472c34e7210e3605be4bac9ddd32f613d44297dc50cf2d067e720c4a9", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2014_ppc64le.tar.gz": - "a0885873298e21297a04be6cb7355a585df4fa4873e436b4c16c0a18fc9073ea", - "openblas-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": - "79b454320817574e20499d58f05259ed35213bea0158953992b910607b17f240", - "openblas64_-v0.3.7-527-g79fd006c-manylinux2014_s390x.tar.gz": - "9fddbebf5301518fc4a5d2022a61886544a0566868c8c014359a1ee6b17f2814", - "openblas-v0.3.7-527-g79fd006c-manylinux1_i686.tar.gz": - "24fb92684ec4676185fff5c9340f50c3db6075948bcef760e9c715a8974e4680", - "openblas-v0.3.7-527-g79fd006c-manylinux1_x86_64.tar.gz": - "ebb8236b57a1b4075fd5cdc3e9246d2900c133a42482e5e714d1e67af5d00e62", - "openblas-v0.3.10-win_amd64-gcc_7_1_0.zip": - "e5356a2aa4aa7ed9233b2ca199fdd445f55ba227f004ebc63071dfa2426e9b09", - "openblas64_-v0.3.10-win_amd64-gcc_7_1_0.zip": - "aea3f9c8bdfe0b837f0d2739a6c755b12b6838f6c983e4ede71b4e1b576e6e77", - "openblas-v0.3.10-win32-gcc_7_1_0.zip": - "af1ad3172b23f7c6ef2234151a71d3be4d92010dad4dfb25d07cf5a20f009202", - "openblas64_-v0.3.10-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "38b61c58d63048731d6884fea7b63f8cbd610e85b138c6bac0e39fd77cd4699b", - "openblas-v0.3.10-manylinux2014_aarch64.tar.gz": - "c4444b9836ec26f7772fae02851961bf73177ff2aa436470e56fab8a1ef8d405", - "openblas-v0.3.10-manylinux2010_x86_64.tar.gz": - "cb7988c4a015aece9c49b1169f51c4ac2287fb9aab8114c8ab67792138ffc85e", - "openblas-v0.3.10-manylinux2010_i686.tar.gz": - "dc637801dd80ebd6394ea8b4a97f8858e4224870ea9214de08bebbdddd8e206e", - "openblas-v0.3.10-manylinux1_x86_64.tar.gz": - "ec1f9e9b2a62d5cb9e2634b88ee2da7cb6b07702d5a0e8b190d680a31adfa23a", - "openblas-v0.3.10-manylinux1_i686.tar.gz": - "b13d9d14e6bd452c0fbadb5cd5fda05b98b1e14043edb13ead90694d4cc07f0e", - "openblas-v0.3.10-manylinux2014_ppc64le.tar.gz": - "1cbc8176986099cf0cbb8f64968d5a14880d602d4b3c59a91d75b69b8760cde3", - "openblas-v0.3.10-manylinux2014_s390x.tar.gz": - "fa6722f0b12507ab0a65f38501ed8435b573df0adc0b979f47cdc4c9e9599475", - "openblas-v0.3.10-macosx_10_9_x86_64-gf_1becaaa.tar.gz": - "c6940b5133e687ae7a4f9c7c794f6a6d92b619cf41e591e5db07aab5da118199", - "openblas64_-v0.3.10-manylinux2014_s390x.tar.gz": - "e0347dd6f3f3a27d2f5e76d382e8a4a68e2e92f5f6a10e54ef65c7b14b44d0e8", - "openblas64_-v0.3.10-manylinux2014_ppc64le.tar.gz": - "4b96a51ac767ec0aabb821c61bcd3420e82e987fc93f7e1f85aebb2a845694eb", - "openblas64_-v0.3.10-manylinux2010_x86_64.tar.gz": - "f68fea21fbc73d06b7566057cad2ed8c7c0eb71fabf9ed8a609f86e5bc60ce5e", - "openblas64_-v0.3.10-manylinux2014_aarch64.tar.gz": - "15e6eed8cb0df8b88e52baa136ffe1769c517e9de7bcdfd81ec56420ae1069e9", - "openblas64_-v0.3.10-win_amd64-gcc_7_1_0.zip": - "aea3f9c8bdfe0b837f0d2739a6c755b12b6838f6c983e4ede71b4e1b576e6e77", -} - - +SUPPORTED_PLATFORMS = [ + 'linux-aarch64', + 'linux-x86_64', + 'linux-i686', + 'linux-ppc64le', + 'linux-s390x', + 'win-amd64', + 'win-32', + 'macosx-x86_64', + 'macosx-arm64', +] IS_32BIT = sys.maxsize < 2**32 -def get_arch(): - if platform.system() == 'Windows': - ret = 'windows' - elif platform.system() == 'Darwin': - ret = 'darwin' - else: - ret = platform.uname().machine - # What do 32 bit machines report? - # If they are a docker, they can report x86_64 - if 'x86' in ret and IS_32BIT: - ret = 'i686' - assert ret in ARCHITECTURES, f'invalid architecture {ret}' - return ret +def get_plat(): + plat = sysconfig.get_platform() + plat_split = plat.split("-") + arch = plat_split[-1] + if arch == "win32": + plat = "win-32" + elif arch in ["universal2", "intel"]: + plat = f"macosx-{platform.uname().machine}" + elif len(plat_split) > 2: + plat = f"{plat_split[0]}-{arch}" + assert plat in SUPPORTED_PLATFORMS, f'invalid platform {plat}' + return plat def get_ilp64(): @@ -122,34 +60,38 @@ def get_manylinux(arch): default = '2014' ret = os.environ.get("MB_ML_VER", default) # XXX For PEP 600 this can be a glibc version - assert ret in ('1', '2010', '2014'), f'invalid MB_ML_VER {ret}' + assert ret in ('1', '2010', '2014', '_2_24'), f'invalid MB_ML_VER {ret}' return ret -def download_openblas(target, arch, ilp64, is_32bit): - ml_ver = get_manylinux(arch) +def download_openblas(target, plat, ilp64): + osname, arch = plat.split("-") fnsuffix = {None: "", "64_": "64_"}[ilp64] filename = '' headers = {'User-Agent': ('Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 ; ' '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3')} - if arch in ('aarch64', 'ppc64le', 's390x', 'x86_64', 'i686'): + suffix = None + if osname == "linux": + ml_ver = get_manylinux(arch) suffix = f'manylinux{ml_ver}_{arch}.tar.gz' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'tar.gz' - elif arch == 'darwin': + elif plat == 'macosx-x86_64': suffix = 'macosx_10_9_x86_64-gf_1becaaa.tar.gz' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'tar.gz' - elif arch == 'windows': - if is_32bit: + elif plat == 'macosx-arm64': + suffix = 'macosx_11_0_arm64-gf_f26990f.tar.gz' + typ = 'tar.gz' + elif osname == 'win': + if plat == "win-32": suffix = 'win32-gcc_8_1_0.zip' else: suffix = 'win_amd64-gcc_8_1_0.zip' - filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' typ = 'zip' - if not filename: + + if not suffix: return None + filename = f'{BASEURL}/openblas{fnsuffix}-{OPENBLAS_LONG}-{suffix}' req = Request(url=filename, headers=headers) try: response = urlopen(req) @@ -164,22 +106,13 @@ def download_openblas(target, arch, ilp64, is_32bit): data = response.read() # Verify hash key = os.path.basename(filename) - sha256_returned = hashlib.sha256(data).hexdigest() - if 0: - if key not in sha256_vals: - raise ValueError( - f'\nkey "{key}" with hash "{sha256_returned}" not in sha256_vals\n') - sha256_expected = sha256_vals[key] - if sha256_returned != sha256_expected: - # print(f'\nkey "{key}" with hash "{sha256_returned}" mismatch\n') - raise ValueError(f'sha256 hash mismatch for filename {filename}') print("Saving to file", file=sys.stderr) with open(target, 'wb') as fid: fid.write(data) return typ -def setup_openblas(arch=get_arch(), ilp64=get_ilp64(), is_32bit=IS_32BIT): +def setup_openblas(plat=get_plat(), ilp64=get_ilp64()): ''' Download and setup an openblas library for building. If successful, the configuration script will find it automatically. @@ -191,12 +124,13 @@ def setup_openblas(arch=get_arch(), ilp64=get_ilp64(), is_32bit=IS_32BIT): To determine success, do ``os.path.exists(msg)`` ''' _, tmp = mkstemp() - if not arch: - raise ValueError('unknown architecture') - typ = download_openblas(tmp, arch, ilp64, is_32bit) + if not plat: + raise ValueError('unknown platform') + typ = download_openblas(tmp, plat, ilp64) if not typ: return '' - if arch == 'windows': + osname, arch = plat.split("-") + if osname == 'win': if not typ == 'zip': return f'expecting to download zipfile on windows, not {typ}' return unpack_windows_zip(tmp) @@ -296,23 +230,22 @@ def make_init(dirname): """)) -def test_setup(arches): +def test_setup(plats): ''' Make sure all the downloadable files exist and can be opened ''' def items(): - """ yields all combinations of arch, ilp64, is_32bit + """ yields all combinations of arch, ilp64 """ - for arch in arches: - yield arch, None, False - if arch not in ('i686',): - yield arch, '64_', False - if arch in ('windows',): - yield arch, None, True - if arch in ('i686', 'x86_64'): + for plat in plats: + yield plat, None + osname, arch = plat.split("-") + if arch not in ('i686', 'arm64', '32'): + yield plat, '64_' + if osname == "linux" and arch in ('i686', 'x86_64'): oldval = os.environ.get('MB_ML_VER', None) os.environ['MB_ML_VER'] = '1' - yield arch, None, False + yield plat, None # Once we create x86_64 and i686 manylinux2014 wheels... # os.environ['MB_ML_VER'] = '2014' # yield arch, None, False @@ -322,25 +255,23 @@ def items(): os.environ.pop('MB_ML_VER') errs = [] - for arch, ilp64, is_32bit in items(): - if arch == '': - continue - if arch not in arches: + for plat, ilp64 in items(): + osname, _ = plat.split("-") + if plat not in plats: continue target = None try: try: - target = setup_openblas(arch, ilp64, is_32bit) + target = setup_openblas(plat, ilp64) except Exception as e: - print(f'Could not setup {arch} with ilp64 {ilp64}, ' - f'32bit {is_32bit}:') + print(f'Could not setup {plat} with ilp64 {ilp64}, ') print(e) errs.append(e) continue if not target: - raise RuntimeError(f'Could not setup {arch}') + raise RuntimeError(f'Could not setup {plat}') print(target) - if arch == 'windows': + if osname == 'win': if not target.endswith('.a'): raise RuntimeError("Not .a extracted!") else: @@ -391,7 +322,7 @@ def test_version(expected_version, ilp64=get_ilp64()): 'architecture') parser.add_argument('--test', nargs='*', default=None, help='Test different architectures. "all", or any of ' - f'{ARCHITECTURES}') + f'{SUPPORTED_PLATFORMS}') parser.add_argument('--check_version', nargs='?', default='', help='Check provided OpenBLAS version string ' 'against available OpenBLAS') @@ -402,6 +333,6 @@ def test_version(expected_version, ilp64=get_ilp64()): print(setup_openblas()) else: if len(args.test) == 0 or 'all' in args.test: - test_setup(ARCHITECTURES) + test_setup(SUPPORTED_PLATFORMS) else: test_setup(args.test) diff --git a/tools/refguide_check.py b/tools/refguide_check.py index 138e0ece7ed5..9a6d1c9f85b4 100644 --- a/tools/refguide_check.py +++ b/tools/refguide_check.py @@ -19,11 +19,12 @@ Another use of this helper script is to check validity of code samples in docstrings:: - $ python refguide_check.py --doctests ma + $ python tools/refguide_check.py --doctests ma or in RST-based documentations:: - $ python refguide_check.py --rst docs + $ python tools/refguide_check.py --rst doc/source + """ import copy import doctest @@ -165,9 +166,8 @@ def short_path(path, cwd=None): Parameters ---------- - path: str or None - - cwd: str or None + path : str or None + cwd : str or None Returns ------- @@ -304,7 +304,7 @@ def compare(all_dict, others, names, module_name): List of non deprecated sub modules for module_name others : list List of sub modules for module_name - names : set + names : set Set of function names or special directives present in docstring of module_name module_name : ModuleType @@ -343,8 +343,8 @@ def is_deprecated(f): """ Check if module `f` is deprecated - Parameter - --------- + Parameters + ---------- f : ModuleType Returns @@ -779,13 +779,12 @@ def _run_doctests(tests, full_name, verbose, doctest_warnings): Parameters ---------- - tests: list + tests : list full_name : str verbose : bool - - doctest_warning : bool + doctest_warnings : bool Returns ------- diff --git a/tools/swig/numpy.i b/tools/swig/numpy.i index 60930c098330..99ed073abe11 100644 --- a/tools/swig/numpy.i +++ b/tools/swig/numpy.i @@ -2872,158 +2872,6 @@ $result = SWIG_Python_AppendOutput($result,obj); } -/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, - DIM_TYPE* DIM3, DIM_TYPE* DIM4) - */ -%typemap(in,numinputs=0) - (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) - (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) -{ - $1 = &data_temp; - $2 = &dim1_temp; - $3 = &dim2_temp; - $4 = &dim3_temp; - $5 = &dim4_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") - (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3) -{ - npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array || !require_fortran(array)) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, - DATA_TYPE** ARGOUTVIEWM_FARRAY4) - */ -%typemap(in,numinputs=0) - (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_FARRAY4) - (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) -{ - $1 = &dim1_temp; - $2 = &dim2_temp; - $3 = &dim3_temp; - $4 = &dim4_temp; - $5 = &data_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Array_Requirements,NumPy_Utilities") - (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_FARRAY4) -{ - npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array || !require_fortran(array)) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, - DIM_TYPE* DIM3, DIM_TYPE* DIM4) - */ -%typemap(in,numinputs=0) - (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 ) - (DATA_TYPE* data_temp = NULL , DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp) -{ - $1 = &data_temp; - $2 = &dim1_temp; - $3 = &dim2_temp; - $4 = &dim3_temp; - $5 = &dim4_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Utilities") - (DATA_TYPE** ARGOUTVIEWM_ARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) -{ - npy_intp dims[4] = { *$2, *$3, *$4 , *$5 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$1)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$1), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$1), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - -/* Typemap suite for (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, - DATA_TYPE** ARGOUTVIEWM_ARRAY4) - */ -%typemap(in,numinputs=0) - (DIM_TYPE* DIM1 , DIM_TYPE* DIM2 , DIM_TYPE* DIM3 , DIM_TYPE* DIM4 , DATA_TYPE** ARGOUTVIEWM_ARRAY4) - (DIM_TYPE dim1_temp, DIM_TYPE dim2_temp, DIM_TYPE dim3_temp, DIM_TYPE dim4_temp, DATA_TYPE* data_temp = NULL ) -{ - $1 = &dim1_temp; - $2 = &dim2_temp; - $3 = &dim3_temp; - $4 = &dim4_temp; - $5 = &data_temp; -} -%typemap(argout, - fragment="NumPy_Backward_Compatibility,NumPy_Utilities") - (DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4, DATA_TYPE** ARGOUTVIEWM_ARRAY4) -{ - npy_intp dims[4] = { *$1, *$2, *$3 , *$4 }; - PyObject* obj = PyArray_SimpleNewFromData(4, dims, DATA_TYPECODE, (void*)(*$5)); - PyArrayObject* array = (PyArrayObject*) obj; - - if (!array) SWIG_fail; - -%#ifdef SWIGPY_USE_CAPSULE - PyObject* cap = PyCapsule_New((void*)(*$5), SWIGPY_CAPSULE_NAME, free_cap); -%#else - PyObject* cap = PyCObject_FromVoidPtr((void*)(*$5), free); -%#endif - -%#if NPY_API_VERSION < 0x00000007 - PyArray_BASE(array) = cap; -%#else - PyArray_SetBaseObject(array,cap); -%#endif - - $result = SWIG_Python_AppendOutput($result,obj); -} - /* Typemap suite for (DATA_TYPE** ARGOUTVIEWM_FARRAY4, DIM_TYPE* DIM1, DIM_TYPE* DIM2, DIM_TYPE* DIM3, DIM_TYPE* DIM4) */ diff --git a/tools/travis-before-install.sh b/tools/travis-before-install.sh index 81737af89e64..65aa4ad13108 100755 --- a/tools/travis-before-install.sh +++ b/tools/travis-before-install.sh @@ -9,6 +9,7 @@ free -m df -h ulimit -a +sudo apt update sudo apt install gfortran eatmydata libgfortran5 if [ "$USE_DEBUG" ] diff --git a/tools/travis-sorter.py b/tools/travis-sorter.py index c13204f7e018..416f9fe761d0 100755 --- a/tools/travis-sorter.py +++ b/tools/travis-sorter.py @@ -103,7 +103,7 @@ def summarise(jobs): print(" " + "-" * end) -class Job(object): +class Job: def __init__(self, length): global count self.id = count diff --git a/tools/travis-test.sh b/tools/travis-test.sh index e9ddae61a725..4667db991e9a 100755 --- a/tools/travis-test.sh +++ b/tools/travis-test.sh @@ -36,21 +36,33 @@ setup_base() sysflags="$($PYTHON -c "from distutils import sysconfig; \ print (sysconfig.get_config_var('CFLAGS'))")" export CFLAGS="$sysflags $werrors -Wlogical-op -Wno-sign-compare" + # SIMD extensions that need to be tested on both runtime and compile-time via (test_simd.py) + # any specified features will be ignored if they're not supported by compiler or platform + # note: it almost the same default value of --simd-test execpt adding policy `$werror` to treat all + # warnings as errors + simd_test="\$werror BASELINE SSE2 SSE42 XOP FMA4 (FMA3 AVX2) AVX512F AVX512_SKX VSX VSX2 VSX3 NEON ASIMD" # We used to use 'setup.py install' here, but that has the terrible # behaviour that if a copy of the package is already installed in the # install location, then the new copy just gets dropped on top of it. # Travis typically has a stable numpy release pre-installed, and if we # don't remove it, then we can accidentally end up e.g. running old # test modules that were in the stable release but have been removed - # from master. (See gh-2765, gh-2768.) Using 'pip install' also has + # from main. (See gh-2765, gh-2768.) Using 'pip install' also has # the advantage that it tests that numpy is 'pip install' compatible, # see e.g. gh-2766... if [ -z "$USE_DEBUG" ]; then - $PIP install -v . 2>&1 | tee log + # activates '-Werror=undef' when DEBUG isn't enabled since _cffi_backend' + # extension breaks the build due to the following error: + # + # error: "HAVE_FFI_PREP_CIF_VAR" is not defined, evaluates to 0 [-Werror=undef] + # #if !HAVE_FFI_PREP_CIF_VAR && defined(__arm64__) && defined(__APPLE__) + # + export CFLAGS="$CFLAGS -Werror=undef" + $PYTHON setup.py build --simd-test "$simd_test" install 2>&1 | tee log else # The job run with USE_DEBUG=1 on travis needs this. export CFLAGS=$CFLAGS" -Wno-maybe-uninitialized" - $PYTHON setup.py build build_src --verbose-cfg build_ext --inplace 2>&1 | tee log + $PYTHON setup.py build --simd-test "$simd_test" build_src --verbose-cfg build_ext --inplace 2>&1 | tee log fi grep -v "_configtest" log \ | grep -vE "ld returned 1|no files found matching" \ diff --git a/tox.ini b/tox.ini index e58dd6efed57..9bc2bbac36dd 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ [tox] envlist = - py36,py37,py38, + py37,py38,py39, py37-not-relaxed-strides [testenv] diff --git a/versioneer.py b/versioneer.py new file mode 100644 index 000000000000..7a77c5ef7fa1 --- /dev/null +++ b/versioneer.py @@ -0,0 +1,1855 @@ + +# Version: 0.19 + +"""The Versioneer - like a rocketeer, but for versions. + +The Versioneer +============== + +* like a rocketeer, but for versions! +* https://github.com/python-versioneer/python-versioneer +* Brian Warner +* License: Public Domain +* Compatible with: Python 3.6, 3.7, 3.8, 3.9 and pypy3 +* [![Latest Version][pypi-image]][pypi-url] +* [![Build Status][travis-image]][travis-url] + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere in your $PATH +* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) +* run `versioneer install` in your source tree, commit the results +* Verify version information with `python setup.py version` + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes). + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +See [INSTALL.md](./INSTALL.md) for detailed installation instructions. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the + commit date in ISO 8601 format. This will be None if the date is not + available. + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See [details.md](details.md) in the Versioneer +source tree for descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Known Limitations + +Some situations are known to cause problems for Versioneer. This details the +most significant ones. More can be found on Github +[issues page](https://github.com/python-versioneer/python-versioneer/issues). + +### Subprojects + +Versioneer has limited support for source trees in which `setup.py` is not in +the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are +two common reasons why `setup.py` might not be in the root: + +* Source trees which contain multiple subprojects, such as + [Buildbot](https://github.com/buildbot/buildbot), which contains both + "master" and "slave" subprojects, each with their own `setup.py`, + `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI + distributions (and upload multiple independently-installable tarballs). +* Source trees whose main purpose is to contain a C library, but which also + provide bindings to Python (and perhaps other languages) in subdirectories. + +Versioneer will look for `.git` in parent directories, and most operations +should get the right version string. However `pip` and `setuptools` have bugs +and implementation details which frequently cause `pip install .` from a +subproject directory to fail to find a correct version string (so it usually +defaults to `0+unknown`). + +`pip install --editable .` should work correctly. `setup.py install` might +work too. + +Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in +some later version. + +[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking +this issue. The discussion in +[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the +issue from the Versioneer side in more detail. +[pip PR#3176](https://github.com/pypa/pip/pull/3176) and +[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve +pip to let Versioneer work correctly. + +Versioneer-0.16 and earlier only looked for a `.git` directory next to the +`setup.cfg`, so subprojects were completely unsupported with those releases. + +### Editable installs with setuptools <= 18.5 + +`setup.py develop` and `pip install --editable .` allow you to install a +project into a virtualenv once, then continue editing the source code (and +test) without re-installing after every change. + +"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a +convenient way to specify executable scripts that should be installed along +with the python package. + +These both work as expected when using modern setuptools. When using +setuptools-18.5 or earlier, however, certain operations will cause +`pkg_resources.DistributionNotFound` errors when running the entrypoint +script, which must be resolved by re-installing the package. This happens +when the install happens with one version, then the egg_info data is +regenerated while a different version is checked out. Many setup.py commands +cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into +a different virtualenv), so this can be surprising. + +[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes +this one, but upgrading to a newer version of setuptools should probably +resolve it. + + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + +## Similar projects + +* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time + dependency +* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of + versioneer + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg +[pypi-url]: https://pypi.python.org/pypi/versioneer/ +[travis-image]: +https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg +[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer + +""" + +import configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(me)[0]) + vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) + if me_dir != vsr_dir: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.ConfigParser() + with open(setup_cfg, "r") as f: + parser.read_file(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip().decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +LONG_VERSION_PY['git'] = r''' +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Create decorator to mark a method as the handler of a VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %%s" %% (commands,)) + return None, None + stdout = p.communicate()[0].strip().decode() + if p.returncode != 0: + if verbose: + print("unable to run %%s (error)" %% dispcmd) + print("stdout was %%s" %% stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %%s but none started with prefix %%s" %% + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %%d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%%s', no digits" %% ",".join(refs - tags)) + if verbose: + print("likely tags: %%s" %% ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %%s" %% r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %%s not under git control" %% root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty=", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], + cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post0.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post0.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post0.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty=", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + # Use only the last line. Previous lines may contain GPG signature + # information. + date = date.splitlines()[-1] + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-subst keyword substitution. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) + present = False + try: + f = open(".gitattributes", "r") + for line in f.readlines(): + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + f.close() + except EnvironmentError: + pass + if not present: + f = open(".gitattributes", "a+") + f.write("%s export-subst\n" % versionfile_source) + f.close() + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +SHORT_VERSION_PY = """ +# This file was generated by 'versioneer.py' (0.19) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json + +version_json = ''' +%s +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) +""" + + +def versions_from_file(filename): + """Try to determine the version from _version.py if present.""" + try: + with open(filename) as f: + contents = f.read() + except EnvironmentError: + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + + +def write_to_version_file(filename, versions): + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + + print("set %s to '%s'" % (filename, versions["version"])) + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post0.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post0.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post0.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post0.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: + pass + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass + + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version", + "date": None} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(cmdclass=None): + """Get the custom setuptools/distutils subclasses used by Versioneer. + + If the package uses a different cmdclass (e.g. one from numpy), it + should be provide as an argument. + """ + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/python-versioneer/python-versioneer/issues/52 + + cmds = {} if cmdclass is None else cmdclass.copy() + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + print(" date: %s" % vers.get("date")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + # pip install: + # copies source tree to a tempdir before running egg_info/etc + # if .git isn't copied too, 'git describe' will fail + # then does setup.py bdist_wheel, or sometimes setup.py install + # setup.py egg_info -> ? + + # we override different "build_py" commands for both environments + if 'build_py' in cmds: + _build_py = cmds['build_py'] + elif "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "setuptools" in sys.modules: + from setuptools.command.build_ext import build_ext as _build_ext + else: + from distutils.command.build_ext import build_ext as _build_ext + + class cmd_build_ext(_build_ext): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_ext.run(self) + if self.inplace: + # build_ext --inplace will only build extensions in + # build/lib<..> dir with no _version.py to write to. + # As in place builds will already have a _version.py + # in the module dir, we do not need to write one. + return + # now locate _version.py in the new build/ directory and replace + # it with an updated value + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_ext"] = cmd_build_ext + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string + # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. + # setup(console=[{ + # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION + # "product_version": versioneer.get_version(), + # ... + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + if 'py2exe' in sys.modules: # py2exe enabled? + from py2exe.distutils_buildexe import py2exe as _py2exe + + class cmd_py2exe(_py2exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _py2exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["py2exe"] = cmd_py2exe + + # we override different "sdist" commands for both environments + if 'sdist' in cmds: + _sdist = cmds['sdist'] + elif "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" + +INIT_PY_SNIPPET = """ +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions +""" + + +def do_setup(): + """Do main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): + try: + with open(ipy, "r") as f: + old = f.read() + except EnvironmentError: + old = "" + if INIT_PY_SNIPPET not in old: + print(" appending to %s" % ipy) + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) + else: + print(" %s unmodified" % ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") + + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-subst keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1)