diff --git a/.autorelease/autorelease.yaml b/.autorelease/autorelease.yaml new file mode 100644 index 000000000..4f13f1221 --- /dev/null +++ b/.autorelease/autorelease.yaml @@ -0,0 +1,48 @@ +project: + repo_owner: openpathsampling + repo_name: openpathsampling + project_name: OpenPathSampling + +repo: + release-branches: + - stable + release-tag: "v{BASE_VERSION}" + # TODO: this format will not work long-term: need a dev branch per release + # branch + dev-branch: master + +release-check: + versions: + - setup-cfg + - getattr: openpathsampling.version.version + - getattr: openpathsampling.netcdfplus.version.version + +notes: + labels: + - label: feature + heading: New features + - label: enhancement + heading: Other enhancements + - label: experimental + heading: Experimental (beta) features + - label: bugfix + heading: Bugs fixed + - label: misc PR + heading: Miscellaneous improvements + topics: + - label: docs + name: Improvements to documentation + - label: code-style + name: Improvements to code readability and style + - label: internal + name: Behind-the-scenes improvements to CI, testing, and deployment + + allow-duplicates: + - label: experimental + show-as: + - experimental + + standard_contributors: + - dwhswenson + - jhprinz + - sroet diff --git a/.github/workflows/autorelease-default-env.sh b/.github/workflows/autorelease-default-env.sh index 4a09393a9..e6ffd17ea 100644 --- a/.github/workflows/autorelease-default-env.sh +++ b/.github/workflows/autorelease-default-env.sh @@ -1,6 +1,6 @@ -# Vendored from Autorelease 0.5.0 +# Vendored from Autorelease 0.6.1 # Update by updating Autorelease and running `autorelease vendor actions` -INSTALL_AUTORELEASE="python -m pip install autorelease==0.5.0" +INSTALL_AUTORELEASE="python -m pip install autorelease==0.6.1" if [ -f autorelease-env.sh ]; then source autorelease-env.sh fi diff --git a/.github/workflows/autorelease-deploy.yml b/.github/workflows/autorelease-deploy.yml index 73fca082a..44c789bfe 100644 --- a/.github/workflows/autorelease-deploy.yml +++ b/.github/workflows/autorelease-deploy.yml @@ -1,4 +1,4 @@ -# Vendored from Autorelease 0.5.0 +# Vendored from Autorelease 0.6.1 # Update by updating Autorelease and running `autorelease vendor actions` name: "Autorelease Deploy" on: @@ -10,9 +10,11 @@ jobs: if: ${{ github.repository == 'openpathsampling/openpathsampling' }} runs-on: ubuntu-latest name: "Deploy to PyPI" + permissions: + id-token: write steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - run: | # TODO: move this to an action @@ -33,8 +35,6 @@ jobs: python setup.py sdist bdist_wheel twine check dist/* name: "Build and check package" - - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.pypi_password }} + - uses: pypa/gh-action-pypi-publish@release/v1 name: "Deploy to pypi" diff --git a/.github/workflows/autorelease-gh-rel.yml b/.github/workflows/autorelease-gh-rel.yml index 8393d79fc..1e0a743b3 100644 --- a/.github/workflows/autorelease-gh-rel.yml +++ b/.github/workflows/autorelease-gh-rel.yml @@ -1,4 +1,4 @@ -# Vendored from Autorelease 0.5.0 +# Vendored from Autorelease 0.6.1 # Update by updating Autorelease and running `autorelease vendor actions` name: "Autorelease Release" on: @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest name: "Cut release" steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.7" + python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh if [ -f "autorelease-env.sh" ]; then diff --git a/.github/workflows/autorelease-prep.yml b/.github/workflows/autorelease-prep.yml index d68853a03..3328704fb 100644 --- a/.github/workflows/autorelease-prep.yml +++ b/.github/workflows/autorelease-prep.yml @@ -1,4 +1,4 @@ -# Vendored from Autorelease 0.5.0 +# Vendored from Autorelease 0.6.1 # Update by updating Autorelease and running `autorelease vendor actions` name: "Autorelease testpypi" on: @@ -13,12 +13,14 @@ defaults: jobs: deploy_testpypi: + permissions: + id-token: write if: ${{ github.repository == 'openpathsampling/openpathsampling' }} runs-on: ubuntu-latest name: "Deployment test" steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - run: | # TODO: move this to an action @@ -43,9 +45,8 @@ jobs: python setup.py sdist bdist_wheel twine check dist/* name: "Build and check package" - - uses: pypa/gh-action-pypi-publish@master + - uses: pypa/gh-action-pypi-publish@release/v1 with: - password: ${{ secrets.testpypi_password }} repository_url: https://test.pypi.org/legacy/ name: "Deploy to testpypi" test_testpypi: @@ -54,10 +55,10 @@ jobs: name: "Test deployed" needs: deploy_testpypi steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.x" - run: | # TODO: move this to an action source ./.github/workflows/autorelease-default-env.sh if [ -f "autorelease-env.sh" ]; then diff --git a/.github/workflows/check-openmm-rc.yml b/.github/workflows/check-openmm-rc.yml index 1d2c0b176..15aa6fcc5 100644 --- a/.github/workflows/check-openmm-rc.yml +++ b/.github/workflows/check-openmm-rc.yml @@ -16,8 +16,8 @@ jobs: runs-on: ubuntu-latest name: "Check for OpenMM RC" steps: - - uses: actions/checkout@v2 - - uses: dwhswenson/conda-rc-check@v1 + - uses: actions/checkout@v6 + - uses: dwhswenson/conda-rc-check@v2 id: checkrc with: channel: conda-forge diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml index d62d14937..d6aa1fd04 100644 --- a/.github/workflows/deploy_docs.yml +++ b/.github/workflows/deploy_docs.yml @@ -15,24 +15,26 @@ jobs: docs: runs-on: ubuntu-latest name: "docs" + env: + CONDA_PY: "3.12" steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - python-version: "3.11" - miniforge-variant: Mambaforge + python-version: "3.12" + miniforge-version: latest - name: "Install requirements" run: source devtools/conda_install_reqs.sh - - name: "Install OPS" - run: | - python -m pip install --no-deps -e . - python -c "import openpathsampling" - name: "Install doc tools" run: | python -m pip install numpydoc s3cmd conda install -y -c conda-forge --file docs/requirements.txt + - name: "Install OPS" + run: | + python -m pip install --no-deps -e . + python -c "import openpathsampling" - name: "Versions" run: conda list - name: "Build docs" @@ -49,4 +51,3 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - diff --git a/.github/workflows/test-openmm-rc.yml b/.github/workflows/test-openmm-rc.yml index 5e27b7fac..86979f5d2 100644 --- a/.github/workflows/test-openmm-rc.yml +++ b/.github/workflows/test-openmm-rc.yml @@ -16,21 +16,24 @@ jobs: tests: runs-on: ubuntu-latest name: "Tests" + strategy: + matrix: + CONDA_PY: + - "3.11" steps: - uses: actions/checkout@v2 with: fetch-depth: 2 - uses: actions/setup-python@v2 - - uses: conda-incubator/setup-miniconda@v2 - with: + - uses: conda-incubator/setup-miniconda@v3 + with: auto-update-conda: true + python-version: ${{ matrix.CONDA_PY }} + miniforge-version: latest - name: "Install requirements" + env: + CONDA_PY: ${{ matrix.CONDA_PY }} run: | - # we'd rather use the default Python version, but for now need to - # pin to 3.9 (see openpathsampling/openpathsampling#1093) - #CONDA_PY=$(python -c "import sys; print('.'.join(str(s) for s in sys.version_info[:2]))") - export CONDA_PY="3.11" - echo "Python version: ${CONDA_PY}" source devtools/conda_install_reqs.sh - name: "Install OpenMM RC" run: | @@ -38,7 +41,9 @@ jobs: - name: "Install" run: | python -m pip install --no-deps -e . - python -c "import openpathsampling" + - name: "Check installation" + run: | + python -c "import openpathsampling; print(openpathsampling.version.full_version)" - name: "Versions" run: conda list - name: "Unit Tests" @@ -46,7 +51,6 @@ jobs: PY_COLORS: "1" run: py.test -vv -s --cov --cov-report xml - name: "Tests: Experimental" - if: matrix.MINIMAL == '' && matrix.CONDA_PY != '2.7' run: py.test openpathsampling/experimental/ -vv -s - name: "Notebook tests" run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 43d9b1fb4..fa054be1e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,12 +31,12 @@ jobs: # CVs) can differ between minor Python versions. matrix: CONDA_PY: + - "3.13" - "3.12" - "3.11" - - "3.10" MINIMAL: [""] include: - - CONDA_PY: "3.10" + - CONDA_PY: "3.11" MINIMAL: "minimal" steps: @@ -44,11 +44,11 @@ jobs: with: fetch-depth: 2 - uses: actions/setup-python@v2 - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.CONDA_PY }} - miniforge-variant: Mambaforge + miniforge-version: latest - name: "Install requirements" env: MINIMAL: ${{ matrix.MINIMAL }} @@ -66,13 +66,17 @@ jobs: - name: "Install" run: | python -m pip install --no-deps -e . + - name: "Working directory" + run: | + pwd + ls -l - name: "Check installation" run: | python -c "import openpathsampling; print(openpathsampling.version.full_version)" - name: "Versions" run: conda list - name: "Autorelease check" - run: python devtools/autorelease_check.py + run: autorelease check --conf ./.autorelease/autorelease.yaml #- name: "DEBUG: enable SSH login" #uses: mxschmitt/action-tmate@v3 - name: "Unit Tests" diff --git a/.gitignore b/.gitignore index 607e931bb..a560fe5af 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,7 @@ target/ # IPython .ipynb_checkpoints + +# pixi environments +.pixi +*.egg-info diff --git a/devtools/autorelease_check.py b/devtools/autorelease_check.py deleted file mode 100644 index bc9b786da..000000000 --- a/devtools/autorelease_check.py +++ /dev/null @@ -1,29 +0,0 @@ -#/usr/bin/env python -from __future__ import print_function -import setup -import openpathsampling -from autorelease import DefaultCheckRunner, conda_recipe_version -from autorelease.version import get_setup_version -from packaging.version import Version - -repo_path = '.' -SETUP_VERSION = get_setup_version(None, directory='.') -versions = { - 'package': openpathsampling.version.version, - 'netcdfplus': openpathsampling.netcdfplus.version.version, - 'setup.py': SETUP_VERSION, -} - -RELEASE_BRANCHES = ['stable'] -RELEASE_TAG = "v" + Version(SETUP_VERSION).base_version - -if __name__ == "__main__": - checker = DefaultCheckRunner( - versions=versions, - setup=setup, - repo_path='.' - ) - checker.release_branches = RELEASE_BRANCHES + [RELEASE_TAG] - - tests = checker.select_tests() - n_fails = checker.run_as_test(tests) diff --git a/devtools/ci/push-docs-to-s3.py b/devtools/ci/push-docs-to-s3.py index d4fb2269c..87c6789d4 100755 --- a/devtools/ci/push-docs-to-s3.py +++ b/devtools/ci/push-docs-to-s3.py @@ -1,8 +1,8 @@ from __future__ import print_function import os -import pkg_resources import tempfile import subprocess +from importlib import metadata import openpathsampling.version import argparse @@ -23,10 +23,11 @@ # PREFIX = openpathsampling.version.short_version def is_s3cmd_installed(): - dists = pkg_resources.working_set - if not any(d.project_name == 's3cmd' for d in dists): + try: + metadata.distribution('s3cmd') + except metadata.PackageNotFoundError as exc: raise ImportError('The s3cmd package is required. ' - 'try $ pip install s3cmd') + 'try $ pip install s3cmd') from exc def run_cmd(template, config_filename, dry=False): cmd = template.format( @@ -73,4 +74,3 @@ def run_cmd(template, config_filename, dry=False): #config=f.name, #bucket=BUCKET_NAME) #return_val = subprocess.call(cmd.split()) - diff --git a/docs/conf.py b/docs/conf.py index 33df42326..a343f45fd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,13 +13,19 @@ # serve to show the default. import sys -import os import shutil +from contextlib import chdir +from pathlib import Path +from importlib.metadata import PackageNotFoundError, version as get_version # we use these to get the version -import pkg_resources import packaging.version +# Ensure local source tree is imported before any installed package. +DOCS_DIR = Path(__file__).resolve().parent +REPO_ROOT = DOCS_DIR.parent +sys.path.insert(0, str(REPO_ROOT)) + import openpathsampling import sphinx_rtd_theme @@ -36,27 +42,19 @@ def gen_cli_docs(config_file, stdout=False): gen_cli_docs = paths_cli.compiling.gendocs.do_main # If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. +# add these directories to sys.path here. -#sys.path.insert(0, os.path.abspath('.')) -sys.path.insert(0,os.path.abspath('../openpathsampling/')) -#sys.path.append(os.path.abspath('_themes')) +#sys.path.insert(0, str(DOCS_DIR)) +#sys.path.append(str(DOCS_DIR / '_themes')) # -- Preparing the CLI docs ----------------------------------------------- -orig_cwd = os.getcwd() -try: - mydir = os.path.dirname(__file__) - cli_inp_dir = os.path.abspath(os.path.join(mydir, "cli/compile/input/")) - os.chdir(cli_inp_dir) +with chdir(DOCS_DIR / "cli" / "compile" / "input"): gen_cli_docs("categories.yml", stdout=False) -finally: - os.chdir(orig_cwd) # -- Copying examples over into the docs/examples ------------------------- try: - shutil.copytree(os.path.abspath("../examples/alanine_dipeptide_tps"), - os.path.abspath("examples/alanine_dipeptide_tps")) + shutil.copytree(REPO_ROOT / "examples" / "alanine_dipeptide_tps", + DOCS_DIR / "examples" / "alanine_dipeptide_tps") except OSError: pass # there should be a backup here.... @@ -108,7 +106,7 @@ def gen_cli_docs(config_file, stdout=False): autosummary_generate = True autodoc_default_flags = ['members', 'inherited-members', 'imported-members'] -sys.path.insert(0, os.path.abspath('sphinxext')) +sys.path.insert(0, str(DOCS_DIR / 'sphinxext')) extensions.append('notebook_sphinxext') extensions.append('pandoc_sphinxext') @@ -136,7 +134,10 @@ def gen_cli_docs(config_file, stdout=False): # built documents. # # The full version, including alpha/beta/rc tags. -release = pkg_resources.get_distribution('openpathsampling').version +try: + release = get_version('openpathsampling') +except PackageNotFoundError: + release = openpathsampling.version.version # The short X.Y version. # version = packaging.version.Version(release).base_version version = release # prefer to have the .dev0 label on 'latest' diff --git a/docs/developers/engines.rst b/docs/developers/engines.rst index b3312f441..0a92f3a09 100644 --- a/docs/developers/engines.rst +++ b/docs/developers/engines.rst @@ -29,4 +29,5 @@ implement engines that do not only deal with standard molecular dynamics. snapshot_features engines_direct_api engines_indirect_api + engines_trajectory_export advanced_engines diff --git a/docs/developers/engines_trajectory_export.rst b/docs/developers/engines_trajectory_export.rst new file mode 100644 index 000000000..cb6c38d8f --- /dev/null +++ b/docs/developers/engines_trajectory_export.rst @@ -0,0 +1,24 @@ +Trajectory Export from Engines +============================== + +Users frequently want trajectories to be exported from Python into format +that they can analyze with other tools. Since the expected trajectory format +can differ between engines, a developer can set a default trajectory writer +using the :meth:`._default_trajectory_writer` method. This method should +return a callable that takes a trajectory, a filename, and the boolean flag +``force``, where the ``force`` flag forces overwrite of an existing file (a +``FileExistsError`` should be raised if the file exists and ``force`` is +``False``. Note that the default writer should be lossless: output from this +should be sufficient to perform an MD restart for, e.g., a shooting move. + +We recommend creating this callable as a subclass for the +:class:`.TrajectoryWriter` class. To implement this, you only need to +implement the :meth:`.TrajectoryWriter._write` method, which takes the +trajectory and the filename to write to as arguments. Additional information +that can't be extracted from the trajectory can be passed in the class +``__init__`` method. + +If you do not implement :meth:`._default_trajectory_writer`, you will use +the default from the base class, which creates a SimStore file for the +trajectory. OPS should be able to read/write to SimStore for snapshots from +any engine, so this will work, although it may not be convenient to users. diff --git a/examples/ipynbtests.sh b/examples/ipynbtests.sh index 6b307340d..09447f5e4 100755 --- a/examples/ipynbtests.sh +++ b/examples/ipynbtests.sh @@ -42,8 +42,12 @@ case $PYTHON_VERSION in mstis="$dropbox_base_url2/wnqnwh0mmnhvm69h7h0br/toy_mstis_1k_OPS1_py312.nc?rlkey=6l2m6xiphvkpspp0ynix2m6zn&dl=1" mistis="$dropbox_base_url2/tvajjucljm83a2eo6m9ps/toy_mistis_1k_OPS1_py312.nc?rlkey=dk99cj8nlwu1re42drqngf59c&dl=1" ;; + "3.13") + mstis="$dropbox_base_url2/ofr9e8doooev99vmk6394/toy_mstis_1k_OPS1_py313.nc?rlkey=hzwzwxa5u57abv43ibuap5hpo&dl=1" + mistis="$dropbox_base_url2/tyxlixrnaog2rqepb7kt3/toy_mistis_1k_OPS1_py313.nc?rlkey=sv860b9clyja0ccn1nfsg0w07&dl=1" + ;; *) - echo "Unsupported Python version: $PYTHON_VERSION" + echo "Unsupported Python version: $PYTHON_VERSION" && exit 1 esac set -x @@ -55,7 +59,7 @@ set +x ls *nc cd toy_model_mstis/ date -py.test --nbval-lax --current-env -v \ +py.test --nbval-lax --nbval-current-env -v \ toy_mstis_1_setup.ipynb \ toy_mstis_2_run.ipynb \ toy_mstis_3_analysis.ipynb \ @@ -65,14 +69,14 @@ py.test --nbval-lax --current-env -v \ cd ../toy_model_mistis/ date # skip toy_mistis_2_flux: not needed -py.test --nbval-lax --current-env -v \ +py.test --nbval-lax --nbval-current-env -v \ toy_mistis_1_setup_run.ipynb \ toy_mistis_3_analysis.ipynb \ || testfail=1 cd ../tests/ cp ../toy_model_mstis/mstis.nc ./ -py.test --nbval --current-env \ +py.test --nbval --nbval-current-env \ test_openmm_integration.ipynb \ test_snapshot.ipynb \ test_netcdfplus.ipynb \ @@ -81,7 +85,7 @@ py.test --nbval --current-env \ cd ../misc/ cp ../toy_model_mstis/mstis.nc ./ -pytest --nbval-lax --current-env tutorial_storage.ipynb || testfail=1 +pytest --nbval-lax --nbval-current-env tutorial_storage.ipynb || testfail=1 cd .. rm toy_mstis_1k_OPS1.nc diff --git a/examples/tests/test_cv.ipynb b/examples/tests/test_cv.ipynb index 88f1b2747..bab727235 100644 --- a/examples/tests/test_cv.ipynb +++ b/examples/tests/test_cv.ipynb @@ -115,7 +115,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "283860163354172696480456782077640048652\n" + "306465958118793424807025199580056649754\n" ] } ], @@ -141,7 +141,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[(store.attributes[PseudoAttribute] : 4 object(s), 20, 283860163354172696480456782077640048654L), (store.attributes[PseudoAttribute] : 4 object(s), 20, 283860163354172696480456782077640048656L), (store.attributes[PseudoAttribute] : 4 object(s), 20, 283860163354172696480456782077640048658L), (store.attributes[PseudoAttribute] : 4 object(s), 20, 283860163354172696480456782077640048660L)]\n" + "[(store.attributes[PseudoAttribute] : 4 object(s), 20, 306465958118793424807025199580056649756), (store.attributes[PseudoAttribute] : 4 object(s), 20, 306465958118793424807025199580056649758), (store.attributes[PseudoAttribute] : 4 object(s), 20, 306465958118793424807025199580056649760), (store.attributes[PseudoAttribute] : 4 object(s), 20, 306465958118793424807025199580056649762)]\n" ] } ], @@ -159,7 +159,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{283860163354172696480456782077640048656L: 1, 283860163354172696480456782077640048658L: 2, 283860163354172696480456782077640048660L: 3, 283860163354172696480456782077640048654L: 0}\n" + "{306465958118793424807025199580056649756: 0, 306465958118793424807025199580056649758: 1, 306465958118793424807025199580056649760: 2, 306465958118793424807025199580056649762: 3}\n" ] } ], @@ -267,11 +267,12 @@ "output_type": "stream", "text": [ "[10.0, 10.0]\n", - "[3.3921003341674805, 3.3921003341674805]\n" + "[np.float32(3.3921003), np.float32(3.3921003)]\n" ] } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", "print(cv0([template, template]))\n", "print(cv1([template, template]))" ] @@ -285,10 +286,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "\n", - "\n", - "\n", - "\n" + "\n", + "\n", + "\n", + "\n" ] } ], @@ -309,10 +310,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "[u'{\"_cls\":\"CollectiveVariable\",\"_dict\":{\"cv_time_reversible\":false,\"name\":\"func0\"}}'\n", - " u'{\"_cls\":\"FunctionCV\",\"_dict\":{\"name\":\"func1\",\"f\":{\"_marshal\":\"YwMAAAADAAAAAwAAAEMAAQBzGwAAAHwCAGoAAHwAAGoBAGoCAGQBABmDAQB8AQAYUygCAAAATmkAAAAAKAMAAAB0AwAAAHN1bXQLAAAAY29vcmRpbmF0ZXN0BgAAAF92YWx1ZSgDAAAAdAgAAABzbmFwc2hvdHQGAAAAY2VudGVydAIAAABucCgAAAAAKAAAAABzHgAAADxpcHl0aG9uLWlucHV0LTUtNzlhNzEwNDZhYjU1PnQEAAAAZGlzdAIAAABzAgAAAAAB\",\"_module_vars\":[],\"_global_vars\":[]},\"cv_time_reversible\":false,\"cv_wrap_numpy_array\":false,\"cv_requires_lists\":false,\"cv_scalarize_numpy_singletons\":false,\"kwargs\":{\"np\":{\"_import\":\"numpy\"},\"center\":1}}}'\n", - " u'{\"_cls\":\"FunctionCV\",\"_dict\":{\"name\":\"func2\",\"f\":{\"_marshal\":\"YwMAAAADAAAAAwAAAEMAAQBzGwAAAHwCAGoAAHwAAGoBAGoCAGQBABmDAQB8AQAYUygCAAAATmkAAAAAKAMAAAB0AwAAAHN1bXQLAAAAY29vcmRpbmF0ZXN0BgAAAF92YWx1ZSgDAAAAdAgAAABzbmFwc2hvdHQGAAAAY2VudGVydAIAAABucCgAAAAAKAAAAABzHgAAADxpcHl0aG9uLWlucHV0LTUtNzlhNzEwNDZhYjU1PnQEAAAAZGlzdAIAAABzAgAAAAAB\",\"_module_vars\":[],\"_global_vars\":[]},\"cv_time_reversible\":false,\"cv_wrap_numpy_array\":true,\"cv_requires_lists\":false,\"cv_scalarize_numpy_singletons\":false,\"kwargs\":{\"np\":{\"_import\":\"numpy\"},\"center\":1}}}'\n", - " u'{\"_cls\":\"FunctionCV\",\"_dict\":{\"name\":\"func3\",\"f\":{\"_marshal\":\"YwMAAAADAAAAAwAAAEMAAQBzGwAAAHwCAGoAAHwAAGoBAGoCAGQBABmDAQB8AQAYUygCAAAATmkAAAAAKAMAAAB0AwAAAHN1bXQLAAAAY29vcmRpbmF0ZXN0BgAAAF92YWx1ZSgDAAAAdAgAAABzbmFwc2hvdHQGAAAAY2VudGVydAIAAABucCgAAAAAKAAAAABzHgAAADxpcHl0aG9uLWlucHV0LTUtNzlhNzEwNDZhYjU1PnQEAAAAZGlzdAIAAABzAgAAAAAB\",\"_module_vars\":[],\"_global_vars\":[]},\"cv_time_reversible\":true,\"cv_wrap_numpy_array\":true,\"cv_requires_lists\":false,\"cv_scalarize_numpy_singletons\":false,\"kwargs\":{\"np\":{\"_import\":\"numpy\"},\"center\":1}}}']\n" + "['{\"_cls\":\"CollectiveVariable\",\"_dict\":{\"name\":\"func0\",\"cv_time_reversible\":false}}'\n", + " '{\"_cls\":\"FunctionCV\",\"_dict\":{\"name\":\"func1\",\"cv_time_reversible\":false,\"f\":{\"__callable_name__\":\"dist\",\"_dilled\":\"gASVpgEAAAAAAACMCmRpbGwuX2RpbGyUjBBfY3JlYXRlX2Z1bmN0aW9ulJOUKGgAjAxfY3JlYXRl\\\\nX2NvZGWUk5QoQwICAZRLA0sASwBLA0sESwNDUpcAfAKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAfABq\\\\nAQAAAAAAAAAAagIAAAAAAAAAAGQBGQAAAAAAAAAAAKYBAACrAQAAAAAAAAAAfAF6CgAAUwCUTksA\\\\nhpSMA3N1bZSMC2Nvb3JkaW5hdGVzlIwGX3ZhbHVllIeUjAhzbmFwc2hvdJSMBmNlbnRlcpSMAm5w\\\\nlIeUjE4vdmFyL2ZvbGRlcnMvdmovMjhjMTA3NDk2c3E1ejR5MTByano1Z2RtMDAwMGduL1QvaXB5\\\\na2VybmVsXzk2Mjk5LzQxODQ5NTE0NDUucHmUjARkaXN0lGgRSwJDJIAA2AsNjzaKNpAo1BIm1BIt\\\\nqGHUEjDRCzHUCzGwRtELOtAEOpRDAJQpKXSUUpR9lIwIX19uYW1lX1+UjAhfX21haW5fX5RzaBFO\\\\nTnSUUpR9lH2UjA9fX2Fubm90YXRpb25zX1+UfZRzhpRiLg==\\\\n\"},\"cv_requires_lists\":false,\"cv_wrap_numpy_array\":false,\"cv_scalarize_numpy_singletons\":false,\"kwargs\":{\"center\":1,\"np\":{\"_import\":\"numpy\"}}}}'\n", + " '{\"_cls\":\"FunctionCV\",\"_dict\":{\"name\":\"func2\",\"cv_time_reversible\":false,\"f\":{\"__callable_name__\":\"dist\",\"_dilled\":\"gASVpgEAAAAAAACMCmRpbGwuX2RpbGyUjBBfY3JlYXRlX2Z1bmN0aW9ulJOUKGgAjAxfY3JlYXRl\\\\nX2NvZGWUk5QoQwICAZRLA0sASwBLA0sESwNDUpcAfAKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAfABq\\\\nAQAAAAAAAAAAagIAAAAAAAAAAGQBGQAAAAAAAAAAAKYBAACrAQAAAAAAAAAAfAF6CgAAUwCUTksA\\\\nhpSMA3N1bZSMC2Nvb3JkaW5hdGVzlIwGX3ZhbHVllIeUjAhzbmFwc2hvdJSMBmNlbnRlcpSMAm5w\\\\nlIeUjE4vdmFyL2ZvbGRlcnMvdmovMjhjMTA3NDk2c3E1ejR5MTByano1Z2RtMDAwMGduL1QvaXB5\\\\na2VybmVsXzk2Mjk5LzQxODQ5NTE0NDUucHmUjARkaXN0lGgRSwJDJIAA2AsNjzaKNpAo1BIm1BIt\\\\nqGHUEjDRCzHUCzGwRtELOtAEOpRDAJQpKXSUUpR9lIwIX19uYW1lX1+UjAhfX21haW5fX5RzaBFO\\\\nTnSUUpR9lH2UjA9fX2Fubm90YXRpb25zX1+UfZRzhpRiLg==\\\\n\"},\"cv_requires_lists\":false,\"cv_wrap_numpy_array\":true,\"cv_scalarize_numpy_singletons\":false,\"kwargs\":{\"center\":1,\"np\":{\"_import\":\"numpy\"}}}}'\n", + " '{\"_cls\":\"FunctionCV\",\"_dict\":{\"name\":\"func3\",\"cv_time_reversible\":true,\"f\":{\"__callable_name__\":\"dist\",\"_dilled\":\"gASVpgEAAAAAAACMCmRpbGwuX2RpbGyUjBBfY3JlYXRlX2Z1bmN0aW9ulJOUKGgAjAxfY3JlYXRl\\\\nX2NvZGWUk5QoQwICAZRLA0sASwBLA0sESwNDUpcAfAKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAfABq\\\\nAQAAAAAAAAAAagIAAAAAAAAAAGQBGQAAAAAAAAAAAKYBAACrAQAAAAAAAAAAfAF6CgAAUwCUTksA\\\\nhpSMA3N1bZSMC2Nvb3JkaW5hdGVzlIwGX3ZhbHVllIeUjAhzbmFwc2hvdJSMBmNlbnRlcpSMAm5w\\\\nlIeUjE4vdmFyL2ZvbGRlcnMvdmovMjhjMTA3NDk2c3E1ejR5MTByano1Z2RtMDAwMGduL1QvaXB5\\\\na2VybmVsXzk2Mjk5LzQxODQ5NTE0NDUucHmUjARkaXN0lGgRSwJDJIAA2AsNjzaKNpAo1BIm1BIt\\\\nqGHUEjDRCzHUCzGwRtELOtAEOpRDAJQpKXSUUpR9lIwIX19uYW1lX1+UjAhfX21haW5fX5RzaBFO\\\\nTnSUUpR9lH2UjA9fX2Fubm90YXRpb25zX1+UfZRzhpRiLg==\\\\n\"},\"cv_requires_lists\":false,\"cv_wrap_numpy_array\":true,\"cv_scalarize_numpy_singletons\":false,\"kwargs\":{\"center\":1,\"np\":{\"_import\":\"numpy\"}}}}']\n" ] } ], @@ -374,7 +375,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "(store.trajectories[Trajectory] : 1 object(s), 0, 283860163354172696480456782077640048728L)\n" + "(store.trajectories[Trajectory] : 1 object(s), 0, 306465958118793424807025199580056649838)\n" ] } ], @@ -411,7 +412,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "283860163354172696480456782077640048730\n" + "306465958118793424807025199580056649840\n" ] } ], @@ -587,7 +588,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 37, @@ -634,7 +635,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.13.3" }, "toc": { "base_numbering": 1, diff --git a/examples/tests/test_netcdfplus.ipynb b/examples/tests/test_netcdfplus.ipynb index 0e3525edc..4183c8910 100644 --- a/examples/tests/test_netcdfplus.ipynb +++ b/examples/tests/test_netcdfplus.ipynb @@ -810,7 +810,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['{\"_hex_uuid\":\"0xe3d4c53a010011ef9d1f000000000026\",\"_store\":\"nodes\"}'\n", + "['{\"_hex_uuid\":\"0x9e916aba419a11f0bd9a000000000026\",\"_store\":\"nodes\"}'\n", " '{\"Hallo\":2,\"Test\":3}']\n" ] } @@ -969,9 +969,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "[[['e3d4c53a-0100-11ef-9d1f-00000000002c'\n", - " 'e3d4c53a-0100-11ef-9d1f-000000000030']\n", - " ['e3d4c53a-0100-11ef-9d1f-00000000002e' '']]\n", + "[[['9e916aba-419a-11f0-bd9a-00000000002c'\n", + " '9e916aba-419a-11f0-bd9a-000000000030']\n", + " ['9e916aba-419a-11f0-bd9a-00000000002e' '']]\n", "\n", " [['' '']\n", " ['' '']]]\n", @@ -1049,7 +1049,7 @@ "text": [ "Type: \n", "Class: \n", - "Content: {'__uuid__': 302839522207420201522962921396994310194, 'value': 'First'}\n", + "Content: {'__uuid__': 210773071070663257590889269553700798514, 'value': 'First'}\n", "Access: First\n" ] } @@ -1817,7 +1817,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[ {\"_hex_uuid\":\"0xe3d4c53a010011ef9d1f00000000004a\",\"_store\":\"nodesunique\"}, {\"_hex_uuid\":\"0xe3d4c53a010011ef9d1f00000000004c\",\"_store\":\"nodesunique\"}, {\"_hex_uuid\":\"0xe3d4c53a010011ef9d1f00000000004e\",\"_store\":\"nodesunique\"} ]\n" + "[ {\"_hex_uuid\":\"0x9e916aba419a11f0bd9a00000000004a\",\"_store\":\"nodesunique\"}, {\"_hex_uuid\":\"0x9e916aba419a11f0bd9a00000000004c\",\"_store\":\"nodesunique\"}, {\"_hex_uuid\":\"0x9e916aba419a11f0bd9a00000000004e\",\"_store\":\"nodesunique\"} ]\n" ] } ], @@ -1965,11 +1965,12 @@ "output_type": "stream", "text": [ "# We had an exception\n", - "invalid literal for int() with base 10: 'test'\n" + "invalid literal for int() with base 10: np.str_('test')\n" ] } ], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", "try:\n", " a = Node('test')\n", " print(st.varstore.save(a))\n", @@ -2092,7 +2093,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "0xe3d4c53a010011ef9d1f000000000014\n" + "0x9e916aba419a11f0bd9a000000000014\n" ] } ], @@ -2150,7 +2151,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.13.3" }, "toc": { "base_numbering": 1, diff --git a/examples/tests/test_openmm_integration.ipynb b/examples/tests/test_openmm_integration.ipynb index b6a43d34d..f3d1e857a 100644 --- a/examples/tests/test_openmm_integration.ipynb +++ b/examples/tests/test_openmm_integration.ipynb @@ -42,7 +42,27 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:pymbar.timeseries:Warning on use of the timeseries module: If the inherent timescales of the system are long compared to those being analyzed, this statistical inefficiency may be an underestimate. The estimate presumes the use of many statistically independent samples. Tests should be performed to assess whether this condition is satisfied. Be cautious in the interpretation of the data.\n", + "WARNING:pymbar.mbar_solvers:\n", + "****** PyMBAR will use 64-bit JAX! *******\n", + "* JAX is currently set to 32-bit bitsize *\n", + "* which is its default. *\n", + "* *\n", + "* PyMBAR requires 64-bit mode and WILL *\n", + "* enable JAX's 64-bit mode when called. *\n", + "* *\n", + "* This MAY cause problems with other *\n", + "* Uses of JAX in the same code. *\n", + "******************************************\n", + "\n" + ] + } + ], "source": [ "# NBVAL_IGNORE_OUTPUT\n", "# hide stderr from openmmtools (apparently nbval still gets it)\n", @@ -66,6 +86,8 @@ "metadata": {}, "outputs": [], "source": [ + "# NBVAL_IGNORE_OUTPUT\n", + "# apparently we get some weird output from openmmtools on this, too\n", "testsystem = omt.testsystems.AlanineDipeptideVacuum()" ] }, @@ -475,9 +497,9 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "dev", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "dev" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -489,7 +511,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.12" }, "toc": { "base_numbering": 1, diff --git a/openpathsampling/collectivevariable.py b/openpathsampling/collectivevariable.py index f4cbc5ad5..d523dfa67 100644 --- a/openpathsampling/collectivevariable.py +++ b/openpathsampling/collectivevariable.py @@ -57,7 +57,7 @@ def __init__( name, cv_time_reversible=False ): - super(CollectiveVariable, self).__init__(name, paths.BaseSnapshot) + super().__init__(name, paths.BaseSnapshot) self.cv_time_reversible = cv_time_reversible self.diskcache_allow_incomplete = not self.cv_time_reversible @@ -207,7 +207,7 @@ def __init__( ``mdtraj`` """ - super(CallableCV, self).__init__( + super().__init__( name, cv_time_reversible=cv_time_reversible ) @@ -323,7 +323,7 @@ def __init__( """ - super(FunctionCV, self).__init__( + super().__init__( name, cv_callable=f, cv_time_reversible=cv_time_reversible, @@ -374,10 +374,11 @@ def __init__( :class:`openpathsampling.collectivevariable.CallableCV` """ - - super(FunctionCV, self).__init__( + # not sure why this used to skip the parent init; all it does is + # rename anyway + super().__init__( name, - cv_callable=f, + f=f, cv_time_reversible=True, cv_requires_lists=cv_requires_lists, cv_wrap_numpy_array=cv_wrap_numpy_array, diff --git a/openpathsampling/collectivevariables/plumed_wrapper.py b/openpathsampling/collectivevariables/plumed_wrapper.py index 85e700f81..cee69249e 100755 --- a/openpathsampling/collectivevariables/plumed_wrapper.py +++ b/openpathsampling/collectivevariables/plumed_wrapper.py @@ -71,7 +71,7 @@ def __init__(self, kwargs """ - super(PLUMEDCV, self).__init__( + super().__init__( name, f=PLUMEDCV.compute_cv, cv_requires_lists=cv_requires_lists, @@ -185,6 +185,11 @@ def to_dict(self): 'cv_scalarize_numpy_singletons': self.cv_scalarize_numpy_singletons } + @classmethod + def from_dict(cls, dct): + return cls(**dct) + + class PLUMEDInterface(StorableNamedObject): diff --git a/openpathsampling/engines/dynamics_engine.py b/openpathsampling/engines/dynamics_engine.py index e77c123fb..dbff93828 100644 --- a/openpathsampling/engines/dynamics_engine.py +++ b/openpathsampling/engines/dynamics_engine.py @@ -10,6 +10,7 @@ from openpathsampling.netcdfplus import StorableNamedObject from openpathsampling.integration_tools import is_simtk_unit_type +from openpathsampling.exports.trajectories import SimStoreTrajectoryWriter from .snapshot import BaseSnapshot from .trajectory import Trajectory @@ -321,6 +322,31 @@ def set_as_default(self): import openpathsampling as p p.EngineMover.engine = self + def _default_trajectory_writer(self): + """Subclasses should override this to provide a default writer + suitable for their engine. + + By default, we use the :class:`.SimStoreTrajectoryWriter`, because + it should be able to handle any OPS objects. + """ + return SimStoreTrajectoryWriter() + + def export_trajectory(self, trajectory, filename, *, writer=None, + force=False): + """Export a trajectory to a file + + Parameters + ---------- + trajectory : :class:`.Trajectory` + the trajectory to export + filename : str + the name of the file to which to export the trajectory + """ + if writer is None: + writer = self._default_trajectory_writer() + + writer(trajectory, filename, force=force) + @property def default_options(self): default_options = {} diff --git a/openpathsampling/engines/features/base.py b/openpathsampling/engines/features/base.py index 8b74db407..97ed17617 100644 --- a/openpathsampling/engines/features/base.py +++ b/openpathsampling/engines/features/base.py @@ -51,8 +51,8 @@ def _register_function(cls, name, code, __features__): try: source_code = '\n'.join(code) cc = compile(source_code, '', 'exec') - #exec cc in locals() - exec_(cc, locals()) + namespace = {'np': np, 'cls': cls} + exec_(cc, namespace) if name not in cls.__dict__: if hasattr(cls, '__features__') and cls.__features__.debug[name] is None: @@ -61,7 +61,7 @@ def _register_function(cls, name, code, __features__): 'function is overridden again, otherwise some features might not be copied. ' 'The general practise of overriding is not recommended.') % name) - setattr(cls, name, locals()[name]) + setattr(cls, name, namespace[name]) __features__['debug'][name] = source_code diff --git a/openpathsampling/engines/gromacs/engine.py b/openpathsampling/engines/gromacs/engine.py index 614b7d5b1..ad7c8fc9b 100644 --- a/openpathsampling/engines/gromacs/engine.py +++ b/openpathsampling/engines/gromacs/engine.py @@ -20,6 +20,7 @@ from openpathsampling.engines.external_snapshots import \ ExternalMDSnapshot, InternalizedMDSnapshot from openpathsampling.tools import ensure_file +from openpathsampling.exports.trajectories import TRRTrajectoryWriter import os import psutil @@ -186,6 +187,9 @@ def __init__(self, gro, mdp, top, options, base_dir="", prefix="gmx"): super(GromacsEngine, self).__init__(options, descriptor, template, first_frame_in_file=True) + def _default_trajectory_writer(self): + return TRRTrajectoryWriter() + def to_dict(self): dct = super(GromacsEngine, self).to_dict() local_dct = { diff --git a/openpathsampling/engines/openmm/engine.py b/openpathsampling/engines/openmm/engine.py index 63209fb1f..95b747b07 100644 --- a/openpathsampling/engines/openmm/engine.py +++ b/openpathsampling/engines/openmm/engine.py @@ -5,6 +5,7 @@ from openpathsampling.engines import DynamicsEngine, SnapshotDescriptor from openpathsampling.engines.openmm import tools +from openpathsampling.exports.trajectories import TRRTrajectoryWriter from .snapshot import Snapshot import numpy as np @@ -78,7 +79,8 @@ def __init__( system, integrator, openmm_properties=None, - options=None): + options=None, + platform=None): """ Parameters ---------- @@ -94,8 +96,6 @@ def __init__( keys include GPU floating point precision. Note that by default the engine selects the fastest currently available OpenMM platform. - If you want to specify the platform you will have to call - `engine.initialize(platform)` after creating the engine. options : dict a dictionary that provides additional settings for the OPS engine. Allowed are @@ -105,6 +105,11 @@ def __init__( 'n_frames_max' : int, default: 5000, the maximal number of frames allowed for a returned trajectory object + platform : str or :class:`openmm.Platform` or None + optional default platform for creating the OpenMM simulation. + This value is used by ``initialize`` and serialization. The + ``.platform`` property reflects the initialized simulation context + and remains ``None`` until initialization occurs. Notes ----- @@ -137,6 +142,8 @@ def __init__( openmm_properties = {} self.openmm_properties = openmm_properties + self._normalize_platform(platform) # validate serializability + self._platform = platform # set no cached snapshot self._current_snapshot = None @@ -201,7 +208,8 @@ def from_new_options( self.system, integrator, openmm_properties=openmm_properties, - options=new_options) + options=new_options, + platform=self._platform) if self._simulation is not None and \ integrator is self.integrator and \ @@ -219,7 +227,12 @@ def from_new_options( @property def platform(self): """ - str : Return the name of the currently used platform + str or None : Return the name of the currently used platform. + + This value is populated from the simulation context and is ``None`` + until the simulation has been initialized. Configured platform values + may exist before initialization for serialization, but are not exposed + through this property. """ if self._simulation is not None: @@ -227,6 +240,22 @@ def platform(self): else: return None + @staticmethod + def _normalize_platform(platform): + match platform: + case None: + return None + case str() as name: + return name + case candidate if isinstance(candidate, openmm.Platform): + return candidate.getName() + case _: + raise TypeError( + "platform must be None, a platform name string, or an " + "openmm.Platform instance. " + f"Got {type(platform)} instead" + ) + @property def simulation(self): if self._simulation is None: @@ -257,7 +286,10 @@ def unload_context(self): del self._simulation.context self._simulation = None - def initialize(self, platform=None): + def _default_trajectory_writer(self): + return TRRTrajectoryWriter() + + def initialize(self, platform=None, openmm_properties=None): """ Create the final OpenMMEngine @@ -266,6 +298,9 @@ def initialize(self, platform=None): platform : str or :class:`openmm.Platform` or None either a string with a name of the platform or a platform object if None it will default to the fastest currently available platform + openmm_properties : dict or None + optional platform properties for this initialize call. If ``None`` + the engine-stored ``openmm_properties`` are used. Notes ----- @@ -277,37 +312,42 @@ def initialize(self, platform=None): """ if self._simulation is None: - if type(platform) is str: - self._simulation = openmm.app.Simulation( - topology=self.topology.mdtraj.to_openmm(), - system=self.system, - integrator=self.integrator, - platform=openmm.Platform.getPlatformByName(platform), - platformProperties=self.openmm_properties - ) - elif platform is None: - # as of OpenMM 8.1, we can't give an empty props dict when - # platform is None. This will still raise the internal - # OpenMM error is platform is None and properties are - # provided. - openmm_props = self.openmm_properties - if openmm_props == {}: - openmm_props = None - - self._simulation = openmm.app.Simulation( - topology=self.topology.mdtraj.to_openmm(), - system=self.system, - integrator=self.integrator, - platformProperties=openmm_props, - ) + if platform is None: + effective_platform = self._platform else: - self._simulation = openmm.app.Simulation( - topology=self.topology.mdtraj.to_openmm(), - system=self.system, - integrator=self.integrator, - platform=platform, - platformProperties=self.openmm_properties - ) + self._normalize_platform(platform) # validate serializability + effective_platform = platform + + if openmm_properties is None: + effective_properties = self.openmm_properties + else: + effective_properties = openmm_properties + + simulation_kwargs = { + 'topology': self.topology.mdtraj.to_openmm(), + 'system': self.system, + 'integrator': self.integrator, + } + + if effective_platform is None: + if effective_properties: + raise ValueError( + "OpenMM platform-specific properties were provided, " + "but no platform was specified." + ) + simulation_kwargs['platformProperties'] = None + else: + if isinstance(effective_platform, str): + resolved_platform = openmm.Platform.getPlatformByName( + effective_platform + ) + else: + resolved_platform = effective_platform + + simulation_kwargs['platform'] = resolved_platform + simulation_kwargs['platformProperties'] = effective_properties + + self._simulation = openmm.app.Simulation(**simulation_kwargs) logger.info( 'Initialized OpenMM engine using platform `%s`' % @@ -329,7 +369,8 @@ def to_dict(self): 'integrator_xml': integrator_xml, 'topology': self.topology, 'options': self.options, - 'properties': self.openmm_properties + 'properties': self.openmm_properties, + 'platform': self._normalize_platform(self._platform) } @classmethod @@ -339,6 +380,7 @@ def from_dict(cls, dct): topology = dct['topology'] options = dct['options'] properties = dct['properties'] + platform = dct.get('platform', None) # we need to have str as keys properties = {str(key): str(value) @@ -351,7 +393,8 @@ def from_dict(cls, dct): system=openmm.XmlSerializer.deserialize(system_xml), integrator=integrator, options=options, - openmm_properties=properties + openmm_properties=properties, + platform=platform ) @property diff --git a/openpathsampling/experimental/simstore/storable_functions.py b/openpathsampling/experimental/simstore/storable_functions.py index f72a2bd1c..1749e1f40 100644 --- a/openpathsampling/experimental/simstore/storable_functions.py +++ b/openpathsampling/experimental/simstore/storable_functions.py @@ -56,10 +56,12 @@ def _scalarize_singletons(values): """ if isinstance(values, np.ndarray): shape = tuple(n for n in values.shape if n != 1) + # NumPy >= 2.0 rejects float() for non-0D ndarrays, so use item() + # when singleton axes collapse to a scalar. if shape == tuple(): - values = values.__float__() + values = float(values.item()) else: - values.shape = shape + values = np.reshape(values, shape) # shape = values.shape # if len(shape) > 1 and shape[1] == 1: diff --git a/openpathsampling/experimental/simstore/test_storable_function.py b/openpathsampling/experimental/simstore/test_storable_function.py index 78fccb97a..f9650793c 100644 --- a/openpathsampling/experimental/simstore/test_storable_function.py +++ b/openpathsampling/experimental/simstore/test_storable_function.py @@ -65,6 +65,19 @@ def test_scalarize_singletons_to_float(): scalarized = scalarize_singletons(arr) assert not isinstance(scalarized, np.ndarray) assert isinstance(scalarized, float) + assert scalarized == 1.0 + +def test_scalarize_singletons_1d_singleton_to_float(): + scalarized = scalarize_singletons(np.array([1.0])) + assert not isinstance(scalarized, np.ndarray) + assert isinstance(scalarized, float) + assert scalarized == 1.0 + +def test_scalarize_singletons_non_0d_singleton_to_float(): + scalarized = scalarize_singletons(np.array([[1.0]])) + assert not isinstance(scalarized, np.ndarray) + assert isinstance(scalarized, float) + assert scalarized == 1.0 def test_wrap_numpy(): for inp in [1, [1, 2]]: diff --git a/openpathsampling/exports/__init__.py b/openpathsampling/exports/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openpathsampling/exports/steps/symlink_step_exporter.py b/openpathsampling/exports/steps/symlink_step_exporter.py new file mode 100644 index 000000000..d1c976ed9 --- /dev/null +++ b/openpathsampling/exports/steps/symlink_step_exporter.py @@ -0,0 +1,182 @@ +_DEFAULT_RAW_DATA_PATTERN = "raw_data/{sample.trajectory.__uuid__}.{ext}" +_DEFAULT_TRIAL_PATTERN = "{step.mccycle}/trials/{ensemble_id}.{ext}" +_DEFAULT_ACTIVE_PATTERN = "{step.mccycle}/active/{ensemble_id}.{ext}" + +import os +import collections +import pathlib + +class SymLinkStepExporter: + """Export steps as raw data and symlink to the raw data. + + In the patterns used for raw data, trials, and active data, the + following substitutions are available: + + - step: The step being exported. This can be used with, e.g., + step.mccycle to get the Monte Carlo cycle number. + - sample: The sample being exported. + - ensemble_id: The ensemble ID for the sample. This is the name of the + ensemble if it is set, or the UUID of the ensemble if the name is not + set. This behavior can be customized by subclassing this class and + overriding the :meth:`._get_ensemble_id` method. + - ext: The extension for the raw data files. + + These parameters can be further customized by subclassing this class and + overriding the :meth:`._substitution_dict` method. + + Parameters + ---------- + writer : Optional[Callable] + The TrajectoryWriter to use for exporting the raw data. If None, the + default writer for the most common engine in the step will be used. + raw_data_pattern : str + File pattern for the raw data files. + trial_pattern : str + File pattern for the trial data symlinks. + active_pattern : str + File pattern for the active data symlinks. + """ + def __init__( + self, + writer=None, + *, + base_dir='.', + # TODO: add in a base using storage handlers, default to cwd + raw_data_pattern=_DEFAULT_RAW_DATA_PATTERN, + trial_pattern=_DEFAULT_TRIAL_PATTERN, + active_pattern=_DEFAULT_ACTIVE_PATTERN, + ): + self.writer = writer + self.base_dir = pathlib.Path(base_dir) + self.raw_data_pattern = raw_data_pattern + self.trial_pattern = trial_pattern + self.active_pattern = active_pattern + + def _get_ensemble_id(self, sample): + """Get the ensemble ID (used in file names) for a sample. + """ + if sample.ensemble.is_named: + ensemble_id = sample.ensemble.name + else: + ensemble_id = str(sample.ensemble.__uuid__) + + return ensemble_id + + def _get_writer(self, sample): + """Get the TrajectoryWriter to use for a sample. + + If ``self.writer`` is set, it will be used. Otherwise, the most + common engine in the sample's trajectory will be used. + """ + if self.writer is not None: + writer = self.writer + else: + if len(sample.trajectory) == 0: + raise ValueError( + "Cannot determine writer from an empty trajectory" + ) + engines = collections.Counter([s.engine for s in + sample.trajectory]) + engine = engines.most_common(1)[0][0] + writer = engine._default_trajectory_writer() + return writer + + def _substitution_dict(self, step, sample): + writer = self._get_writer(sample) + ensemble_id = self._get_ensemble_id(sample) + return { + "step": step, + "sample": sample, + "ensemble_id": ensemble_id, + "ext": writer.ext, + } + + def _export_sample_symlink(self, pattern, step, sample): + if pattern is None: + return + + subs_dict = self._substitution_dict(step, sample) + path = pattern.format(**subs_dict) + raw_data_path = self.raw_data_pattern.format(**subs_dict) + if not pathlib.Path(raw_data_path).exists(): + self.export_raw_sample(step, sample) + + pathlib.Path(path).parent.mkdir(parents=True, exist_ok=True) + symlink_path = pathlib.Path(path) + target_path = pathlib.Path(raw_data_path) + relative_target = os.path.relpath(target_path, symlink_path.parent) + + if not symlink_path.exists(): + os.symlink(relative_target, path) + + def export_trial_sample(self, step, sample): + """Export a symlink to the raw data for a trial sample. + + Parameters + ---------- + step : Step + The step containing the sample. + sample : Sample + The trial sample to export. + """ + self._export_sample_symlink(self.trial_pattern, step, sample) + + def export_active_sample(self, step, sample): + """Export a symlink to the raw data for an active sample. + + Parameters + ---------- + step : Step + The step containing the sample. + sample : Sample + The active sample to export. + """ + self._export_sample_symlink(self.active_pattern, step, sample) + + def export_raw_sample(self, step, sample): + """Export the raw data for a sample. + + Parameters + ---------- + step : Step + The step containing the sample. + sample : Sample + The sample to export. + """ + subs_dict = self._substitution_dict(step, sample) + raw_data_path = self.raw_data_pattern.format(**subs_dict) + if os.path.exists(raw_data_path): + return + + # ensure parent directory exists + pathlib.Path(raw_data_path).parent.mkdir(parents=True, exist_ok=True) + writer = self._get_writer(sample) + writer(sample.trajectory, raw_data_path) + + def export_step(self, step): + """Export a step. + + Parameters + ---------- + step : Step + The step to export. + """ + for sample in step.change.trials: + self.export_raw_sample(step, sample) + self.export_trial_sample(step, sample) + + for sample in step.active: + self.export_active_sample(step, sample) + + +def export_steps(steps, writer=None, *, export_trials=True, + export_active=True): + trial_pattern = _DEFAULT_TRIAL_PATTERN if export_trials else None + active_pattern = _DEFAULT_ACTIVE_PATTERN if export_active else None + exporter = SymLinkStepExporter( + writer=writer, + trial_pattern=trial_pattern, + active_pattern=active_pattern, + ) + for step in steps: + exporter.export_step(step) diff --git a/openpathsampling/exports/trajectories/__init__.py b/openpathsampling/exports/trajectories/__init__.py new file mode 100644 index 000000000..dd0d3adcd --- /dev/null +++ b/openpathsampling/exports/trajectories/__init__.py @@ -0,0 +1,3 @@ +from .core import TrajectoryWriter, SimStoreTrajectoryWriter +from .trrtrajectorywriter import TRRTrajectoryWriter +from .mdtrajtrajectorywriter import MDTrajTrajectoryWriter diff --git a/openpathsampling/exports/trajectories/core.py b/openpathsampling/exports/trajectories/core.py new file mode 100644 index 000000000..a385e942d --- /dev/null +++ b/openpathsampling/exports/trajectories/core.py @@ -0,0 +1,64 @@ +import pathlib +import contextlib +import logging + +import numpy as np + +_logger = logging.getLogger(__name__) + + +class TrajectoryWriter: + """Base class for tools to write trajectories to extenral files. + + This is essentially a wrapper for a function to write the trajectory to + a file. The function should take two arguments: the trajectory to write, + and the filename to write to. + + We use an object-oriented approach here so that initialization can + inlude arbitrary parameters, but the interface used by other code is + consistent. Additionally, :meth:`__call__` is can handle some standard + error checking. + """ + def __call__(self, trajectory, filename, force=False): + if not force and pathlib.Path(filename).exists(): + raise FileExistsError(f"File {filename} already exists") + + self._write(trajectory, filename) + + @property + def ext(self): + """The file extension used by this writer.""" + raise NotImplementedError() + + def _write(self, trajectory, filename): + """Write the trajectory to the file. + + Parameters + ---------- + trajectory : openpathsampling.Trajectory + the trajectory to save + filename : str + the name of the file to save to + """ + raise NotImplementedError() + +class SimStoreTrajectoryWriter(TrajectoryWriter): + """Trajectory writer that uses the OpenPathSampling storage format. + + This is the default trajectory writer, since all engines should be able + to use it. + """ + @property + def ext(self): + return "db" + + def _write(self, trajectory, filename): + from openpathsampling.experimental.storage import Storage + from openpathsampling.experimental.storage.monkey_patches import ( + _IS_PATCHED_SAVING + ) + if not _IS_PATCHED_SAVING: + raise RuntimeError("SimStoreTrajectoryWriter requires the " + "monkey-patch to be active") + storage = Storage(filename, mode='w') + storage.save(trajectory) diff --git a/openpathsampling/exports/trajectories/mdtrajtrajectorywriter.py b/openpathsampling/exports/trajectories/mdtrajtrajectorywriter.py new file mode 100644 index 000000000..026716b17 --- /dev/null +++ b/openpathsampling/exports/trajectories/mdtrajtrajectorywriter.py @@ -0,0 +1,40 @@ +from .core import TrajectoryWriter +from openpathsampling.integration_tools import HAS_MDTRAJ +from collections.abc import Iterable + + +class MDTrajTrajectoryWriter(TrajectoryWriter): + """Generic save to MDTraj. + + Note that this will not include velocities, and therefore isn't suitable + for saving data that could be used in a restart. + """ + def __init__(self, ext, mdtraj_selection=None): + self._ext = ext + if not HAS_MDTRAJ: # -no-cov- + raise ImportError("MDTraj is not available") + + self.mdtraj_selection = mdtraj_selection + + @property + def ext(self): + return self._ext + + def _write(self, trajectory, filename): + mdt = trajectory.to_mdtraj() + + sel = None + if isinstance(self.mdtraj_selection, str): + sel = mdt.topology.select(self.mdtraj_selection) + elif isinstance(self.mdtraj_selection, Iterable): + sel = list(self.mdtraj_selection) + elif self.mdtraj_selection is None: + pass + else: + raise TypeError("mdtraj_selection must be a string or an " + f"iterable; got '{self.mdtraj_selection}'") + + if sel is not None: + mdt = mdt.atom_slice(sel) + + mdt.save(filename) diff --git a/openpathsampling/exports/trajectories/trrtrajectorywriter.py b/openpathsampling/exports/trajectories/trrtrajectorywriter.py new file mode 100644 index 000000000..5b3164b75 --- /dev/null +++ b/openpathsampling/exports/trajectories/trrtrajectorywriter.py @@ -0,0 +1,30 @@ +from openpathsampling.integration_tools import HAS_MDTRAJ +import numpy as np +from .core import TrajectoryWriter + +class TRRTrajectoryWriter(TrajectoryWriter): + """Write a trajectory to a Gromacs TRR file. + + This inludes velocities, so it can be used for restarts. + """ + def __init__(self): + if not HAS_MDTRAJ: # -no-cov- + raise ImportError("MDTraj is not available") + + @property + def ext(self): + return "trr" + + def _write(self, trajectory, filename): + # this uses some "unofficial" MDTraj API + import mdtraj as md + nframes = len(trajectory) + xyz = np.asarray(trajectory.xyz, dtype=np.float32) + time = np.asarray([0.0]*nframes, dtype=np.float32) + box = np.asarray(trajectory.box_vectors, dtype=np.float32) + lambd = np.asarray([0.0]*nframes, dtype=np.float32) + vel = np.asarray(trajectory.velocities, dtype=np.float32) + trr = md.formats.TRRTrajectoryFile(str(filename), 'w') + step = np.arange(0, nframes, dtype=np.int32) + trr._write(xyz, time, step, box, lambd, vel) + trr.close() diff --git a/openpathsampling/netcdfplus/version.py b/openpathsampling/netcdfplus/version.py index 2342e1b7c..952a2d0ae 100644 --- a/openpathsampling/netcdfplus/version.py +++ b/openpathsampling/netcdfplus/version.py @@ -1,4 +1,4 @@ -short_version = '1.7.0' +short_version = '1.7.1.dev0' version = short_version full_version = short_version git_revision = 'alpha' diff --git a/openpathsampling/numerics/resampling_statistics.py b/openpathsampling/numerics/resampling_statistics.py index 115a196f1..9583c3e3d 100644 --- a/openpathsampling/numerics/resampling_statistics.py +++ b/openpathsampling/numerics/resampling_statistics.py @@ -66,7 +66,7 @@ def std_df(objects, mean_x=None): n_obj = float(len(objects)) sq = [o**2 for o in objects] variance = mean_df(sq) - mean_x**2 - return variance.applymap(np.sqrt) + return variance.apply(lambda s: s.map(np.sqrt)) class ResamplingStatistics(object): """ diff --git a/openpathsampling/numerics/wham.py b/openpathsampling/numerics/wham.py index cb439f7f9..f08d310ed 100755 --- a/openpathsampling/numerics/wham.py +++ b/openpathsampling/numerics/wham.py @@ -188,10 +188,7 @@ def unweighting_tis(self, cleaned_df): pandas.DataFrame unweighting values for the input dataframe """ - unweighting = cleaned_df.copy().applymap( - lambda x: 1.0 if x > 0.0 else 0.0 - ) - return unweighting + return cleaned_df.gt(0.0).astype(float) def sum_k_Hk_Q(self, cleaned_df): r"""Sum over histograms for each histogram bin. Length is n_bins diff --git a/openpathsampling/tests/exports/steps/conftest.py b/openpathsampling/tests/exports/steps/conftest.py new file mode 100644 index 000000000..e6b4fa953 --- /dev/null +++ b/openpathsampling/tests/exports/steps/conftest.py @@ -0,0 +1,5 @@ +# Import fixtures from analysis conftest +from openpathsampling.tests.analysis.conftest import default_unidirectional_tis + +# Make the fixture available in this module +__all__ = ['default_unidirectional_tis'] \ No newline at end of file diff --git a/openpathsampling/tests/exports/steps/test_symlink_step_exporter.py b/openpathsampling/tests/exports/steps/test_symlink_step_exporter.py new file mode 100644 index 000000000..281f04945 --- /dev/null +++ b/openpathsampling/tests/exports/steps/test_symlink_step_exporter.py @@ -0,0 +1,369 @@ +import pytest +import pathlib +import contextlib +import openpathsampling as paths +from unittest.mock import Mock + +from openpathsampling.exports.steps.symlink_step_exporter import ( + SymLinkStepExporter, + _DEFAULT_TRIAL_PATTERN, _DEFAULT_ACTIVE_PATTERN, _DEFAULT_RAW_DATA_PATTERN, + export_steps, +) + +from openpathsampling.tests.analysis.utils.mock_movers import ( + MockForwardShooting, MockRepex, + MockPathReversal, run_moves, +) + +from openpathsampling.tests.test_helpers import make_1d_traj + +@pytest.fixture +def shooting_step(default_unidirectional_tis): + scheme = default_unidirectional_tis.scheme + traj = default_unidirectional_tis.make_tis_trajectory(5) + init_conds = scheme.initial_conditions_from_trajectories(traj) + ensemble = scheme.network.sampling_ensembles[0] + + # Create a shooting move that will be accepted + partial_traj = default_unidirectional_tis.make_trajectory(-1, 2).reversed + move = MockForwardShooting( + shooting_index=2, + partial_traj=partial_traj, + scheme=scheme, + ensemble=ensemble, + accepted=True + ) + + steplist = list(run_moves(init_conds, [move])) + assert len(steplist) == 1 + step = steplist[0] + assert ensemble(step.change.trials[0]) # acceptance is allowed + return step + +@pytest.fixture +def repex_step(default_unidirectional_tis): + scheme = default_unidirectional_tis.scheme + t1 = default_unidirectional_tis.make_tis_trajectory(4) + t2 = default_unidirectional_tis.make_tis_trajectory(10) + init_conds = scheme.initial_conditions_from_trajectories([t1, t2]) + + e1, e2 = scheme.network.sampling_ensembles[:2] + ensembles = [e1, e2] + + move = MockRepex(scheme, ensembles) + + steplist = list(run_moves(init_conds, [move])) + assert len(steplist) == 1 + step = steplist[0] + return step + +@pytest.fixture +def pathreversal_step(default_unidirectional_tis): + scheme = default_unidirectional_tis.scheme + traj = default_unidirectional_tis.make_tis_trajectory(6) + init_conds = scheme.initial_conditions_from_trajectories(traj) + + move = MockPathReversal(scheme) + + steplist = list(run_moves(init_conds, [move])) + assert len(steplist) == 1 + step = steplist[0] + return step + +@pytest.fixture +def all_steps(shooting_step, repex_step, pathreversal_step): + return [shooting_step, repex_step, pathreversal_step] + +@pytest.fixture +def unnamed_ensemble(): + cv = paths.FunctionCV("x", lambda s: s.xyz[0][0]) + ensemble = paths.CVDefinedVolume(cv, -1.0, 1.0) + return ensemble + +@pytest.fixture +def named_ensemble(): + cv = paths.FunctionCV("x", lambda s: s.xyz[0][0]) + ensemble = paths.CVDefinedVolume(cv, -1.0, 1.0).named("EnsembleName") + return ensemble + + +class TestSymLinkStepExporter: + def setup_method(self): + """Setup method to create exporter instance for tests.""" + self.mock_writer = Mock() + self.mock_writer.ext = "dat" + def mock_write_func(trajectory, filename): + pathlib.Path(filename).parent.mkdir(parents=True, exist_ok=True) + pathlib.Path(filename).touch() + self.mock_writer.side_effect = mock_write_func + self.exporter = SymLinkStepExporter(writer=self.mock_writer) + + @pytest.mark.parametrize("source", ["name", "uuid"]) + def test_get_ensemble_id(self, source, request): + fixture = {'name': 'named_ensemble', + 'uuid': 'unnamed_ensemble'}[source] + ensemble = request.getfixturevalue(fixture) + expected = {'name': 'EnsembleName', + 'uuid': str(ensemble.__uuid__)}[source] + + sample = Mock() + sample.ensemble = ensemble + assert self.exporter._get_ensemble_id(sample) == expected + + @pytest.mark.parametrize("source", ["obj", "most_common"]) + def test_get_writer(self, source): + if source == "obj": + mock_writer = Mock() + mock_writer.ext = "dat" + exporter = SymLinkStepExporter(writer=mock_writer) + sample = Mock() + sample.trajectory = [] + + result = exporter._get_writer(sample) + assert result is mock_writer + + elif source == "most_common": + exporter = SymLinkStepExporter(writer=None) + + traj1 = make_1d_traj([1.0]) + traj2 = make_1d_traj([2.0, 3.0]) + + engine1 = traj1[0].engine + engine2 = traj2[0].engine + assert engine1 is not engine2 + + combined_traj = traj1 + traj2 + + sample = Mock() + sample.trajectory = combined_traj + + result = exporter._get_writer(sample) + default_writer = engine2._default_trajectory_writer() + assert result.__class__ is default_writer.__class__ + + def test_get_writer_empty_trajectory_error(self): + exporter = SymLinkStepExporter(writer=None) + sample = Mock() + sample.trajectory = [] + + with pytest.raises( + ValueError, + match="Cannot determine writer from an empty trajectory" + ): + exporter._get_writer(sample) + + def test_substitution_dict(self, shooting_step): + exporter = SymLinkStepExporter(writer=self.mock_writer) + + step = shooting_step + sample = step.change.trials[0] + + subs = exporter._substitution_dict(step, sample) + + assert subs["step"] is step + assert subs["sample"] is sample + assert subs["ensemble_id"] == exporter._get_ensemble_id(sample) + assert subs["ext"] == self.mock_writer.ext + + def test_export_trial_sample(self, shooting_step, tmp_path): + step = shooting_step + sample = step.change.trials[0] + + exporter = SymLinkStepExporter( + base_dir=tmp_path, writer=self.mock_writer + ) + assert tmp_path.exists() + assert tmp_path.is_dir() + assert len(list(tmp_path.iterdir())) == 0 + + with contextlib.chdir(tmp_path): + exporter.export_trial_sample(step, sample) + + subs_dict = exporter._substitution_dict(step, sample) + raw_data_path = exporter.raw_data_pattern.format(**subs_dict) + trial_path = exporter.trial_pattern.format(**subs_dict) + assert pathlib.Path(raw_data_path).exists() + + assert pathlib.Path(trial_path).exists() + assert pathlib.Path(trial_path).is_symlink() + + assert pathlib.Path(raw_data_path).samefile(trial_path) + + def test_export_active_sample(self, shooting_step, tmp_path): + step = shooting_step + sample = step.active[0] + + exporter = SymLinkStepExporter( + base_dir=tmp_path, writer=self.mock_writer + ) + + with contextlib.chdir(tmp_path): + exporter.export_active_sample(step, sample) + + subs_dict = exporter._substitution_dict(step, sample) + raw_data_path = exporter.raw_data_pattern.format(**subs_dict) + assert pathlib.Path(raw_data_path).exists() + + active_path = exporter.active_pattern.format(**subs_dict) + assert pathlib.Path(active_path).exists() + assert pathlib.Path(active_path).is_symlink() + + assert pathlib.Path(raw_data_path).samefile(active_path) + + def test_export_raw_sample(self, shooting_step, tmp_path): + step = shooting_step + sample = step.active[0] + + exporter = SymLinkStepExporter( + base_dir=tmp_path, writer=self.mock_writer + ) + + with contextlib.chdir(tmp_path): + exporter.export_raw_sample(step, sample) + + subs_dict = exporter._substitution_dict(step, sample) + raw_data_path = exporter.raw_data_pattern.format(**subs_dict) + assert pathlib.Path(raw_data_path).exists() + + assert pathlib.Path(raw_data_path).is_file() + assert not pathlib.Path(raw_data_path).is_symlink() + + @pytest.mark.parametrize("step_type", ["shooting", "repex", + "pathreversal"]) + def test_export_step(self, step_type, request, tmp_path): + expected_trials_by_type = { + "shooting": 1, + "repex": 2, + "pathreversal": 1, + } + expected_trials = expected_trials_by_type[step_type] + + step = request.getfixturevalue(f"{step_type}_step") + + unique_trajectories = { + s.trajectory for s in step.change.trials + list(step.active) + } + + actual_trials = len(step.change.trials) + expected_active = len(step.active) + + exporter = SymLinkStepExporter( + base_dir=tmp_path, writer=self.mock_writer + ) + + with contextlib.chdir(tmp_path): + assert actual_trials == expected_trials + + exporter.export_step(step) + + raw_dir = pathlib.Path("raw_data") + assert raw_dir.exists() + raw_files = list(raw_dir.glob(f"*.{self.mock_writer.ext}")) + assert len(raw_files) == len(unique_trajectories) + + trials_dir = pathlib.Path(f"{step.mccycle}/trials") + assert trials_dir.exists() + trial_files = list(trials_dir.glob(f"*.{self.mock_writer.ext}")) + assert len(trial_files) == actual_trials + + for trial_file in trial_files: + assert trial_file.is_symlink() + + active_dir = pathlib.Path(f"{step.mccycle}/active") + assert active_dir.exists() + active_files = list( + active_dir.glob(f"*.{self.mock_writer.ext}") + ) + + assert len(active_files) == expected_active + + for active_file in active_files: + assert active_file.is_symlink() + + +def test_export_steps(all_steps, tmp_path): + mock_writer = Mock() + mock_writer.ext = "db" + def mock_write_func(trajectory, filename): + pathlib.Path(filename).parent.mkdir(parents=True, exist_ok=True) + pathlib.Path(filename).touch() + mock_writer.side_effect = mock_write_func + + with contextlib.chdir(tmp_path): + export_steps(all_steps, writer=mock_writer) + + assert pathlib.Path("raw_data").exists() + raw_files = list(pathlib.Path("raw_data").glob(f"*.{mock_writer.ext}")) + assert len(raw_files) > 0 + + for step in all_steps: + step_dir = pathlib.Path(str(step.mccycle)) + assert step_dir.exists() + + trials_dir = step_dir / "trials" + assert trials_dir.exists() + + active_dir = step_dir / "active" + assert active_dir.exists() + + writer_db2 = Mock() + writer_db2.ext = "db2" + writer_db2.side_effect = mock_write_func + + export_steps(all_steps, writer=writer_db2, export_trials=False) + + for step in all_steps: + trials_dir = pathlib.Path(str(step.mccycle)) / "trials" + assert trials_dir.exists() + trial_files = list(trials_dir.glob(f"*.{writer_db2.ext}")) + assert len(trial_files) == 0 + + writer_db3 = Mock() + writer_db3.ext = "db3" + writer_db3.side_effect = mock_write_func + + export_steps(all_steps, writer=writer_db3, export_active=False) + + for step in all_steps: + active_dir = pathlib.Path(str(step.mccycle)) / "active" + assert active_dir.exists() + active_files = list(active_dir.glob(f"*.{writer_db3.ext}")) + assert len(active_files) == 0 + + +def test_default_raw_pattern_paths(shooting_step): + writer = Mock() + writer.ext = "dat" + exporter = SymLinkStepExporter(writer=writer) + + sample = shooting_step.change.trials[0] + subs = exporter._substitution_dict(shooting_step, sample) + + expected = f"raw_data/{sample.trajectory.__uuid__}.{writer.ext}" + assert _DEFAULT_RAW_DATA_PATTERN.format(**subs) == expected + + +def test_default_trial_pattern_paths(shooting_step): + writer = Mock() + writer.ext = "dat" + exporter = SymLinkStepExporter(writer=writer) + + sample = shooting_step.change.trials[0] + subs = exporter._substitution_dict(shooting_step, sample) + ensemble_id = exporter._get_ensemble_id(sample) + + expected = f"{shooting_step.mccycle}/trials/{ensemble_id}.{writer.ext}" + assert _DEFAULT_TRIAL_PATTERN.format(**subs) == expected + + +def test_default_active_pattern_paths(shooting_step): + writer = Mock() + writer.ext = "dat" + exporter = SymLinkStepExporter(writer=writer) + + sample = shooting_step.active[0] + subs = exporter._substitution_dict(shooting_step, sample) + ensemble_id = exporter._get_ensemble_id(sample) + + expected = f"{shooting_step.mccycle}/active/{ensemble_id}.{writer.ext}" + assert _DEFAULT_ACTIVE_PATTERN.format(**subs) == expected diff --git a/openpathsampling/tests/exports/trajectories/conftest.py b/openpathsampling/tests/exports/trajectories/conftest.py new file mode 100644 index 000000000..f855fad12 --- /dev/null +++ b/openpathsampling/tests/exports/trajectories/conftest.py @@ -0,0 +1,37 @@ +import pytest +import pathlib +from openpathsampling.tests.test_helpers import data_filename +import openpathsampling as paths + +@pytest.fixture +def ad_trajpath(): + test_dir = pathlib.Path(data_filename("gromacs_engine")) + trajfile = test_dir / "project_trr/0000000.trr" + return trajfile + +@pytest.fixture +def ad_grofile(): + test_dir = pathlib.Path(data_filename("gromacs_engine")) + topfile = test_dir / "conf.gro" + return str(topfile) + + +@pytest.fixture +def ad_trajectory(ad_trajpath): + engine = paths.engines.gromacs.Engine( + gro="conf.gro", + mdp="md.mdp", + top="topol.top", + options = { + 'mdrun_args': '-nt 1', + 'grompp_args': '-maxwarn 2', + }, + base_dir=data_filename("gromacs_engine"), + prefix="project" + ) + + traj = paths.Trajectory([ + engine.read_frame_from_file(str(ad_trajpath), i) + for i in [0, 1, 2] + ]) + return traj diff --git a/openpathsampling/tests/exports/trajectories/test_core.py b/openpathsampling/tests/exports/trajectories/test_core.py new file mode 100644 index 000000000..7ec1d0e5d --- /dev/null +++ b/openpathsampling/tests/exports/trajectories/test_core.py @@ -0,0 +1,108 @@ +from openpathsampling.exports.trajectories import * + +from openpathsampling.tests.test_helpers import ( + data_filename, make_1d_traj +) +import openpathsampling as paths + +import pytest +import pathlib + + +@pytest.fixture +def toy_trajectory(): + return make_1d_traj([1.0, 2.0, 3.0]) + + +class TrajectoryWriterTestBase: + def _read_trajectory(self, filename): + raise NotImplementedError() + + def _test_trajectory(self, trajectory, outfile): + assert not outfile.exists() + self.writer(trajectory, outfile) + assert outfile.exists() + + traj = self._read_trajectory(outfile) + assert len(traj) == len(trajectory) + assert traj == trajectory + + def _test_call_outfile_exists(self, trajectory, outfile): + outfile.touch() + with pytest.raises(FileExistsError): + self.writer(trajectory, outfile) + + def _test_call_outfile_exists_force(self, trajectory, outfile): + outfile.touch() + assert outfile.exists() + assert len(outfile.read_bytes()) == 0 + self.writer(trajectory, outfile, force=True) + assert outfile.exists() + assert len(outfile.read_bytes()) > 0 + + def test_call(self, trajectory_fixture, request, tmp_path): + raise NotImplementedError() + + def test_call_outfile_exists(self, request, tmp_path): + raise NotImplementedError() + + def test_call_outfile_exists_force(self, request, tmp_path): + raise NotImplementedError() + + +class TestSimStoreTrajectoryWriter(TrajectoryWriterTestBase): + def setup_method(self): + self.writer = SimStoreTrajectoryWriter() + + def _read_trajectory(self, filename): + from openpathsampling.experimental.storage import Storage + storage = Storage(filename, mode='r') + return storage.trajectories[0] + + @pytest.mark.parametrize("trajectory_fixture", [ + "ad_trajectory", + "toy_trajectory", + ]) + def test_call(self, trajectory_fixture, request, tmp_path): + # monkey patch for SimStore + trajectory = request.getfixturevalue(trajectory_fixture) + import openpathsampling as paths + from openpathsampling.experimental.storage import monkey_patches + paths = monkey_patches.monkey_patch_all(paths) + + try: + outfile = tmp_path / f"{trajectory_fixture}.nc" + self._test_trajectory(trajectory, outfile) + finally: + # undo the monkey patch + monkey_patches.unpatch(paths) + + def test_call_outfile_exists(self, request, tmp_path): + trajectory = request.getfixturevalue("ad_trajectory") + import openpathsampling as paths + from openpathsampling.experimental.storage import monkey_patches + paths = monkey_patches.monkey_patch_all(paths) + + try: + self._test_call_outfile_exists(trajectory, tmp_path / "test.db") + finally: + monkey_patches.unpatch(paths) + + def test_call_outfile_exists_force(self, request, tmp_path): + trajectory = request.getfixturevalue("ad_trajectory") + import openpathsampling as paths + from openpathsampling.experimental.storage import monkey_patches + paths = monkey_patches.monkey_patch_all(paths) + try: + self._test_call_outfile_exists_force(trajectory, + tmp_path / "test.db") + finally: + monkey_patches.unpatch(paths) + + def test_call_not_patched_fail(self, request, tmp_path): + trajectory = request.getfixturevalue("ad_trajectory") + with pytest.raises(RuntimeError, match="monkey-patch"): + self.writer(trajectory, tmp_path / "test.db") + + def test_ext(self): + assert self.writer.ext == "db" diff --git a/openpathsampling/tests/exports/trajectories/test_mdtrajtrajectorywriter.py b/openpathsampling/tests/exports/trajectories/test_mdtrajtrajectorywriter.py new file mode 100644 index 000000000..d0f8b90ea --- /dev/null +++ b/openpathsampling/tests/exports/trajectories/test_mdtrajtrajectorywriter.py @@ -0,0 +1,71 @@ +import pytest + +from openpathsampling.exports.trajectories.mdtrajtrajectorywriter import * +from openpathsampling.integration_tools import HAS_MDTRAJ +import numpy.testing as npt + +def test_mdtraj_trajectory_writer(ad_trajectory, ad_grofile, tmp_path): + if not HAS_MDTRAJ: + pytest.skip("mdtraj is not available") + + import mdtraj as md + outfile = tmp_path / "test.xtc" + writer = MDTrajTrajectoryWriter(ext="xtc") + assert not outfile.exists() + writer(ad_trajectory, outfile) + assert outfile.exists() + + reloaded = md.load(str(outfile), top=ad_grofile) + assert len(reloaded) == len(ad_trajectory) + assert reloaded.xyz.shape == ad_trajectory.xyz.shape + npt.assert_allclose(reloaded.xyz, ad_trajectory.xyz, atol=1e-3) + + +@pytest.mark.parametrize("selection", [ + "backbone", + [4, 5, 6, 8, 14, 15, 16, 18] +]) +def test_subtrajectory_selection(ad_trajectory, selection, tmp_path): + if not HAS_MDTRAJ: + pytest.skip("mdtraj is not available") + + import mdtraj as md + outfile = tmp_path / "test.xtc" + writer = MDTrajTrajectoryWriter(ext="xtc", mdtraj_selection=selection) + assert not outfile.exists() + writer(ad_trajectory, outfile) + assert outfile.exists() + + # save file for the modified topology + mdt = ad_trajectory.to_mdtraj() + if isinstance(selection, str): + subset = mdt.topology.select(selection) + else: + subset = selection + + subtraj = mdt.atom_slice(subset) + subgro = tmp_path / "subset.gro" + subtraj[0].save(str(subgro)) + + reloaded = md.load(str(outfile), top=subgro) + assert len(reloaded) == len(ad_trajectory) + assert reloaded.xyz.shape == subtraj.xyz.shape + assert reloaded.xyz.shape != ad_trajectory.xyz.shape + npt.assert_allclose(reloaded.xyz, subtraj.xyz, atol=1e-3) + + +def test_mdtraj_trajectory_writer_selection_error(ad_trajectory, tmp_path): + if not HAS_MDTRAJ: + pytest.skip("mdtraj is not available") + + writer = MDTrajTrajectoryWriter(ext="xtc", mdtraj_selection=object()) + with pytest.raises(TypeError): + writer(ad_trajectory, tmp_path / "test.xtc") + + +def test_mdtraj_writer_ext(): + if not HAS_MDTRAJ: + pytest.skip("mdtraj is not available") + + writer = MDTrajTrajectoryWriter(ext="xtc") + assert writer.ext == "xtc" diff --git a/openpathsampling/tests/exports/trajectories/test_trrtrajectorywriter.py b/openpathsampling/tests/exports/trajectories/test_trrtrajectorywriter.py new file mode 100644 index 000000000..48a1d378a --- /dev/null +++ b/openpathsampling/tests/exports/trajectories/test_trrtrajectorywriter.py @@ -0,0 +1,38 @@ +import pytest + +from openpathsampling.exports.trajectories.trrtrajectorywriter import * +from openpathsampling.integration_tools import HAS_MDTRAJ +import numpy.testing as npt + +def test_trr_trajectory_writer(ad_trajectory, tmp_path): + if not HAS_MDTRAJ: + pytest.skip("mdtraj is not available") + + import mdtraj as md + outfile = tmp_path / "test.trr" + writer = TRRTrajectoryWriter() + assert not outfile.exists() + writer(ad_trajectory, outfile) + assert outfile.exists() + + trr = md.formats.TRRTrajectoryFile(str(outfile), mode='r') + xyz, time, step, box, lambd, vel, forces = trr._read( + int(len(ad_trajectory)), + atom_indices=None, + get_velocities=True, + ) + + assert forces is None + npt.assert_allclose(xyz, ad_trajectory.xyz) + npt.assert_allclose(vel, ad_trajectory.velocities) + npt.assert_allclose(box, ad_trajectory.box_vectors) + npt.assert_allclose(lambd, np.zeros(len(ad_trajectory))) + npt.assert_allclose(time, np.zeros(len(ad_trajectory))) + + +def test_trr_writer_ext(): + if not HAS_MDTRAJ: + pytest.skip("mdtraj is not available") + + writer = TRRTrajectoryWriter() + assert writer.ext == "trr" diff --git a/openpathsampling/tests/test_dynamicsengine.py b/openpathsampling/tests/test_dynamicsengine.py index cf762ab21..0b8b136a8 100644 --- a/openpathsampling/tests/test_dynamicsengine.py +++ b/openpathsampling/tests/test_dynamicsengine.py @@ -107,3 +107,97 @@ def test_generate_multiple_running_conditions(self, stoppable): # doesn't work traj = self.stupid.generate(init_snap, conditions) assert len(traj) == 2 + + def test_export_trajectory(self, tmp_path): + global paths + outfile = tmp_path / "test_traj.db" + init_snap = make_1d_traj([0.0])[0] + continue_condition = paths.LengthEnsemble(5).can_append + traj = self.stupid.generate(init_snap, [continue_condition]) + assert len(traj) == 5 + assert not outfile.exists() + from openpathsampling.experimental.storage import ( + monkey_patches, Storage + ) + paths = monkey_patches.monkey_patch_all(paths) + try: + self.stupid.export_trajectory(traj, outfile) + assert outfile.exists() + storage = Storage(outfile, mode='r') + assert len(storage.trajectories) == 1 + reloaded = storage.trajectories[0] + assert len(reloaded) == 5 + assert reloaded == traj + finally: + monkey_patches.unpatch(paths) + + + def test_export_trajectory_force(self, tmp_path): + global paths + outfile = tmp_path / "test_traj.db" + outfile.touch() + init_snap = make_1d_traj([0.0])[0] + continue_condition = paths.LengthEnsemble(5).can_append + traj = self.stupid.generate(init_snap, [continue_condition]) + assert len(traj) == 5 + assert outfile.exists() + from openpathsampling.experimental.storage import ( + monkey_patches, Storage + ) + paths = monkey_patches.monkey_patch_all(paths) + + try: + self.stupid.export_trajectory(traj, outfile, force=True) + assert outfile.exists() + storage = Storage(outfile, mode='r') + assert len(storage.trajectories) == 1 + reloaded = storage.trajectories[0] + assert len(reloaded) == 5 + assert reloaded == traj + finally: + monkey_patches.unpatch(paths) + + def test_export_trajecory_file_exists_fail(self, tmp_path): + global paths + outfile = tmp_path / "test_traj.db" + outfile.touch() + init_snap = make_1d_traj([0.0])[0] + continue_condition = paths.LengthEnsemble(5).can_append + traj = self.stupid.generate(init_snap, [continue_condition]) + assert len(traj) == 5 + assert outfile.exists() + from openpathsampling.experimental.storage import ( + monkey_patches, Storage + ) + paths = monkey_patches.monkey_patch_all(paths) + + try: + with pytest.raises(FileExistsError): + self.stupid.export_trajectory(traj, outfile) + finally: + monkey_patches.unpatch(paths) + + def test_export_trajectory_custom_writer(self, tmp_path): + # here we use a custom writer that will write to a string + from openpathsampling.exports.trajectories.core import ( + TrajectoryWriter + ) + outfile = tmp_path / "test_traj.txt" + class StringXWriter(TrajectoryWriter): + def _write(self, trajectory, filename): + outstring = " ".join([ + f"{snap.xyz[0][0]:.2f}" for snap in trajectory + ]) + with open(filename, 'w') as f: + f.write(outstring) + + init_snap = make_1d_traj([0.0])[0] + continue_condition = paths.LengthEnsemble(5).can_append + traj = self.stupid.generate(init_snap, [continue_condition]) + assert len(traj) == 5 + writer = StringXWriter() + self.stupid.export_trajectory(traj, outfile, writer=writer) + with open(outfile, 'r') as f: + outstring = f.read() + + assert outstring == ("0.00 " * 5)[:-1] diff --git a/openpathsampling/tests/test_external_engine.py b/openpathsampling/tests/test_external_engine.py index 8f498ccbe..7e3edf84f 100644 --- a/openpathsampling/tests/test_external_engine.py +++ b/openpathsampling/tests/test_external_engine.py @@ -138,6 +138,7 @@ def setup_method(self): self.ensemble = paths.LengthEnsemble(5) def test_deprecation(self): + NEW_DEFAULT_FILENAME_SETTER.has_warned = False slow_options = {'n_frames_max': 10000, 'engine_sleep': 100, 'name_prefix': "test", diff --git a/openpathsampling/tests/test_gromacs_engine.py b/openpathsampling/tests/test_gromacs_engine.py index a1282278a..616cf351f 100644 --- a/openpathsampling/tests/test_gromacs_engine.py +++ b/openpathsampling/tests/test_gromacs_engine.py @@ -318,6 +318,29 @@ def test_serialization_cycle(self): reserialized = deserialized.to_dict() assert serialized == reserialized + def test_export_trajectory(self, tmp_path): + if not has_gmx: + pytest.skip("gmx not found. Skipping test.") + + if not HAS_MDTRAJ: + pytest.skip("MDTraj not found. Skipping test.") + + traj_0 = self.engine.trajectory_filename(0) + snap = self.engine.read_frame_from_file(traj_0, 0) + self.engine.filename_setter.reset(0) + + ens = paths.LengthEnsemble(5) + traj = self.engine.generate(snap, running=[ens.can_append]) + + filename = tmp_path / "test.trr" + assert not filename.exists() + self.engine.export_trajectory(traj, filename) + assert filename.exists() + + traj_md = md.load(str(filename), top=self.engine.gro) + assert len(traj_md) == 5 + npt.assert_allclose(traj_md.xyz, traj.xyz) + class TestGromacsExternalMDSnapshot(object): def setup_method(self): diff --git a/openpathsampling/tests/test_helpers.py b/openpathsampling/tests/test_helpers.py index 3ed2ed199..29819f92e 100644 --- a/openpathsampling/tests/test_helpers.py +++ b/openpathsampling/tests/test_helpers.py @@ -5,8 +5,8 @@ @author David W.H. Swenson """ -import os from functools import wraps +from importlib.resources import files from openpathsampling.engines import NoEngine import numpy as np import numpy.testing as npt @@ -19,8 +19,6 @@ except ImportError: md = None -from pkg_resources import resource_filename - import openpathsampling as paths import openpathsampling.engines.openmm as peng import openpathsampling.engines.toy as toys @@ -178,8 +176,7 @@ def prepend_exception_message(e, failmsg): e.args = tuple([arg0] + list(e.args[1:])) def data_filename(fname, subdir='test_data'): - return resource_filename('openpathsampling', - os.path.join('tests', subdir, fname)) + return str(files('openpathsampling.tests').joinpath(subdir, fname)) def true_func(value, *args, **kwargs): return True diff --git a/openpathsampling/tests/test_openmm_engine.py b/openpathsampling/tests/test_openmm_engine.py index 177594779..76ec39ed8 100644 --- a/openpathsampling/tests/test_openmm_engine.py +++ b/openpathsampling/tests/test_openmm_engine.py @@ -20,6 +20,8 @@ import openpathsampling.engines.openmm as peng import openpathsampling.engines as dyn +import numpy.testing as npt + import openpathsampling as paths from .test_helpers import ( @@ -81,6 +83,13 @@ def setup_method(self): ) integrator.setConstraintTolerance(0.00001) + uninitialized_integrator = mm.LangevinIntegrator( + 300*u.kelvin, + old_div(1.0, u.picoseconds), + 2.0*u.femtoseconds + ) + uninitialized_integrator.setConstraintTolerance(0.00001) + # Engine options options = { 'n_steps_per_frame': 2, @@ -94,6 +103,13 @@ def setup_method(self): options=options ) + self.uninitialized_engine = peng.Engine( + template.topology, + system, + uninitialized_integrator, + options=options + ) + self.engine.initialize('CPU') context = self.engine.simulation.context zero_array = np.zeros((template.topology.n_atoms, 3)) @@ -277,3 +293,106 @@ def test_has_constraints(self, has_constraints): integrator=omt.integrators.VVVRIntegrator() ) assert engine.has_constraints() == has_constraints + + def test_export_trajectory(self, tmp_path): + filename = tmp_path / "test.trr" + assert not filename.exists() + traj = self.engine.generate(self.engine.current_snapshot, [ + paths.LengthEnsemble(3).can_append + ]) + self.engine.export_trajectory(traj, filename) + assert filename.exists() + + # reload the trajectory with MDTraj; check positions only + import mdtraj as md + topfile = data_filename("ala_small_traj.pdb") + reloaded = md.load(str(filename), top=str(topfile)) + assert len(reloaded) == len(traj) + npt.assert_allclose(reloaded.xyz, traj.xyz) + + def test_serialization_cycle(self): + integrator = mm.LangevinIntegrator( + 300*u.kelvin, + old_div(1.0, u.picoseconds), + 2.0*u.femtoseconds + ) + integrator.setConstraintTolerance(0.00001) + engine = peng.Engine( + template.topology, + system, + integrator, + options={'n_steps_per_frame': 2, 'n_frames_max': 5}, + platform='CPU' + ) + + dct = engine.to_dict() + rebuilt = peng.Engine.from_dict(dct) + dct2 = rebuilt.to_dict() + assert dct == dct2 + assert dct2['platform'] == 'CPU' + + def test_initialize_uses_exact_platform_object(self): + platform_obj = mm.Platform.getPlatformByName('CPU') + self.uninitialized_engine.initialize(platform=platform_obj) + + assert self.uninitialized_engine.platform == 'CPU' + assert self.uninitialized_engine._platform is None + assert self.uninitialized_engine.to_dict()['platform'] is None + + def test_duck_typed_platform_like_object_raises_typeerror(self): + class PlatformLikeObject(object): + def getName(self): + return 'CPU' + + integrator = mm.LangevinIntegrator( + 300*u.kelvin, + old_div(1.0, u.picoseconds), + 2.0*u.femtoseconds + ) + integrator.setConstraintTolerance(0.00001) + + with pytest.raises(TypeError, match=r'platform must be None.*PlatformLikeObject'): + peng.Engine( + template.topology, + system, + integrator, + options={'n_steps_per_frame': 2, 'n_frames_max': 5}, + platform=PlatformLikeObject() + ) + + def test_from_dict_without_platform_key_is_backward_compatible(self): + serialized = self.engine.to_dict() + del serialized['platform'] + + restored = peng.Engine.from_dict(serialized) + assert restored is not None + assert restored.platform is None + + def test_initialize_uses_stored_properties_when_override_is_none( + self): + self.uninitialized_engine.openmm_properties = {'__ops_bad__': '1'} + with pytest.raises(Exception, match='Illegal property name'): + self.uninitialized_engine.initialize( + platform='CPU', + openmm_properties=None + ) + + def test_initialize_explicit_properties_override_stored(self): + self.uninitialized_engine.openmm_properties = {'__ops_bad__': '1'} + self.uninitialized_engine.initialize( + platform='CPU', + openmm_properties={} + ) + assert isinstance(self.uninitialized_engine.simulation, mm.app.Simulation) + assert self.uninitialized_engine.platform == 'CPU' + + def test_initialize_nonempty_properties_without_platform_raises(self): + self.uninitialized_engine.openmm_properties = {'Threads': '1'} + with pytest.raises(ValueError, match='no platform was specified'): + self.uninitialized_engine.initialize() + + def test_initialize_empty_properties_without_platform_is_valid(self): + self.uninitialized_engine.openmm_properties = {} + self.uninitialized_engine.initialize() + assert isinstance(self.uninitialized_engine.simulation, mm.app.Simulation) + assert self.uninitialized_engine.platform is not None diff --git a/openpathsampling/tests/test_pathmover.py b/openpathsampling/tests/test_pathmover.py index afda5fcbd..389c95fe5 100644 --- a/openpathsampling/tests/test_pathmover.py +++ b/openpathsampling/tests/test_pathmover.py @@ -14,7 +14,6 @@ import pytest import openpathsampling as paths -from openpathsampling.collectivevariable import FunctionCV from openpathsampling.engines.trajectory import Trajectory from openpathsampling.ensemble import LengthEnsemble from openpathsampling.pathmover import * @@ -128,7 +127,7 @@ class TestShootingMover(object): def setup_method(self): self.dyn = CalvinistDynamics([-0.1, 0.1, 0.3, 0.5, 0.7, -0.1, 0.2, 0.4, 0.6, 0.8]) - op = FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) + op = paths.FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) self.stateA = CVDefinedVolume(op, -100, 0.0) self.stateB = CVDefinedVolume(op, 0.65, 100) self.tps = A2BEnsemble(self.stateA, self.stateB) @@ -509,7 +508,7 @@ def test_to_dict_from_dict(self): class TestPathReversalMover(object): def setup_method(self): - op = FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) + op = paths.FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) volA = CVDefinedVolume(op, -100, 0.0) volB = CVDefinedVolume(op, 1.0, 100) @@ -576,7 +575,7 @@ def test_replica_not_in_sample_set(self): class TestReplicaExchangeMover(object): def setup_method(self): - op = FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) + op = paths.FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) state1 = CVDefinedVolume(op, -100, 0.0) state2 = CVDefinedVolume(op, 1, 100) @@ -724,7 +723,7 @@ def setup_method(self): ]) self.dyn.initialized = True # SampleMover.engine = self.dyn - op = FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) + op = paths.FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) stateA = CVDefinedVolume(op, -100, 0.0) stateB = CVDefinedVolume(op, 0.65, 100) volX = CVDefinedVolume(op, -100, 0.25) @@ -1176,7 +1175,7 @@ def test_move(self): class TestMinusMover(object): def setup_method(self): - op = FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) + op = paths.FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) volA = CVDefinedVolume(op, -100, 0.0) volB = CVDefinedVolume(op, 1.0, 100) volX = CVDefinedVolume(op, -100, 0.25) @@ -1410,7 +1409,7 @@ def test_extension_fails(self): class TestSingleReplicaMinusMover(object): def setup_method(self): - op = FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) + op = paths.FunctionCV("myid", f=lambda snap: snap.coordinates[0][0]) volA = CVDefinedVolume(op, -100, 0.0) volB = CVDefinedVolume(op, 1.0, 100) volX = CVDefinedVolume(op, -100, 0.25) diff --git a/openpathsampling/tests/test_resampling_statistics.py b/openpathsampling/tests/test_resampling_statistics.py index 8bac9f588..778b13d33 100644 --- a/openpathsampling/tests/test_resampling_statistics.py +++ b/openpathsampling/tests/test_resampling_statistics.py @@ -36,6 +36,16 @@ def test_std(self): ) assert_frame_equal(std_df(self.inputs), expected_std) + def test_std_object_dtype(self): + from openpathsampling.numerics.resampling_statistics import std_df + object_inputs = [df.astype(object) for df in self.inputs] + expected_std = pd.DataFrame( + [[0.17677669529663689, 0.17677669529663689], + [0.10606601717798207, 0.17677669529663689]], + columns=['A', 'B'], index=['A', 'B'] + ) + assert_frame_equal(std_df(object_inputs), expected_std) + def test_initialization(self): stats = paths.numerics.ResamplingStatistics( function=lambda x: x, diff --git a/openpathsampling/tests/test_volume.py b/openpathsampling/tests/test_volume.py index 1fc687796..1e74b901b 100644 --- a/openpathsampling/tests/test_volume.py +++ b/openpathsampling/tests/test_volume.py @@ -224,6 +224,14 @@ def test_get_cv_float(self, inp): expected = inp in ['float', 'array1'] assert isinstance(val, float) is expected + @pytest.mark.filterwarnings("ignore:The CV 'cv' returns an iterable") + def test_get_cv_float_array_raises(self): + snap = make_1d_traj([0.0])[0] + volume = self._vol_for_cv_type('array') + with pytest.raises(TypeError, + match="only 0-dimensional arrays can be converted"): + _ = volume._get_cv_float(snap) + class TestCVRangeVolumePeriodic(object): def setup_method(self): diff --git a/openpathsampling/tests/utils/test_storage_interfaces.py b/openpathsampling/tests/utils/test_storage_interfaces.py new file mode 100644 index 000000000..9d351eea8 --- /dev/null +++ b/openpathsampling/tests/utils/test_storage_interfaces.py @@ -0,0 +1,193 @@ +from openpathsampling.utils.storage_interfaces import * +import pytest +import tempfile + +class StorageInterfaceTest: + def setup_method(self): + self.tmpdir_manager = tempfile.TemporaryDirectory() + self.tmpdir = pathlib.Path(self.tmpdir_manager.__enter__()) + self.localdir = self.tmpdir / "local" + self.localdir.mkdir() + + # unstored local file + self.localfile = self.localdir / "localfile" + with open(self.localfile, mode='w') as f: + f.write("localfile contents") + + self._initialize_interface() + + def _initialize_interface(self): + """Subclasses must implement this initialization routine. + + This method must create the following stored objects: + + * key ``prestored``, contents ``prestored contents`` + * key ``nested/nest_prestored``, contents ``nested prestored + contents`` + * key ``nested/deeply/prestored``, contents ``deeply nested`` + """ + raise NotImplementedError() + + def teardown_method(self): + self.tmpdir_manager.__exit__(None, None, None) + + def test_store(self): + raise NotImplementedError() + + def test_delete(self): + raise NotImplementedError() + + def test_delete_directory(self): + raise NotImplementedError() + + def test_load(self): + target_file = self.localdir / "foo" + assert not target_file.exists() + self.interface.load("prestored", target_file) + assert target_file.exists() + with open(target_file, mode='r') as f: + assert f.read() == "prestored contents" + + def test_transfer(self): + raise NotImplementedError() + + def test_transfer_directory(self): + raise NotImplementedError() + + def test_list_directory(self): + expected = {"nested/nest_prestored", "nested/deeply/prestored"} + assert set(self.interface.list_directory("nested")) == expected + + def test_contains(self): + assert "prestored" in self.interface + assert "nested/nest_prestored" in self.interface + assert "nested" not in self.interface + assert "nonexistent" not in self.interface + + +class TestLocalFileStorageInterface(StorageInterfaceTest): + def _initialize_interface(self): + root = self.tmpdir / "stored" + self.interface = LocalFileStorageInterface(root) + # pre-stored file + with open(root / "prestored", mode='w') as f: + f.write("prestored contents") + + # pre-stored nested files (for directory testing) + nested_prestored = root / "nested/nest_prestored" + nested_prestored.parent.mkdir() + with open(nested_prestored, mode='w') as f: + f.write("nested prestored contents") + + # deeply nested file + deeply_prestored = root / "nested/deeply/prestored" + deeply_prestored.parent.mkdir(parents=True) + with open(deeply_prestored, mode='w') as f: + f.write("deeply nested") + + def test_store(self): + stored_file = self.interface.root / "foo" + assert not stored_file.exists() + self.interface.store("foo", self.localfile) + assert stored_file.exists() + assert self.localfile.exists() + with open(stored_file, mode='r') as f: + assert f.read() == "localfile contents" + + def test_delete(self): + assert (self.interface.root / 'prestored').exists() + assert 'prestored' in self.interface + self.interface.delete('prestored') + assert "prestored" not in self.interface + assert not (self.interface.root / 'prestored').exists() + + def test_delete_directory(self): + nested_file = "nested/nest_prestored" + assert (self.interface.root / nested_file).exists() + assert nested_file in self.interface + with pytest.raises(ValueError, match="is a directory"): + self.interface.delete('nested') + + def test_transfer(self): + stored_target = self.interface.root / "foo" + assert not stored_target.exists() + self.interface.transfer("foo", self.localfile) + assert stored_target.exists() + assert not self.localfile.exists() + with open(stored_target, mode='r') as f: + assert f.read() == "localfile contents" + + def test_transfer_directory(self): + source_dir = self.localdir / "directory" + source_dir.mkdir(exist_ok=True, parents=True) + subfile = source_dir / "file" + with open(subfile, mode='w') as f: + f.write("directory/file contents") + + assert "directory/file" not in self.interface + assert subfile.exists() + + with pytest.raises(ValueError, match="is a directory"): + self.interface.transfer("directory", source_dir) + + def test_list_directory_not_directory(self): + with pytest.raises(ValueError, match="is not a directory"): + self.interface.list_directory("prestored") + + +class TestMemoryStorageInterface(StorageInterfaceTest): + def _initialize_interface(self): + self.interface = MemoryStorageInterface() + data = { + "prestored": b"prestored contents", + "nested/nest_prestored": b"nested prestored contents", + "nested/deeply/prestored": b"deeply nested", + } + self.interface._data = data + + def test_store(self): + stored_target = "foo" + assert stored_target not in self.interface._data + self.interface.store(stored_target, self.localfile) + assert stored_target in self.interface._data + assert self.localfile.exists() + assert self.interface._data[stored_target] == b"localfile contents" + + def test_transfer(self): + stored_target = "foo" + assert stored_target not in self.interface._data + self.interface.transfer(stored_target, self.localfile) + assert stored_target in self.interface._data + assert not self.localfile.exists() + assert self.interface._data[stored_target] == b"localfile contents" + + def test_delete(self): + assert "prestored" in self.interface._data + self.interface.delete("prestored") + assert "prestored" not in self.interface._data + + def test_delete_directory(self): + with pytest.raises(KeyError): + self.interface.delete("nested") + + def test_transfer_directory(self): + source_dir = self.localdir / "directory" + source_dir.mkdir(exist_ok=True, parents=True) + subfile = source_dir / "file" + with open(subfile, mode='w') as f: + f.write("directory/file contents") + + assert "directory/file" not in self.interface._data + assert subfile.exists() + + with pytest.raises(ValueError, match="is a directory"): + self.interface.transfer("directory", source_dir) + + def test_list_root_directory(self): + expected = { + "prestored", + "nested/nest_prestored", + "nested/deeply/prestored" + } + root = pathlib.Path("") + assert set(self.interface.list_directory(root)) == expected diff --git a/openpathsampling/utils/storage_interfaces.py b/openpathsampling/utils/storage_interfaces.py new file mode 100644 index 000000000..3870925be --- /dev/null +++ b/openpathsampling/utils/storage_interfaces.py @@ -0,0 +1,159 @@ +from abc import ABC, abstractmethod +import os +import pathlib +import shutil + +import logging +_logger = logging.getLogger(__name__) + + + +class StorageInterface(ABC): + """Abstract treatment of a key-value-like file/object store. + + This is generally assuming file-based semantics. This may typically mean + putting things into a temporary directory. This is particularly focused + on checkpointing, where we will copy the data to put it in a zip file. + """ + @abstractmethod + def store(self, storage_label, source_path): + """Store the data in ``source_path`` at key ``storage_label`` + + Parameters + ---------- + storage_label : str + The key to store the data at. + source_path : os.PathLike + The path to the data to store. + """ + raise NotImplementedError() + + @abstractmethod + def load(self, storage_label, target_path): + """Load the data from ``storage_label`` into file at ``target_path`` + + Parameters + ---------- + storage_label : str + The key to load the data from. + target_path : os.PathLike + The path to store the data at. + """ + raise NotImplementedError() + + @abstractmethod + def delete(self, storage_label): + """Delete key ``storage_label`` from the object store. + """ + raise NotImplementedError() + + @abstractmethod + def __contains__(self, storage_label): + raise NotImplementedError() + + def transfer(self, storage_label, source_path): + """Transfer a file to the storage label from the source path. + + In some cases, this can be made faster than store followed by + os.remove, so this method can be overridden. (Example: moving on a + file system is faster than copying.) + """ + if pathlib.Path(source_path).is_dir(): + raise ValueError(f"'{source_path}' is a directory, and can't " + "be transferred.") + self.store(storage_label, source_path) + os.remove(source_path) + + @abstractmethod + def list_directory(self, storage_label): + """List all objects in subdirectories of the given storage label. + """ + raise NotImplementedError() + + +class LocalFileStorageInterface(StorageInterface): + """Concrete implementation of StorageInterface for local files. + + Parameters + ---------- + root : os.PathLike + The root directory for the storage interface. + """ + def __init__(self, root): + self.root = pathlib.Path(root) + self.root.mkdir(parents=True, exist_ok=True) + + def store(self, storage_label, source_path): + local_path = self.root / storage_label + local_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source_path, self.root / storage_label) + + def load(self, storage_label, target_path): + target_path.parent.mkdir(parents=True, exist_ok=True) + local_path = self.root / storage_label + _logger.debug("Copying file from {str(local_path)} " + f"to {str(target_path)}") + shutil.copyfile(local_path, target_path) + + def delete(self, storage_label): + obj = self.root / storage_label + if obj.is_dir(): + raise ValueError(f"'{obj}' is a directory, and can't be " + "deleted.") + else: + _logger.debug("Deleting file {str(obj)}") + os.remove(obj) + + def __contains__(self, storage_label): + expected = self.root / storage_label + return expected.exists() and not expected.is_dir() + + def transfer(self, storage_label, source_path): + if pathlib.Path(source_path).is_dir(): + raise ValueError(f"'{source_path}' is a directory, and can't " + "be transferred.") + shutil.move(source_path, self.root / storage_label) + + def list_directory(self, storage_label): + path = self.root / storage_label + if not path.is_dir(): + raise ValueError(f"'{path}' is not a directory.") + return [ + str((pathlib.Path(p[0]) / subp).relative_to(self.root)) + for p in os.walk(path) + for subp in p[2] + ] + + +class MemoryStorageInterface(StorageInterface): + """In-memory storage interface. + + Useful in testing. + """ + def __init__(self): + self._data = {} + + def store(self, storage_label, source_path): + with open(source_path, mode='rb') as f: + self._data[storage_label] = f.read() + + def load(self, storage_label, target_path): + with open(target_path, mode='wb') as f: + f.write(self._data[storage_label]) + return self._data[storage_label] + + def delete(self, storage_label): + del self._data[storage_label] + + def __contains__(self, storage_label): + return storage_label in self._data + + def list_directory(self, storage_label): + # special case because the empty path becomes '.' as a string + if storage_label == pathlib.Path(""): + storage_label = "" + elif not storage_label.endswith("/"): + storage_label += "/" + + return [key for key in self._data + if str(key).startswith(str(storage_label))] diff --git a/openpathsampling/volume.py b/openpathsampling/volume.py index 2c8d3f888..ac58f354e 100644 --- a/openpathsampling/volume.py +++ b/openpathsampling/volume.py @@ -418,6 +418,17 @@ def _get_cv_float(self, snapshot): val = self.collectivevariable(snapshot) if self._cv_returns_iterable is None: self._cv_returns_iterable = self._is_iterable(val) + + # NumPy >= 2.0 (e.g., 2.4.2 in CI) no longer allows __float__ for + # non-0D ndarrays, so we explicitly scalarize only size-1 arrays here. + if isinstance(val, np.ndarray): + if val.size == 1: + return float(val.item()) + + raise TypeError( + "only 0-dimensional arrays can be converted to Python scalars" + ) + return val.__float__() def __call__(self, snapshot): diff --git a/setup.cfg b/setup.cfg index d06868d5f..b1fe47cb1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = openpathsampling -version = 1.7.0 +version = 1.7.1.dev0 description = A Python package for path sampling simulations long_description = file: README.md long_description_content_type = text/markdown @@ -14,9 +14,9 @@ classifiers = Intended Audience :: Developers License :: OSI Approved :: MIT License Programming Language :: Python - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 Topic :: Scientific/Engineering :: Bio-Informatics Topic :: Scientific/Engineering :: Chemistry Topic :: Scientific/Engineering :: Physics @@ -26,11 +26,11 @@ classifiers = [options] include_package_data = True -python_requires = >=3.10 +python_requires = >=3.11 install_requires = future psutil - numpy<2.0 + numpy scipy pandas netcdf4