From 0720aa1ac9c672e8b7a6ede4828e603e62778b86 Mon Sep 17 00:00:00 2001 From: mauriliogenovese Date: Fri, 22 Mar 2024 13:56:24 +0100 Subject: [PATCH 01/34] support for gpu queue --- nipype/pipeline/engine/nodes.py | 4 ++ nipype/pipeline/plugins/multiproc.py | 62 ++++++++++++++++--- .../pipeline/plugins/tests/test_multiproc.py | 15 +++++ 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 3756d00ce8..5afea316c2 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -821,6 +821,10 @@ def update(self, **opts): """Update inputs""" self.inputs.update(**opts) + def is_gpu_node(self): + return ((hasattr(self.inputs, 'use_cuda') and self.inputs.use_cuda) + or (hasattr(self.inputs, 'use_gpu') and self.inputs.use_gpu)) + class JoinNode(Node): """Wraps interface objects that join inputs into a list. diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 401b01b388..8213c6c821 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -100,6 +100,7 @@ class MultiProcPlugin(DistributedPluginBase): - non_daemon: boolean flag to execute as non-daemon processes - n_procs: maximum number of threads to be executed in parallel + - n_gpu_procs: maximum number of GPU threads to be executed in parallel - memory_gb: maximum memory (in GB) that can be used at once. - raise_insufficient: raise error if the requested resources for a node over the maximum `n_procs` and/or `memory_gb` @@ -130,10 +131,22 @@ def __init__(self, plugin_args=None): ) self.raise_insufficient = self.plugin_args.get("raise_insufficient", True) + # GPU found on system + self.n_gpus_visible = MultiProcPlugin.gpu_count() + # proc per GPU set by user + self.n_gpu_procs = plugin_args.get('n_gpu_procs', self.n_gpus_visible) + + # total no. of processes allowed on all gpus + if self.n_gpu_procs > self.n_gpus_visible: + logger.info( + 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!' % ( + self.n_gpu_procs, self.n_gpus_visible)) + # Instantiate different thread pools for non-daemon processes logger.debug( - "[MultiProc] Starting (n_procs=%d, mem_gb=%0.2f, cwd=%s)", + "[MultiProc] Starting (n_procs=%d, n_gpu_procs=%d, mem_gb=%0.2f, cwd=%s)", self.processors, + self.n_gpu_procs, self.memory_gb, self._cwd, ) @@ -184,9 +197,12 @@ def _prerun_check(self, graph): """Check if any node exceeds the available resources""" tasks_mem_gb = [] tasks_num_th = [] + tasks_gpu_th = [] for node in graph.nodes(): tasks_mem_gb.append(node.mem_gb) tasks_num_th.append(node.n_procs) + if node.is_gpu_node(): + tasks_gpu_th.append(node.n_procs) if np.any(np.array(tasks_mem_gb) > self.memory_gb): logger.warning( @@ -203,6 +219,12 @@ def _prerun_check(self, graph): ) if self.raise_insufficient: raise RuntimeError("Insufficient resources available for job") + if np.any(np.array(tasks_gpu_th) > self.n_gpu_procs): + logger.warning( + 'Nodes demand more GPU than allowed (%d).', + self.n_gpu_procs) + if self.raise_insufficient: + raise RuntimeError('Insufficient GPU resources available for job') def _postrun_check(self): self.pool.shutdown() @@ -213,11 +235,14 @@ def _check_resources(self, running_tasks): """ free_memory_gb = self.memory_gb free_processors = self.processors + free_gpu_slots = self.n_gpu_procs for _, jobid in running_tasks: free_memory_gb -= min(self.procs[jobid].mem_gb, free_memory_gb) free_processors -= min(self.procs[jobid].n_procs, free_processors) + if self.procs[jobid].is_gpu_node(): + free_gpu_slots -= min(self.procs[jobid].n_procs, free_gpu_slots) - return free_memory_gb, free_processors + return free_memory_gb, free_processors, free_gpu_slots def _send_procs_to_workers(self, updatehash=False, graph=None): """ @@ -232,7 +257,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): ) # Check available resources by summing all threads and memory used - free_memory_gb, free_processors = self._check_resources(self.pending_tasks) + free_memory_gb, free_processors, free_gpu_slots = self._check_resources(self.pending_tasks) stats = ( len(self.pending_tasks), @@ -241,6 +266,8 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): self.memory_gb, free_processors, self.processors, + free_gpu_slots, + self.n_gpu_procs ) if self._stats != stats: tasks_list_msg = "" @@ -256,13 +283,15 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): tasks_list_msg = indent(tasks_list_msg, " " * 21) logger.info( "[MultiProc] Running %d tasks, and %d jobs ready. Free " - "memory (GB): %0.2f/%0.2f, Free processors: %d/%d.%s", + "memory (GB): %0.2f/%0.2f, Free processors: %d/%d, Free GPU slot:%d/%d.%s", len(self.pending_tasks), len(jobids), free_memory_gb, self.memory_gb, free_processors, self.processors, + free_gpu_slots, + self.n_gpu_procs, tasks_list_msg, ) self._stats = stats @@ -304,28 +333,36 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): # Check requirements of this job next_job_gb = min(self.procs[jobid].mem_gb, self.memory_gb) next_job_th = min(self.procs[jobid].n_procs, self.processors) + next_job_gpu_th = min(self.procs[jobid].n_procs, self.n_gpu_procs) + + is_gpu_node = self.procs[jobid].is_gpu_node() # If node does not fit, skip at this moment - if next_job_th > free_processors or next_job_gb > free_memory_gb: + if (next_job_th > free_processors or next_job_gb > free_memory_gb + or (is_gpu_node and next_job_gpu_th > free_gpu_slots)): logger.debug( - "Cannot allocate job %d (%0.2fGB, %d threads).", + "Cannot allocate job %d (%0.2fGB, %d threads, %d GPU slots).", jobid, next_job_gb, next_job_th, + next_job_gpu_th, ) continue free_memory_gb -= next_job_gb free_processors -= next_job_th + if is_gpu_node: + free_gpu_slots -= next_job_gpu_th logger.debug( "Allocating %s ID=%d (%0.2fGB, %d threads). Free: " - "%0.2fGB, %d threads.", + "%0.2fGB, %d threads, %d GPU slots.", self.procs[jobid].fullname, jobid, next_job_gb, next_job_th, free_memory_gb, free_processors, + free_gpu_slots, ) # change job status in appropriate queues @@ -352,6 +389,8 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): self._remove_node_dirs() free_memory_gb += next_job_gb free_processors += next_job_th + if is_gpu_node: + free_gpu_slots -= next_job_gpu_th # Display stats next loop self._stats = None @@ -379,3 +418,12 @@ def _sort_jobs(self, jobids, scheduler="tsort"): key=lambda item: (self.procs[item].mem_gb, self.procs[item].n_procs), ) return jobids + + @staticmethod + def gpu_count(): + n_gpus = 1 + try: + import GPUtil + return len(GPUtil.getGPUs()) + except ImportError: + return n_gpus diff --git a/nipype/pipeline/plugins/tests/test_multiproc.py b/nipype/pipeline/plugins/tests/test_multiproc.py index 938e1aab9e..b954cb9517 100644 --- a/nipype/pipeline/plugins/tests/test_multiproc.py +++ b/nipype/pipeline/plugins/tests/test_multiproc.py @@ -56,6 +56,7 @@ def test_run_multiproc(tmpdir): class InputSpecSingleNode(nib.TraitedSpec): input1 = nib.traits.Int(desc="a random int") input2 = nib.traits.Int(desc="a random int") + use_gpu = nib.traits.Bool(False, mandatory = False, desc="boolean for GPU nodes") class OutputSpecSingleNode(nib.TraitedSpec): @@ -116,6 +117,20 @@ def test_no_more_threads_than_specified(tmpdir): with pytest.raises(RuntimeError): pipe.run(plugin="MultiProc", plugin_args={"n_procs": max_threads}) +def test_no_more_gpu_threads_than_specified(tmpdir): + tmpdir.chdir() + + pipe = pe.Workflow(name="pipe") + n1 = pe.Node(SingleNodeTestInterface(), name="n1", n_procs=2) + n1.inputs.use_gpu = True + n1.inputs.input1 = 4 + pipe.add_nodes([n1]) + + max_threads = 2 + max_gpu = 1 + with pytest.raises(RuntimeError): + pipe.run(plugin="MultiProc", plugin_args={"n_procs": max_threads, 'n_gpu_procs': max_gpu}) + @pytest.mark.skipif( sys.version_info >= (3, 8), reason="multiprocessing issues in Python 3.8" From 6c47dc009aeac77be993d17d5b1fb6ab1fa5d9d2 Mon Sep 17 00:00:00 2001 From: mauriliogenovese Date: Sun, 24 Mar 2024 10:52:44 +0100 Subject: [PATCH 02/34] gputil requirement gputils is required for gpu queue management --- nipype/info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nipype/info.py b/nipype/info.py index a550e4b389..f4fc365e7e 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -149,6 +149,7 @@ def get_nipype_gitversion(): "filelock>=3.0.0", "etelemetry>=0.2.0", "looseversion!=1.2", + "gputil=1.4.0", ] TESTS_REQUIRES = [ From f1f5d764a3d7452a04c807388b454cf216744e8b Mon Sep 17 00:00:00 2001 From: mauriliogenovese Date: Mon, 25 Mar 2024 07:38:29 +0100 Subject: [PATCH 03/34] Update info.py --- nipype/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/info.py b/nipype/info.py index f4fc365e7e..280c641ed6 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -149,7 +149,7 @@ def get_nipype_gitversion(): "filelock>=3.0.0", "etelemetry>=0.2.0", "looseversion!=1.2", - "gputil=1.4.0", + "gputil==1.4.0", ] TESTS_REQUIRES = [ From a6424301d7bd3440fe5f465ba05fde0b38e37aed Mon Sep 17 00:00:00 2001 From: mauriliogenovese Date: Mon, 25 Mar 2024 09:59:41 +0100 Subject: [PATCH 04/34] refactor and fix --- nipype/pipeline/engine/nodes.py | 5 ++-- nipype/pipeline/plugins/multiproc.py | 25 +++++++++++-------- .../pipeline/plugins/tests/test_multiproc.py | 8 ++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 5afea316c2..d9c066a795 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -822,8 +822,9 @@ def update(self, **opts): self.inputs.update(**opts) def is_gpu_node(self): - return ((hasattr(self.inputs, 'use_cuda') and self.inputs.use_cuda) - or (hasattr(self.inputs, 'use_gpu') and self.inputs.use_gpu)) + return (hasattr(self.inputs, 'use_cuda') and self.inputs.use_cuda) or ( + hasattr(self.inputs, 'use_gpu') and self.inputs.use_gpu + ) class JoinNode(Node): diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 8213c6c821..9aec6ae072 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -134,13 +134,14 @@ def __init__(self, plugin_args=None): # GPU found on system self.n_gpus_visible = MultiProcPlugin.gpu_count() # proc per GPU set by user - self.n_gpu_procs = plugin_args.get('n_gpu_procs', self.n_gpus_visible) + self.n_gpu_procs = self.plugin_args.get('n_gpu_procs', self.n_gpus_visible) # total no. of processes allowed on all gpus if self.n_gpu_procs > self.n_gpus_visible: logger.info( - 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!' % ( - self.n_gpu_procs, self.n_gpus_visible)) + 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!' + % (self.n_gpu_procs, self.n_gpus_visible) + ) # Instantiate different thread pools for non-daemon processes logger.debug( @@ -220,9 +221,7 @@ def _prerun_check(self, graph): if self.raise_insufficient: raise RuntimeError("Insufficient resources available for job") if np.any(np.array(tasks_gpu_th) > self.n_gpu_procs): - logger.warning( - 'Nodes demand more GPU than allowed (%d).', - self.n_gpu_procs) + logger.warning('Nodes demand more GPU than allowed (%d).', self.n_gpu_procs) if self.raise_insufficient: raise RuntimeError('Insufficient GPU resources available for job') @@ -257,7 +256,9 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): ) # Check available resources by summing all threads and memory used - free_memory_gb, free_processors, free_gpu_slots = self._check_resources(self.pending_tasks) + free_memory_gb, free_processors, free_gpu_slots = self._check_resources( + self.pending_tasks + ) stats = ( len(self.pending_tasks), @@ -267,7 +268,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): free_processors, self.processors, free_gpu_slots, - self.n_gpu_procs + self.n_gpu_procs, ) if self._stats != stats: tasks_list_msg = "" @@ -338,8 +339,11 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): is_gpu_node = self.procs[jobid].is_gpu_node() # If node does not fit, skip at this moment - if (next_job_th > free_processors or next_job_gb > free_memory_gb - or (is_gpu_node and next_job_gpu_th > free_gpu_slots)): + if ( + next_job_th > free_processors + or next_job_gb > free_memory_gb + or (is_gpu_node and next_job_gpu_th > free_gpu_slots) + ): logger.debug( "Cannot allocate job %d (%0.2fGB, %d threads, %d GPU slots).", jobid, @@ -424,6 +428,7 @@ def gpu_count(): n_gpus = 1 try: import GPUtil + return len(GPUtil.getGPUs()) except ImportError: return n_gpus diff --git a/nipype/pipeline/plugins/tests/test_multiproc.py b/nipype/pipeline/plugins/tests/test_multiproc.py index b954cb9517..484c0d07bc 100644 --- a/nipype/pipeline/plugins/tests/test_multiproc.py +++ b/nipype/pipeline/plugins/tests/test_multiproc.py @@ -56,7 +56,7 @@ def test_run_multiproc(tmpdir): class InputSpecSingleNode(nib.TraitedSpec): input1 = nib.traits.Int(desc="a random int") input2 = nib.traits.Int(desc="a random int") - use_gpu = nib.traits.Bool(False, mandatory = False, desc="boolean for GPU nodes") + use_gpu = nib.traits.Bool(False, mandatory=False, desc="boolean for GPU nodes") class OutputSpecSingleNode(nib.TraitedSpec): @@ -117,6 +117,7 @@ def test_no_more_threads_than_specified(tmpdir): with pytest.raises(RuntimeError): pipe.run(plugin="MultiProc", plugin_args={"n_procs": max_threads}) + def test_no_more_gpu_threads_than_specified(tmpdir): tmpdir.chdir() @@ -129,7 +130,10 @@ def test_no_more_gpu_threads_than_specified(tmpdir): max_threads = 2 max_gpu = 1 with pytest.raises(RuntimeError): - pipe.run(plugin="MultiProc", plugin_args={"n_procs": max_threads, 'n_gpu_procs': max_gpu}) + pipe.run( + plugin="MultiProc", + plugin_args={"n_procs": max_threads, 'n_gpu_procs': max_gpu}, + ) @pytest.mark.skipif( From 69b62a408c8e909e5a9fd2412abd84b6824d127a Mon Sep 17 00:00:00 2001 From: Umesh Kumar Date: Thu, 5 Oct 2023 17:26:07 +0530 Subject: [PATCH 05/34] Adding .wci.yml file to the project --- .wci.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .wci.yml diff --git a/.wci.yml b/.wci.yml new file mode 100644 index 0000000000..c3909ea8e5 --- /dev/null +++ b/.wci.yml @@ -0,0 +1,32 @@ +# Project available at https://github.com/nipy/nipype + +name: nipype + +headline: Neuroimaging in Python pipelines and interfaces package. + +description: Algorithms,Image manipulation,I/O Operations,Self-reporting interfaces,Utilities + +language: Python3 + +documentation: + general: https://nipype.readthedocs.io/en/latest/ + installation: https://nipype.readthedocs.io/en/latest/users/install.html + tutorial: https://miykael.github.io/nipype_tutorial/ + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: doc/conf.py + +execution_environment: + interfaces: + - docker + - conda + - pypi + + From f54475cfb84a07aa12eea3794365bd2f5c1bf2f1 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sun, 17 Mar 2024 11:54:35 -0400 Subject: [PATCH 06/34] Apply suggestions from code review --- .wci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.wci.yml b/.wci.yml index c3909ea8e5..6439e9fe08 100644 --- a/.wci.yml +++ b/.wci.yml @@ -2,9 +2,14 @@ name: nipype -headline: Neuroimaging in Python pipelines and interfaces package. - -description: Algorithms,Image manipulation,I/O Operations,Self-reporting interfaces,Utilities +headline: Neuroimaging in Python: Pipelines and Interfaces + +description: | + Nipype, an open-source, community-developed initiative under the umbrella of NiPy, is a Python project that + provides a uniform interface to existing neuroimaging software and facilitates interaction between these + packages within a single workflow. Nipype provides an environment that encourages interactive exploration of + algorithms from different packages (e.g., SPM, FSL, FreeSurfer, AFNI, Slicer, ANTS), eases the design of + workflows within and between packages, and reduces the learning curve necessary to use different packages. language: Python3 From 6fb6a4e9447d4679c8278a598eaea77543cdb7aa Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Sun, 5 May 2024 21:22:05 -0400 Subject: [PATCH 07/34] Update .wci.yml --- .wci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.wci.yml b/.wci.yml index 6439e9fe08..937cfc6dae 100644 --- a/.wci.yml +++ b/.wci.yml @@ -33,5 +33,3 @@ execution_environment: - docker - conda - pypi - - From ceb1cc851443dac074ecc0c4ed8b891ef02af1d5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 17 Dec 2024 12:48:18 -0500 Subject: [PATCH 08/34] chore: Bump dev version --- doc/interfaces.rst | 2 +- nipype/info.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/interfaces.rst b/doc/interfaces.rst index da817fa163..795574a5e6 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -8,7 +8,7 @@ Interfaces and Workflows :Release: |version| :Date: |today| -Previous versions: `1.9.1 `_ `1.9.0 `_ +Previous versions: `1.9.2 `_ `1.9.1 `_ Workflows --------- diff --git a/nipype/info.py b/nipype/info.py index 3fd328e995..1341fc5ba8 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -5,7 +5,7 @@ # nipype version information # Remove .dev0 for release -__version__ = "1.9.2" +__version__ = "1.9.3.dev0" def get_nipype_gitversion(): From 478b663859dccb0c8dd5b4f916b02080a7f99610 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2024 04:20:57 +0000 Subject: [PATCH 09/34] Chore(deps): Bump astral-sh/setup-uv from 4 to 5 Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 4 to 5. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/v4...v5) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5b0943c4ca..d789ec9061 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: with: fetch-depth: 0 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@v5 - run: uv build - run: uvx twine check dist/* - uses: actions/upload-artifact@v4 @@ -102,7 +102,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@v5 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -152,7 +152,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@v5 - name: Show tox config run: uvx tox c - name: Show tox config (this call) From 2717267423c26597ffde6f598ebd58eb19c8af81 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 17 Jan 2025 08:35:12 -0500 Subject: [PATCH 10/34] Update .wci.yml --- .wci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.wci.yml b/.wci.yml index 937cfc6dae..ff7937c045 100644 --- a/.wci.yml +++ b/.wci.yml @@ -18,16 +18,6 @@ documentation: installation: https://nipype.readthedocs.io/en/latest/users/install.html tutorial: https://miykael.github.io/nipype_tutorial/ -# Set the OS, Python version and other tools you might need -build: - os: ubuntu-22.04 - tools: - python: "3.10" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: doc/conf.py - execution_environment: interfaces: - docker From ce9511ffa8ac93f3fe311356253d19c747f4068e Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 17 Jan 2025 08:42:29 -0500 Subject: [PATCH 11/34] Update .wci.yml --- .wci.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.wci.yml b/.wci.yml index ff7937c045..1e6fbc8ddf 100644 --- a/.wci.yml +++ b/.wci.yml @@ -19,7 +19,12 @@ documentation: tutorial: https://miykael.github.io/nipype_tutorial/ execution_environment: - interfaces: - - docker - - conda - - pypi + resource_managers: + - SLURM + - Condor + - DAGMan + - LSF + - OAR + - PBS + - SGE + - Soma-workflow From 9e61c82848e3c00fe6e3b73e85ff6823b20908af Mon Sep 17 00:00:00 2001 From: Rafael Ferreira da Silva Date: Sat, 18 Jan 2025 08:28:05 -0300 Subject: [PATCH 12/34] Update to .wci.yml The headline information contains a colon, thus it is necessary to put the text within quotes. --- .wci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.wci.yml b/.wci.yml index 1e6fbc8ddf..2adbae9fcc 100644 --- a/.wci.yml +++ b/.wci.yml @@ -2,7 +2,7 @@ name: nipype -headline: Neuroimaging in Python: Pipelines and Interfaces +headline: "Neuroimaging in Python: Pipelines and Interfaces" description: | Nipype, an open-source, community-developed initiative under the umbrella of NiPy, is a Python project that From 684b9b0e15618537f78248bf3c8953ad3f4f6eeb Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sat, 18 Jan 2025 16:37:31 +0100 Subject: [PATCH 13/34] removed hard pin --- nipype/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/info.py b/nipype/info.py index 280c641ed6..ad45791a50 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -149,7 +149,7 @@ def get_nipype_gitversion(): "filelock>=3.0.0", "etelemetry>=0.2.0", "looseversion!=1.2", - "gputil==1.4.0", + "gputil>=1.4.0", ] TESTS_REQUIRES = [ From 8f74c5dd362e73e28282900297492c9dd7da7ed8 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sat, 18 Jan 2025 16:42:00 +0100 Subject: [PATCH 14/34] gpu_count refactor --- nipype/pipeline/plugins/multiproc.py | 12 ++---------- nipype/pipeline/plugins/tools.py | 9 +++++++++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 9aec6ae072..4d7eaa6c6b 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -21,6 +21,7 @@ from ...utils.profiler import get_system_total_memory_gb from ..engine import MapNode from .base import DistributedPluginBase +from .tools import gpu_count try: from textwrap import indent @@ -132,7 +133,7 @@ def __init__(self, plugin_args=None): self.raise_insufficient = self.plugin_args.get("raise_insufficient", True) # GPU found on system - self.n_gpus_visible = MultiProcPlugin.gpu_count() + self.n_gpus_visible = gpu_count() # proc per GPU set by user self.n_gpu_procs = self.plugin_args.get('n_gpu_procs', self.n_gpus_visible) @@ -423,12 +424,3 @@ def _sort_jobs(self, jobids, scheduler="tsort"): ) return jobids - @staticmethod - def gpu_count(): - n_gpus = 1 - try: - import GPUtil - - return len(GPUtil.getGPUs()) - except ImportError: - return n_gpus diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index 8c28f36246..c767be398e 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -178,3 +178,12 @@ def create_pyscript(node, updatehash=False, store_exception=True): with open(pyscript, "w") as fp: fp.writelines(cmdstr) return pyscript + +def gpu_count(): + n_gpus = 1 + try: + import GPUtil + except ImportError: + return 1 + else: + return len(GPUtil.getGPUs()) From a307845390115b1b65659159c662334b39575ff5 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sat, 18 Jan 2025 16:45:24 +0100 Subject: [PATCH 15/34] more readable --- nipype/pipeline/engine/nodes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index d9c066a795..f9036529ac 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -822,9 +822,8 @@ def update(self, **opts): self.inputs.update(**opts) def is_gpu_node(self): - return (hasattr(self.inputs, 'use_cuda') and self.inputs.use_cuda) or ( - hasattr(self.inputs, 'use_gpu') and self.inputs.use_gpu - ) + return bool(getattr(self.inputs, 'use_cuda', False)) or bool( + getattr(self.inputs, 'use_gpu', False)) class JoinNode(Node): From 27448bcfb2ea34029f561dff5efd275f31609096 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sat, 18 Jan 2025 16:46:21 +0100 Subject: [PATCH 16/34] logger argument --- nipype/pipeline/plugins/multiproc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 4d7eaa6c6b..e1aa07d13b 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -140,9 +140,8 @@ def __init__(self, plugin_args=None): # total no. of processes allowed on all gpus if self.n_gpu_procs > self.n_gpus_visible: logger.info( - 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!' - % (self.n_gpu_procs, self.n_gpus_visible) - ) + 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!', + self.n_gpu_procs, self.n_gpus_visible) # Instantiate different thread pools for non-daemon processes logger.debug( From 2c2c066d846f12738c3ddd57af0e5ca3dc97df31 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sat, 18 Jan 2025 18:52:56 +0100 Subject: [PATCH 17/34] code refactory --- nipype/pipeline/engine/nodes.py | 3 ++- nipype/pipeline/plugins/multiproc.py | 9 +++++---- nipype/pipeline/plugins/tools.py | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index f9036529ac..2d524c3efe 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -823,7 +823,8 @@ def update(self, **opts): def is_gpu_node(self): return bool(getattr(self.inputs, 'use_cuda', False)) or bool( - getattr(self.inputs, 'use_gpu', False)) + getattr(self.inputs, 'use_gpu', False) + ) class JoinNode(Node): diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index e1aa07d13b..054d0150e6 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -141,7 +141,9 @@ def __init__(self, plugin_args=None): if self.n_gpu_procs > self.n_gpus_visible: logger.info( 'Total number of GPUs proc requested (%d) exceeds the available number of GPUs (%d) on the system. Using requested GPU slots at your own risk!', - self.n_gpu_procs, self.n_gpus_visible) + self.n_gpu_procs, + self.n_gpus_visible, + ) # Instantiate different thread pools for non-daemon processes logger.debug( @@ -394,7 +396,7 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): free_memory_gb += next_job_gb free_processors += next_job_th if is_gpu_node: - free_gpu_slots -= next_job_gpu_th + free_gpu_slots += next_job_gpu_th # Display stats next loop self._stats = None @@ -421,5 +423,4 @@ def _sort_jobs(self, jobids, scheduler="tsort"): jobids, key=lambda item: (self.procs[item].mem_gb, self.procs[item].n_procs), ) - return jobids - + return jobids \ No newline at end of file diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index c767be398e..eb50d4b40c 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -179,6 +179,7 @@ def create_pyscript(node, updatehash=False, store_exception=True): fp.writelines(cmdstr) return pyscript + def gpu_count(): n_gpus = 1 try: @@ -186,4 +187,4 @@ def gpu_count(): except ImportError: return 1 else: - return len(GPUtil.getGPUs()) + return len(GPUtil.getGPUs()) \ No newline at end of file From 66d628022155fbb2dcbacc798abd5e0242727804 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sat, 18 Jan 2025 18:56:10 +0100 Subject: [PATCH 18/34] newlines for style check --- nipype/pipeline/plugins/multiproc.py | 2 +- nipype/pipeline/plugins/tools.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 054d0150e6..ce191b0f7c 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -423,4 +423,4 @@ def _sort_jobs(self, jobids, scheduler="tsort"): jobids, key=lambda item: (self.procs[item].mem_gb, self.procs[item].n_procs), ) - return jobids \ No newline at end of file + return jobids diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index 43e50c276b..dabd75116c 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -185,4 +185,5 @@ def gpu_count(): except ImportError: return 1 else: - return len(GPUtil.getGPUs()) \ No newline at end of file + return len(GPUtil.getGPUs()) + \ No newline at end of file From 610f1cbe33490ef5e6f4d2e43037655594a64f18 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sun, 19 Jan 2025 10:25:09 +0100 Subject: [PATCH 19/34] newline for code check --- nipype/pipeline/plugins/tools.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index dabd75116c..3d879a1971 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -186,4 +186,3 @@ def gpu_count(): return 1 else: return len(GPUtil.getGPUs()) - \ No newline at end of file From 4298707009e58fdab8eb5db4113059cda975b4a1 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sun, 19 Jan 2025 10:27:59 +0100 Subject: [PATCH 20/34] fix for updatehash crash --- nipype/pipeline/engine/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 31ee29e04d..ace18e8fec 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -452,7 +452,7 @@ def run(self, updatehash=False): cached, updated = self.is_cached() # If the node is cached, check on pklz files and finish - if not force_run and (updated or (not updated and updatehash)): + if cached and not force_run and (updated or (not updated and updatehash)): logger.debug("Only updating node hashes or skipping execution") inputs_file = op.join(outdir, "_inputs.pklz") if not op.exists(inputs_file): From 3e555989af20b2d2d2acf7d66192307475de788b Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sun, 19 Jan 2025 16:56:59 +0100 Subject: [PATCH 21/34] Update nipype/pipeline/engine/nodes.py Co-authored-by: Chris Markiewicz --- nipype/pipeline/engine/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index ace18e8fec..eb524a6a6f 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -452,7 +452,7 @@ def run(self, updatehash=False): cached, updated = self.is_cached() # If the node is cached, check on pklz files and finish - if cached and not force_run and (updated or (not updated and updatehash)): + if cached and not force_run and (updated or updatehash): logger.debug("Only updating node hashes or skipping execution") inputs_file = op.join(outdir, "_inputs.pklz") if not op.exists(inputs_file): From 88e30f8e0442d12ba3cfb194e857b118bc727aef Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Sun, 19 Jan 2025 17:43:26 +0100 Subject: [PATCH 22/34] keep multiprocess with updatehash=True --- nipype/pipeline/plugins/multiproc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 401b01b388..9b27b47383 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -336,8 +336,9 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): if self._local_hash_check(jobid, graph): continue + cached, updated = self.procs[jobid].is_cached() # updatehash and run_without_submitting are also run locally - if updatehash or self.procs[jobid].run_without_submitting: + if (cached and updatehash and not updated) or self.procs[jobid].run_without_submitting: logger.debug("Running node %s on master thread", self.procs[jobid]) try: self.procs[jobid].run(updatehash=updatehash) From 0afbb81b98b38dba4d0a54501ba05acf730b54f8 Mon Sep 17 00:00:00 2001 From: Himanshu Aggarwal Date: Tue, 11 Feb 2025 16:32:03 +0100 Subject: [PATCH 23/34] [FIX] Set length to 1 when np.squeeze returns a 0D array --- nipype/algorithms/misc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index e1a67f0b08..a29b629703 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1495,7 +1495,11 @@ def merge_rois(in_files, in_idxs, in_ref, dtype=None, out_file=None): for d, fname in enumerate(nii): data = np.asanyarray(nb.load(fname).dataobj).reshape(-1) cdata = nb.load(cname).dataobj[..., d].reshape(-1) - nels = len(idxs) + try: + nels = len(idxs) + except TypeError: + nels = 1 + idata = (idxs,) data[idata] = cdata[0:nels] nb.Nifti1Image(data.reshape(rsh[:3]), aff, hdr).to_filename(fname) From 2c8017df7e2c16ae76f55ce5939d73f4784f6d17 Mon Sep 17 00:00:00 2001 From: Himanshu Aggarwal Date: Tue, 11 Feb 2025 17:38:39 +0100 Subject: [PATCH 24/34] apply code review suggestion --- nipype/algorithms/misc.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/nipype/algorithms/misc.py b/nipype/algorithms/misc.py index a29b629703..fe27b877a2 100644 --- a/nipype/algorithms/misc.py +++ b/nipype/algorithms/misc.py @@ -1490,18 +1490,13 @@ def merge_rois(in_files, in_idxs, in_ref, dtype=None, out_file=None): for cname, iname in zip(in_files, in_idxs): f = np.load(iname) - idxs = np.squeeze(f["arr_0"]) + idxs = np.atleast_1d(np.squeeze(f["arr_0"])) + nels = len(idxs) for d, fname in enumerate(nii): data = np.asanyarray(nb.load(fname).dataobj).reshape(-1) cdata = nb.load(cname).dataobj[..., d].reshape(-1) - try: - nels = len(idxs) - except TypeError: - nels = 1 - - idata = (idxs,) - data[idata] = cdata[0:nels] + data[idxs] = cdata[:nels] nb.Nifti1Image(data.reshape(rsh[:3]), aff, hdr).to_filename(fname) imgs = [nb.load(im) for im in nii] From 8ed2b2306aeb7d89de4958b5293223ffe27a4f34 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Feb 2025 14:06:53 -0500 Subject: [PATCH 25/34] sty: black [ignore-rev] --- nipype/caching/tests/test_memory.py | 3 +-- nipype/external/cloghandler.py | 2 +- nipype/info.py | 2 +- nipype/interfaces/ants/registration.py | 2 +- nipype/interfaces/ants/resampling.py | 3 +-- nipype/interfaces/ants/visualization.py | 3 +-- nipype/interfaces/bru2nii.py | 3 +-- nipype/interfaces/camino/__init__.py | 3 +-- nipype/interfaces/cmtk/base.py | 2 +- nipype/interfaces/diffusion_toolkit/dti.py | 3 +-- nipype/interfaces/diffusion_toolkit/odf.py | 3 +-- nipype/interfaces/diffusion_toolkit/postproc.py | 3 +-- nipype/interfaces/dipy/base.py | 2 +- nipype/interfaces/freesurfer/longitudinal.py | 3 +-- nipype/interfaces/freesurfer/model.py | 2 +- nipype/interfaces/freesurfer/petsurfer.py | 3 +-- nipype/interfaces/freesurfer/preprocess.py | 3 +-- nipype/interfaces/freesurfer/registration.py | 3 +-- nipype/interfaces/freesurfer/utils.py | 3 +-- nipype/interfaces/io.py | 14 +++++++------- nipype/interfaces/mixins/reporting.py | 2 +- nipype/interfaces/nipy/base.py | 2 +- nipype/interfaces/nitime/base.py | 2 +- nipype/interfaces/spm/preprocess.py | 3 +-- nipype/interfaces/utility/base.py | 6 +++--- nipype/interfaces/utility/csv.py | 3 +-- nipype/pipeline/engine/tests/test_engine.py | 3 +-- nipype/pipeline/engine/tests/test_join.py | 3 +-- nipype/pipeline/engine/tests/test_utils.py | 3 +-- nipype/pipeline/engine/tests/test_workflows.py | 3 +-- nipype/pipeline/plugins/condor.py | 3 +-- nipype/pipeline/plugins/dagman.py | 3 +-- nipype/pipeline/plugins/debug.py | 3 +-- nipype/pipeline/plugins/ipython.py | 3 +-- nipype/pipeline/plugins/linear.py | 3 +-- nipype/pipeline/plugins/lsf.py | 3 +-- nipype/pipeline/plugins/multiproc.py | 4 +++- nipype/pipeline/plugins/oar.py | 3 +-- nipype/pipeline/plugins/pbs.py | 3 +-- nipype/pipeline/plugins/pbsgraph.py | 3 +-- nipype/pipeline/plugins/sge.py | 3 +-- nipype/pipeline/plugins/sgegraph.py | 3 +-- nipype/pipeline/plugins/slurmgraph.py | 3 +-- nipype/pipeline/plugins/somaflow.py | 3 +-- nipype/pipeline/plugins/tests/test_base.py | 3 +-- .../tests/test_legacymultiproc_nondaemon.py | 3 +-- nipype/pipeline/plugins/tests/test_tools.py | 3 +-- nipype/pipeline/plugins/tools.py | 3 +-- nipype/testing/tests/test_utils.py | 3 +-- nipype/testing/utils.py | 3 +-- nipype/utils/filemanip.py | 3 +-- nipype/utils/matlabtools.py | 2 +- nipype/utils/misc.py | 3 +-- nipype/utils/subprocess.py | 3 +-- 54 files changed, 64 insertions(+), 103 deletions(-) diff --git a/nipype/caching/tests/test_memory.py b/nipype/caching/tests/test_memory.py index 5bd9fad528..cd5b8f8075 100644 --- a/nipype/caching/tests/test_memory.py +++ b/nipype/caching/tests/test_memory.py @@ -1,5 +1,4 @@ -""" Test the nipype interface caching mechanism -""" +"""Test the nipype interface caching mechanism""" from .. import Memory from ...pipeline.engine.tests.test_engine import EngineTestInterface diff --git a/nipype/external/cloghandler.py b/nipype/external/cloghandler.py index 289c8dfa2f..680ba30e2e 100644 --- a/nipype/external/cloghandler.py +++ b/nipype/external/cloghandler.py @@ -9,7 +9,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" cloghandler.py: A smart replacement for the standard RotatingFileHandler +"""cloghandler.py: A smart replacement for the standard RotatingFileHandler ConcurrentRotatingFileHandler: This class is a log handler which is a drop-in replacement for the python standard log handler 'RotateFileHandler', the primary diff --git a/nipype/info.py b/nipype/info.py index 1341fc5ba8..729689ae5d 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -1,4 +1,4 @@ -""" This file contains defines parameters for nipy that we use to fill +"""This file contains defines parameters for nipy that we use to fill settings in setup.py, the nipy top-level docstring, and for building the docs. In setup.py in particular, we exec this file, so it cannot import nipy """ diff --git a/nipype/interfaces/ants/registration.py b/nipype/interfaces/ants/registration.py index 91b131bbf3..55e9738170 100644 --- a/nipype/interfaces/ants/registration.py +++ b/nipype/interfaces/ants/registration.py @@ -1,5 +1,5 @@ """The ants module provides basic functions for interfacing with ants - functions. +functions. """ import os diff --git a/nipype/interfaces/ants/resampling.py b/nipype/interfaces/ants/resampling.py index 95f29d5982..883eff1de3 100644 --- a/nipype/interfaces/ants/resampling.py +++ b/nipype/interfaces/ants/resampling.py @@ -1,5 +1,4 @@ -"""ANTS Apply Transforms interface -""" +"""ANTS Apply Transforms interface""" import os diff --git a/nipype/interfaces/ants/visualization.py b/nipype/interfaces/ants/visualization.py index c73b64c632..cdfa3529a7 100644 --- a/nipype/interfaces/ants/visualization.py +++ b/nipype/interfaces/ants/visualization.py @@ -1,5 +1,4 @@ -"""The ants visualisation module provides basic functions based on ITK. -""" +"""The ants visualisation module provides basic functions based on ITK.""" import os diff --git a/nipype/interfaces/bru2nii.py b/nipype/interfaces/bru2nii.py index 746af18f1a..b07f6a58d3 100644 --- a/nipype/interfaces/bru2nii.py +++ b/nipype/interfaces/bru2nii.py @@ -1,5 +1,4 @@ -"""The bru2nii module provides basic functions for dicom conversion -""" +"""The bru2nii module provides basic functions for dicom conversion""" import os from .base import ( diff --git a/nipype/interfaces/camino/__init__.py b/nipype/interfaces/camino/__init__.py index 766fa9c906..67e973df66 100644 --- a/nipype/interfaces/camino/__init__.py +++ b/nipype/interfaces/camino/__init__.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Camino top level namespace -""" +"""Camino top level namespace""" from .connectivity import Conmat from .convert import ( diff --git a/nipype/interfaces/cmtk/base.py b/nipype/interfaces/cmtk/base.py index d0c226dc49..c4c997288b 100644 --- a/nipype/interfaces/cmtk/base.py +++ b/nipype/interfaces/cmtk/base.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Base interface for cmtk """ +"""Base interface for cmtk""" from ..base import LibraryBaseInterface from ...utils.misc import package_check diff --git a/nipype/interfaces/diffusion_toolkit/dti.py b/nipype/interfaces/diffusion_toolkit/dti.py index fa031799e3..bf6336c96d 100644 --- a/nipype/interfaces/diffusion_toolkit/dti.py +++ b/nipype/interfaces/diffusion_toolkit/dti.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by diffusion toolkit -""" +"""Provides interfaces to various commands provided by diffusion toolkit""" import os import re diff --git a/nipype/interfaces/diffusion_toolkit/odf.py b/nipype/interfaces/diffusion_toolkit/odf.py index 00f86a322c..daadffc200 100644 --- a/nipype/interfaces/diffusion_toolkit/odf.py +++ b/nipype/interfaces/diffusion_toolkit/odf.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by diffusion toolkit -""" +"""Provides interfaces to various commands provided by diffusion toolkit""" import os import re diff --git a/nipype/interfaces/diffusion_toolkit/postproc.py b/nipype/interfaces/diffusion_toolkit/postproc.py index 5190843875..d05cfadff6 100644 --- a/nipype/interfaces/diffusion_toolkit/postproc.py +++ b/nipype/interfaces/diffusion_toolkit/postproc.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by diffusion toolkit -""" +"""Provides interfaces to various commands provided by diffusion toolkit""" import os from ..base import ( diff --git a/nipype/interfaces/dipy/base.py b/nipype/interfaces/dipy/base.py index 1b9bdea6d5..44290cd1d7 100644 --- a/nipype/interfaces/dipy/base.py +++ b/nipype/interfaces/dipy/base.py @@ -1,4 +1,4 @@ -""" Base interfaces for dipy """ +"""Base interfaces for dipy""" import os.path as op import inspect diff --git a/nipype/interfaces/freesurfer/longitudinal.py b/nipype/interfaces/freesurfer/longitudinal.py index 227ea76775..41e95c091b 100644 --- a/nipype/interfaces/freesurfer/longitudinal.py +++ b/nipype/interfaces/freesurfer/longitudinal.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various longitudinal commands provided by freesurfer -""" +"""Provides interfaces to various longitudinal commands provided by freesurfer""" import os diff --git a/nipype/interfaces/freesurfer/model.py b/nipype/interfaces/freesurfer/model.py index 6376c1b971..5e245a9a85 100644 --- a/nipype/interfaces/freesurfer/model.py +++ b/nipype/interfaces/freesurfer/model.py @@ -1,7 +1,7 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """The freesurfer module provides basic functions for interfacing with - freesurfer tools. +freesurfer tools. """ import os diff --git a/nipype/interfaces/freesurfer/petsurfer.py b/nipype/interfaces/freesurfer/petsurfer.py index 4505985127..28aa763b06 100644 --- a/nipype/interfaces/freesurfer/petsurfer.py +++ b/nipype/interfaces/freesurfer/petsurfer.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands for running PET analyses provided by FreeSurfer -""" +"""Provides interfaces to various commands for running PET analyses provided by FreeSurfer""" import os diff --git a/nipype/interfaces/freesurfer/preprocess.py b/nipype/interfaces/freesurfer/preprocess.py index 5b2fd19a0b..89c218f969 100644 --- a/nipype/interfaces/freesurfer/preprocess.py +++ b/nipype/interfaces/freesurfer/preprocess.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various commands provided by FreeSurfer -""" +"""Provides interfaces to various commands provided by FreeSurfer""" import os import os.path as op from glob import glob diff --git a/nipype/interfaces/freesurfer/registration.py b/nipype/interfaces/freesurfer/registration.py index bc70fc44a6..790066d0ec 100644 --- a/nipype/interfaces/freesurfer/registration.py +++ b/nipype/interfaces/freesurfer/registration.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Provides interfaces to various longitudinal commands provided by freesurfer -""" +"""Provides interfaces to various longitudinal commands provided by freesurfer""" import os import os.path diff --git a/nipype/interfaces/freesurfer/utils.py b/nipype/interfaces/freesurfer/utils.py index 777f42f019..2c1cdbcc94 100644 --- a/nipype/interfaces/freesurfer/utils.py +++ b/nipype/interfaces/freesurfer/utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Interfaces to assorted Freesurfer utility programs. -""" +"""Interfaces to assorted Freesurfer utility programs.""" import os import re import shutil diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 46cdfb44f2..d6af1ba073 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -1,14 +1,14 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Set of interfaces that allow interaction with data. Currently - available interfaces are: +"""Set of interfaces that allow interaction with data. Currently +available interfaces are: - DataSource: Generic nifti to named Nifti interface - DataSink: Generic named output from interfaces to data store - XNATSource: preliminary interface to XNAT +DataSource: Generic nifti to named Nifti interface +DataSink: Generic named output from interfaces to data store +XNATSource: preliminary interface to XNAT - To come : - XNATSink +To come : +XNATSink """ import glob import fnmatch diff --git a/nipype/interfaces/mixins/reporting.py b/nipype/interfaces/mixins/reporting.py index 90ca804618..a836cfa3fa 100644 --- a/nipype/interfaces/mixins/reporting.py +++ b/nipype/interfaces/mixins/reporting.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" class mixin and utilities for enabling reports for nipype interfaces """ +"""class mixin and utilities for enabling reports for nipype interfaces""" import os from abc import abstractmethod diff --git a/nipype/interfaces/nipy/base.py b/nipype/interfaces/nipy/base.py index 25aef8b873..1f8f1e4657 100644 --- a/nipype/interfaces/nipy/base.py +++ b/nipype/interfaces/nipy/base.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Base interface for nipy """ +"""Base interface for nipy""" from ..base import LibraryBaseInterface from ...utils.misc import package_check diff --git a/nipype/interfaces/nitime/base.py b/nipype/interfaces/nitime/base.py index 7e434f1d3e..4109bc3a74 100644 --- a/nipype/interfaces/nitime/base.py +++ b/nipype/interfaces/nitime/base.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Base interface for nitime """ +"""Base interface for nitime""" from ..base import LibraryBaseInterface diff --git a/nipype/interfaces/spm/preprocess.py b/nipype/interfaces/spm/preprocess.py index c7f69785ff..8d931a72ba 100644 --- a/nipype/interfaces/spm/preprocess.py +++ b/nipype/interfaces/spm/preprocess.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""SPM wrappers for preprocessing data -""" +"""SPM wrappers for preprocessing data""" import os from copy import deepcopy diff --git a/nipype/interfaces/utility/base.py b/nipype/interfaces/utility/base.py index 564966cb5b..ecc1bf7935 100644 --- a/nipype/interfaces/utility/base.py +++ b/nipype/interfaces/utility/base.py @@ -1,9 +1,9 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: """ - # changing to temporary directories - >>> tmp = getfixture('tmpdir') - >>> old = tmp.chdir() +# changing to temporary directories +>>> tmp = getfixture('tmpdir') +>>> old = tmp.chdir() """ import os import re diff --git a/nipype/interfaces/utility/csv.py b/nipype/interfaces/utility/csv.py index 979e328bb6..7470eecbfe 100644 --- a/nipype/interfaces/utility/csv.py +++ b/nipype/interfaces/utility/csv.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""CSV Handling utilities -""" +"""CSV Handling utilities""" import csv from ..base import traits, TraitedSpec, DynamicTraitedSpec, File, BaseInterface from ..io import add_traits diff --git a/nipype/pipeline/engine/tests/test_engine.py b/nipype/pipeline/engine/tests/test_engine.py index f1b6817e74..7650be1cd3 100644 --- a/nipype/pipeline/engine/tests/test_engine.py +++ b/nipype/pipeline/engine/tests/test_engine.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine module -""" +"""Tests for the engine module""" from copy import deepcopy from glob import glob import os diff --git a/nipype/pipeline/engine/tests/test_join.py b/nipype/pipeline/engine/tests/test_join.py index 2fe5f70564..c177ad24d3 100644 --- a/nipype/pipeline/engine/tests/test_join.py +++ b/nipype/pipeline/engine/tests/test_join.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for join expansion -""" +"""Tests for join expansion""" import pytest from .... import config diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 78483b6923..7ae8ce5b33 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine utils module -""" +"""Tests for the engine utils module""" import os from copy import deepcopy import pytest diff --git a/nipype/pipeline/engine/tests/test_workflows.py b/nipype/pipeline/engine/tests/test_workflows.py index 12d56de285..980b54fa28 100644 --- a/nipype/pipeline/engine/tests/test_workflows.py +++ b/nipype/pipeline/engine/tests/test_workflows.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine workflows module -""" +"""Tests for the engine workflows module""" from glob import glob import os from shutil import rmtree diff --git a/nipype/pipeline/plugins/condor.py b/nipype/pipeline/plugins/condor.py index 0fff477377..789eaecfab 100644 --- a/nipype/pipeline/plugins/condor.py +++ b/nipype/pipeline/plugins/condor.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via Condor -""" +"""Parallel workflow execution via Condor""" import os from time import sleep diff --git a/nipype/pipeline/plugins/dagman.py b/nipype/pipeline/plugins/dagman.py index 55f3f03bee..1c424c24ef 100644 --- a/nipype/pipeline/plugins/dagman.py +++ b/nipype/pipeline/plugins/dagman.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via Condor DAGMan -""" +"""Parallel workflow execution via Condor DAGMan""" import os import sys diff --git a/nipype/pipeline/plugins/debug.py b/nipype/pipeline/plugins/debug.py index 1dac35cf8f..4798e083bd 100644 --- a/nipype/pipeline/plugins/debug.py +++ b/nipype/pipeline/plugins/debug.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Debug plugin -""" +"""Debug plugin""" import networkx as nx from .base import PluginBase, logger diff --git a/nipype/pipeline/plugins/ipython.py b/nipype/pipeline/plugins/ipython.py index f52b3e6282..2c80eb4655 100644 --- a/nipype/pipeline/plugins/ipython.py +++ b/nipype/pipeline/plugins/ipython.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Parallel workflow execution via IPython controller -""" +"""Parallel workflow execution via IPython controller""" from pickle import dumps import sys diff --git a/nipype/pipeline/plugins/linear.py b/nipype/pipeline/plugins/linear.py index 93029ee1b9..aa29a5951b 100644 --- a/nipype/pipeline/plugins/linear.py +++ b/nipype/pipeline/plugins/linear.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Local serial workflow execution -""" +"""Local serial workflow execution""" import os from .base import PluginBase, logger, report_crash, report_nodes_not_run, str2bool diff --git a/nipype/pipeline/plugins/lsf.py b/nipype/pipeline/plugins/lsf.py index cf334be051..4ca380dfaa 100644 --- a/nipype/pipeline/plugins/lsf.py +++ b/nipype/pipeline/plugins/lsf.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via LSF -""" +"""Parallel workflow execution via LSF""" import os import re diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index 9b27b47383..086ee4430c 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -338,7 +338,9 @@ def _send_procs_to_workers(self, updatehash=False, graph=None): cached, updated = self.procs[jobid].is_cached() # updatehash and run_without_submitting are also run locally - if (cached and updatehash and not updated) or self.procs[jobid].run_without_submitting: + if (cached and updatehash and not updated) or self.procs[ + jobid + ].run_without_submitting: logger.debug("Running node %s on master thread", self.procs[jobid]) try: self.procs[jobid].run(updatehash=updatehash) diff --git a/nipype/pipeline/plugins/oar.py b/nipype/pipeline/plugins/oar.py index df56391bae..b9c4a050ab 100644 --- a/nipype/pipeline/plugins/oar.py +++ b/nipype/pipeline/plugins/oar.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via OAR http://oar.imag.fr -""" +"""Parallel workflow execution via OAR http://oar.imag.fr""" import os import stat diff --git a/nipype/pipeline/plugins/pbs.py b/nipype/pipeline/plugins/pbs.py index d967af0bed..01c80efc5a 100644 --- a/nipype/pipeline/plugins/pbs.py +++ b/nipype/pipeline/plugins/pbs.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via PBS/Torque -""" +"""Parallel workflow execution via PBS/Torque""" import os from time import sleep diff --git a/nipype/pipeline/plugins/pbsgraph.py b/nipype/pipeline/plugins/pbsgraph.py index 4b245dedb7..0cb925af38 100644 --- a/nipype/pipeline/plugins/pbsgraph.py +++ b/nipype/pipeline/plugins/pbsgraph.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via PBS/Torque -""" +"""Parallel workflow execution via PBS/Torque""" import os import sys diff --git a/nipype/pipeline/plugins/sge.py b/nipype/pipeline/plugins/sge.py index 38079e947d..ce8e046f01 100644 --- a/nipype/pipeline/plugins/sge.py +++ b/nipype/pipeline/plugins/sge.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via SGE -""" +"""Parallel workflow execution via SGE""" import os import pwd diff --git a/nipype/pipeline/plugins/sgegraph.py b/nipype/pipeline/plugins/sgegraph.py index 5cd1c7bfb7..3b33b73dee 100644 --- a/nipype/pipeline/plugins/sgegraph.py +++ b/nipype/pipeline/plugins/sgegraph.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via SGE -""" +"""Parallel workflow execution via SGE""" import os import sys diff --git a/nipype/pipeline/plugins/slurmgraph.py b/nipype/pipeline/plugins/slurmgraph.py index c74ab05a87..05824b016b 100644 --- a/nipype/pipeline/plugins/slurmgraph.py +++ b/nipype/pipeline/plugins/slurmgraph.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via SLURM -""" +"""Parallel workflow execution via SLURM""" import os import sys diff --git a/nipype/pipeline/plugins/somaflow.py b/nipype/pipeline/plugins/somaflow.py index 2105204979..16bedaab23 100644 --- a/nipype/pipeline/plugins/somaflow.py +++ b/nipype/pipeline/plugins/somaflow.py @@ -1,5 +1,4 @@ -"""Parallel workflow execution via PBS/Torque -""" +"""Parallel workflow execution via PBS/Torque""" import os import sys diff --git a/nipype/pipeline/plugins/tests/test_base.py b/nipype/pipeline/plugins/tests/test_base.py index 43471a7d64..11acb369e9 100644 --- a/nipype/pipeline/plugins/tests/test_base.py +++ b/nipype/pipeline/plugins/tests/test_base.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine module -""" +"""Tests for the engine module""" import numpy as np import scipy.sparse as ssp diff --git a/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py b/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py index 2f35579a40..cd79fbe31c 100644 --- a/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py +++ b/nipype/pipeline/plugins/tests/test_legacymultiproc_nondaemon.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Testing module for functions and classes from multiproc.py -""" +"""Testing module for functions and classes from multiproc.py""" # Import packages import os import sys diff --git a/nipype/pipeline/plugins/tests/test_tools.py b/nipype/pipeline/plugins/tests/test_tools.py index e21ef42072..e352253dbe 100644 --- a/nipype/pipeline/plugins/tests/test_tools.py +++ b/nipype/pipeline/plugins/tests/test_tools.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Tests for the engine module -""" +"""Tests for the engine module""" import re from unittest import mock diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index bce3eb82da..7e066b0ea3 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Common graph operations for execution -""" +"""Common graph operations for execution""" import os import getpass from socket import gethostname diff --git a/nipype/testing/tests/test_utils.py b/nipype/testing/tests/test_utils.py index 9217d54694..c3b1cae638 100644 --- a/nipype/testing/tests/test_utils.py +++ b/nipype/testing/tests/test_utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Test testing utilities -""" +"""Test testing utilities""" import os import subprocess diff --git a/nipype/testing/utils.py b/nipype/testing/utils.py index 71a75a41c7..96a94d6564 100644 --- a/nipype/testing/utils.py +++ b/nipype/testing/utils.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Additional handy utilities for testing -""" +"""Additional handy utilities for testing""" import os import time import shutil diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index 52558f59f0..4916cbacef 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Miscellaneous file manipulation functions -""" +"""Miscellaneous file manipulation functions""" import sys import pickle import errno diff --git a/nipype/utils/matlabtools.py b/nipype/utils/matlabtools.py index ea06cd4126..d871885c06 100644 --- a/nipype/utils/matlabtools.py +++ b/nipype/utils/matlabtools.py @@ -1,6 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -""" Useful Functions for working with matlab""" +"""Useful Functions for working with matlab""" # Stdlib imports import os diff --git a/nipype/utils/misc.py b/nipype/utils/misc.py index ed8a539e66..3f76fbab3c 100644 --- a/nipype/utils/misc.py +++ b/nipype/utils/misc.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Miscellaneous utility functions -""" +"""Miscellaneous utility functions""" import os import sys import re diff --git a/nipype/utils/subprocess.py b/nipype/utils/subprocess.py index acd6b63256..2fa9e52c3b 100644 --- a/nipype/utils/subprocess.py +++ b/nipype/utils/subprocess.py @@ -1,7 +1,6 @@ # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- # vi: set ft=python sts=4 ts=4 sw=4 et: -"""Miscellaneous utility functions -""" +"""Miscellaneous utility functions""" import os import sys import gc From 0c67bb385bec4b0cb1d64010cb8b3c5b6a6e09c4 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 21 Feb 2025 14:07:40 -0500 Subject: [PATCH 26/34] chore: Update .git-blame-ignore-revs --- .git-blame-ignore-revs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 826a62fe9f..6f762e919b 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,3 +1,7 @@ +# Fri Feb 21 14:06:53 2025 -0500 - markiewicz@stanford.edu - sty: black [ignore-rev] +8ed2b2306aeb7d89de4958b5293223ffe27a4f34 +# Tue Apr 13 10:16:17 2021 -0400 - markiewicz@stanford.edu - STY: black +b1690d5beb391e08c1e5463f1e3c641cf1e9f58e # Thu Oct 31 10:01:38 2024 -0400 - effigies@gmail.com - STY: black [ignore-rev] bd0d5856d183ba3918eda31f80db3b1d4387c55c # Thu Mar 21 13:34:09 2024 -0400 - effigies@gmail.com - STY: black [ignore-rev] From 74149e03713def8a1552f12a72a73902e4736327 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 26 Feb 2025 07:31:26 -0500 Subject: [PATCH 27/34] fix: Allow nipype.sphinx.ext.apidoc Config to work with Sphinx 8.2.1+ --- nipype/sphinxext/apidoc/__init__.py | 36 +++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/nipype/sphinxext/apidoc/__init__.py b/nipype/sphinxext/apidoc/__init__.py index 151011bdfc..429848d2f5 100644 --- a/nipype/sphinxext/apidoc/__init__.py +++ b/nipype/sphinxext/apidoc/__init__.py @@ -2,6 +2,9 @@ # vi: set ft=python sts=4 ts=4 sw=4 et: """Settings for sphinxext.interfaces and connection to sphinx-apidoc.""" import re +from packaging.version import Version + +import sphinx from sphinx.ext.napoleon import ( Config as NapoleonConfig, _patch_python_domain, @@ -39,13 +42,24 @@ class Config(NapoleonConfig): """ - _config_values = { - "nipype_skip_classes": ( - ["Tester", "InputSpec", "OutputSpec", "Numpy", "NipypeTester"], - "env", - ), - **NapoleonConfig._config_values, - } + if Version(sphinx.__version__) >= Version("8.2.1"): + _config_values = ( + ( + "nipype_skip_classes", + ["Tester", "InputSpec", "OutputSpec", "Numpy", "NipypeTester"], + "env", + frozenset({list[str]}), + ), + *NapoleonConfig._config_values, + ) + else: + _config_values = { + "nipype_skip_classes": ( + ["Tester", "InputSpec", "OutputSpec", "Numpy", "NipypeTester"], + "env", + ), + **NapoleonConfig._config_values, + } def setup(app): @@ -82,8 +96,12 @@ def setup(app): app.connect("autodoc-process-docstring", _process_docstring) app.connect("autodoc-skip-member", _skip_member) - for name, (default, rebuild) in Config._config_values.items(): - app.add_config_value(name, default, rebuild) + if Version(sphinx.__version__) >= Version("8.2.1"): + for name, default, rebuild, types in Config._config_values: + app.add_config_value(name, default, rebuild, types=types) + else: + for name, (default, rebuild) in Config._config_values.items(): + app.add_config_value(name, default, rebuild) return {"version": __version__, "parallel_read_safe": True} From 4dfbbbee48fb92927dacb1905dc77bb5ed7382e4 Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Tue, 4 Mar 2025 18:43:14 +0100 Subject: [PATCH 28/34] reimplementation of gpu_count --- nipype/pipeline/plugins/multiproc.py | 2 +- nipype/pipeline/plugins/tools.py | 10 ----- nipype/utils/gpu_count.py | 55 ++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 nipype/utils/gpu_count.py diff --git a/nipype/pipeline/plugins/multiproc.py b/nipype/pipeline/plugins/multiproc.py index b403749ff9..be0e006229 100644 --- a/nipype/pipeline/plugins/multiproc.py +++ b/nipype/pipeline/plugins/multiproc.py @@ -21,7 +21,7 @@ from ...utils.profiler import get_system_total_memory_gb from ..engine import MapNode from .base import DistributedPluginBase -from .tools import gpu_count +from ...utils.gpu_count import gpu_count try: from textwrap import indent diff --git a/nipype/pipeline/plugins/tools.py b/nipype/pipeline/plugins/tools.py index 37c841a208..7e066b0ea3 100644 --- a/nipype/pipeline/plugins/tools.py +++ b/nipype/pipeline/plugins/tools.py @@ -175,13 +175,3 @@ def create_pyscript(node, updatehash=False, store_exception=True): with open(pyscript, "w") as fp: fp.writelines(cmdstr) return pyscript - - -def gpu_count(): - n_gpus = 1 - try: - import GPUtil - except ImportError: - return 1 - else: - return len(GPUtil.getGPUs()) diff --git a/nipype/utils/gpu_count.py b/nipype/utils/gpu_count.py new file mode 100644 index 0000000000..57f749baa2 --- /dev/null +++ b/nipype/utils/gpu_count.py @@ -0,0 +1,55 @@ +# -*- DISCLAIMER: this file contains code derived from gputil (https://github.com/anderskm/gputil) +# and therefore is distributed under to the following license: +# +# MIT License +# +# Copyright (c) 2017 anderskm +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import platform +import shutil +from subprocess import Popen, PIPE +import os + + +def gpu_count(): + try: + if platform.system() == "Windows": + nvidia_smi = shutil.which('nvidia-smi') + if nvidia_smi is None: + nvidia_smi = ( + "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" + % os.environ['systemdrive'] + ) + else: + nvidia_smi = "nvidia-smi" + + p = Popen( + [nvidia_smi, "--query-gpu=name", "--format=csv,noheader,nounits"], + stdout=PIPE, + ) + stdout, stderror = p.communicate() + + output = stdout.decode('UTF-8') + lines = output.split(os.linesep) + num_devices = len(lines) - 1 + return num_devices + except: + return 0 From 151facd7ad97615a6ebe1c5907cec87dada7dffa Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Wed, 5 Mar 2025 19:41:20 +0100 Subject: [PATCH 29/34] remove gputil requirement --- nipype/info.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nipype/info.py b/nipype/info.py index 072a0330ef..729689ae5d 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -149,7 +149,6 @@ def get_nipype_gitversion(): "acres", "etelemetry>=0.3.1", "looseversion!=1.2", - "gputil>=1.4.0", "puremagic", ] From e23315c4e787daa932472d2edfdb65b25b461527 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 6 Mar 2025 11:06:47 -0500 Subject: [PATCH 30/34] chore: Remove direct URL for traits on py313 --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 9704158bec..571b93628b 100644 --- a/tox.ini +++ b/tox.ini @@ -68,8 +68,6 @@ pass_env = CLICOLOR CLICOLOR_FORCE PYTHON_GIL -deps = - py313: traits @ git+https://github.com/enthought/traits.git@10954eb extras = tests full: doc From c18991264169a3787df7d16a072d5fbb23b79acf Mon Sep 17 00:00:00 2001 From: mauriliogenovese <125388969+mauriliogenovese@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:15:38 +0100 Subject: [PATCH 31/34] code cleanup --- nipype/utils/gpu_count.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/nipype/utils/gpu_count.py b/nipype/utils/gpu_count.py index 57f749baa2..70eb6d724e 100644 --- a/nipype/utils/gpu_count.py +++ b/nipype/utils/gpu_count.py @@ -25,31 +25,22 @@ import platform import shutil -from subprocess import Popen, PIPE +import subprocess import os def gpu_count(): + nvidia_smi = shutil.which('nvidia-smi') + if nvidia_smi is None and platform.system() == "Windows": + nvidia_smi = f'{os.environ["systemdrive"]}\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe' + if nvidia_smi is None: + return 0 try: - if platform.system() == "Windows": - nvidia_smi = shutil.which('nvidia-smi') - if nvidia_smi is None: - nvidia_smi = ( - "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" - % os.environ['systemdrive'] - ) - else: - nvidia_smi = "nvidia-smi" - - p = Popen( + p = subprocess.run( [nvidia_smi, "--query-gpu=name", "--format=csv,noheader,nounits"], - stdout=PIPE, + stdout=subprocess.PIPE, + text=True, ) - stdout, stderror = p.communicate() - - output = stdout.decode('UTF-8') - lines = output.split(os.linesep) - num_devices = len(lines) - 1 - return num_devices - except: + except (OSError, UnicodeDecodeError): return 0 + return len(p.stdout.splitlines()) From f2119d663257a5de8aac5255b88b4304198061cc Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 19 Mar 2025 19:10:02 -0400 Subject: [PATCH 32/34] doc: 1.10.0 changelog --- doc/changelog/1.X.X-changelog.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/changelog/1.X.X-changelog.rst b/doc/changelog/1.X.X-changelog.rst index a51ef7f13e..6373d501f7 100644 --- a/doc/changelog/1.X.X-changelog.rst +++ b/doc/changelog/1.X.X-changelog.rst @@ -1,3 +1,24 @@ +1.10.0 (March 20, 2025) +======================= + +New feature release in the 1.10.x series. + +This release adds GPUs to multiprocess resource management. +In general, no changes to existing code should be required if the GPU-enabled +interface has a ``use_gpu`` input. +The ``n_gpu_procs`` can be used to set the number of GPU processes that may +be run in parallel, which will override the default of GPUs identified by +``nvidia-smi``, or 1 if no GPUs are detected. + + * FIX: Reimplement ``gpu_count()`` (https://github.com/nipy/nipype/pull/3718) + * FIX: Avoid 0D array in ``algorithms.misc.merge_rois`` (https://github.com/nipy/nipype/pull/3713) + * FIX: Allow nipype.sphinx.ext.apidoc Config to work with Sphinx 8.2.1+ (https://github.com/nipy/nipype/pull/3716) + * FIX: Resolve crashes when running workflows with updatehash=True (https://github.com/nipy/nipype/pull/3709) + * ENH: Support for gpu queue (https://github.com/nipy/nipype/pull/3642) + * ENH: Update to .wci.yml (https://github.com/nipy/nipype/pull/3708) + * ENH: Add Workflow Community Initiative (WCI) descriptor (https://github.com/nipy/nipype/pull/3608) + + 1.9.2 (December 17, 2024) ========================= From 1c9953f3397d434fb7296c189b2c828490612d66 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Wed, 19 Mar 2025 19:17:56 -0400 Subject: [PATCH 33/34] Update doc/changelog/1.X.X-changelog.rst --- doc/changelog/1.X.X-changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog/1.X.X-changelog.rst b/doc/changelog/1.X.X-changelog.rst index 6373d501f7..e31e508edf 100644 --- a/doc/changelog/1.X.X-changelog.rst +++ b/doc/changelog/1.X.X-changelog.rst @@ -1,4 +1,4 @@ -1.10.0 (March 20, 2025) +1.10.0 (March 19, 2025) ======================= New feature release in the 1.10.x series. From 15801539f05a204c0ef1ee662ecbe5324f262408 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 19 Mar 2025 19:20:28 -0400 Subject: [PATCH 34/34] rel: 1.10.0 --- nipype/info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/info.py b/nipype/info.py index 729689ae5d..c546e4c2fc 100644 --- a/nipype/info.py +++ b/nipype/info.py @@ -5,7 +5,7 @@ # nipype version information # Remove .dev0 for release -__version__ = "1.9.3.dev0" +__version__ = "1.10.0" def get_nipype_gitversion():