From 1d7bb5e954d273a3e1969a89d923ffa6caf07b7f Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 01:00:17 -0700 Subject: [PATCH 01/13] FIX: Confused traits after loading results file ... when a file exists with name coinciding with the value of some output trait (e.g., a file './2' and an integer output v = 2). --- nipype/interfaces/base/core.py | 84 ++++++++++++++--------- nipype/interfaces/base/tests/test_core.py | 62 ++++++++++++++++- 2 files changed, 111 insertions(+), 35 deletions(-) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index 0011c925dd..b23916de13 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -33,12 +33,12 @@ from ...utils.provenance import write_provenance from ...utils.misc import str2bool, rgetcwd from ...utils.filemanip import (FileNotFoundError, split_filename, - which, get_dependencies) + which, get_dependencies, Path) from ...utils.subprocess import run_command from ...external.due import due -from .traits_extension import traits, isdefined +from .traits_extension import traits, isdefined, BasePath from .specs import (BaseInterfaceInputSpec, CommandLineInputSpec, StdOutCommandLineInputSpec, MpiCommandLineInputSpec, get_filecopy_info) @@ -110,7 +110,7 @@ def run(self): """Execute the command.""" raise NotImplementedError - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): """Called to populate outputs""" raise NotImplementedError @@ -204,10 +204,10 @@ def _check_requires(self, spec, name, value): ] if any(values) and isdefined(value): if len(values) > 1: - fmt = ("%s requires values for inputs %s because '%s' is set. " + fmt = ("%s requires values for inputs %s because '%s' is set. " "For a list of required inputs, see %s.help()") else: - fmt = ("%s requires a value for input %s because '%s' is set. " + fmt = ("%s requires a value for input %s because '%s' is set. " "For a list of required inputs, see %s.help()") msg = fmt % (self.__class__.__name__, ', '.join("'%s'" % req for req in spec.requires), @@ -299,7 +299,7 @@ def _duecredit_cite(self): r['path'] = self.__module__ due.cite(**r) - def run(self, cwd=None, ignore_exception=None, **inputs): + def run(self, cwd=None, ignore_exception=None, rebase_cwd=None, **inputs): """Execute this interface. This interface will not raise an exception if runtime.returncode is @@ -327,6 +327,9 @@ def run(self, cwd=None, ignore_exception=None, **inputs): if cwd is None: cwd = syscwd + if rebase_cwd: + rebase_cwd = cwd + os.chdir(cwd) # Change to the interface wd enable_rm = config.resource_monitor and self.resource_monitor @@ -375,7 +378,7 @@ def run(self, cwd=None, ignore_exception=None, **inputs): runtime = self._pre_run_hook(runtime) runtime = self._run_interface(runtime) runtime = self._post_run_hook(runtime) - outputs = self.aggregate_outputs(runtime) + outputs = self.aggregate_outputs(runtime, rebase_cwd=rebase_cwd) except Exception as e: import traceback # Retrieve the maximum info fast @@ -449,34 +452,47 @@ def _list_outputs(self): else: return None - def aggregate_outputs(self, runtime=None, needed_outputs=None): - """ Collate expected outputs and check for existence - """ + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + """Collate expected outputs and check for existence.""" + outputs = self._outputs() # Generate an output spec object + if needed_outputs is not None and not needed_outputs: + return outputs - predicted_outputs = self._list_outputs() - outputs = self._outputs() - if predicted_outputs: - _unavailable_outputs = [] - if outputs: - _unavailable_outputs = \ - self._check_version_requirements(self._outputs()) - for key, val in list(predicted_outputs.items()): - if needed_outputs and key not in needed_outputs: - continue - if key in _unavailable_outputs: - raise KeyError(('Output trait %s not available in version ' - '%s of interface %s. Please inform ' - 'developers.') % (key, self.version, - self.__class__.__name__)) - try: - setattr(outputs, key, val) - except TraitError as error: - if getattr(error, 'info', - 'default').startswith('an existing'): - msg = ("File/Directory '%s' not found for %s output " - "'%s'." % (val, self.__class__.__name__, key)) - raise FileNotFoundError(msg) - raise error + predicted_outputs = self._list_outputs() # Predictions from _list_outputs + if not predicted_outputs: + return outputs + + # Precalculate the list of output trait names that should be + # aggregated + aggregate_names = set(predicted_outputs.keys()) + if needed_outputs: + aggregate_names = set(needed_outputs).intersection(aggregate_names) + + if outputs and aggregate_names: + _na_outputs = self._check_version_requirements(outputs) + na_names = aggregate_names.intersection(set(_na_outputs)) + if na_names: + raise TypeError("""\ +Output trait(s) %s not available in version %s of interface %s.\ +""" % (', '.join(na_names), self.version, self.__class__.__name__)) + + for key in aggregate_names: + val = predicted_outputs[key] + + # All the magic to fix #2944 resides here: + # TODO: test compounds/lists/tuples + if rebase_cwd and outputs.trait(key).is_trait_type(BasePath): + val = Path(val) + if val.is_absolute(): + val = Path(val).relative_to(rebase_cwd) + try: + setattr(outputs, key, val) + except TraitError as error: + if 'an existing' in getattr(error, 'info', 'default'): + msg = "No such file or directory for output '%s' of a %s interface" % \ + (key, self.__class__.__name__) + raise OSError(2, msg, val) + raise error return outputs diff --git a/nipype/interfaces/base/tests/test_core.py b/nipype/interfaces/base/tests/test_core.py index 265edc444f..c95d46f6e5 100644 --- a/nipype/interfaces/base/tests/test_core.py +++ b/nipype/interfaces/base/tests/test_core.py @@ -10,6 +10,7 @@ import pytest from .... import config +from ....utils.filemanip import Path from ....testing import example_data from ... import base as nib from ..support import _inputs_help @@ -289,7 +290,7 @@ def _list_outputs(self): return {'foo': 1} obj = DerivedInterface1() - with pytest.raises(KeyError): + with pytest.raises(TypeError): obj.run() @@ -529,3 +530,62 @@ def _run_interface(self, runtime): with pytest.raises(RuntimeError): BrokenRuntime().run() + + +def test_aggregate_outputs(tmpdir): + tmpdir.chdir() + b_value = '%s' % tmpdir.join('filename.txt') + c_value = '%s' % tmpdir.join('sub') + + class TestInterface(nib.BaseInterface): + class input_spec(nib.TraitedSpec): + a = nib.traits.Any() + class output_spec(nib.TraitedSpec): + b = nib.File() + c = nib.Directory(exists=True) + d = nib.File() + + def _list_outputs(self): + return {'b': b_value, 'c': c_value, 'd': './log.txt'} + + # Test aggregate_outputs without needed_outputs + outputs = TestInterface().aggregate_outputs(needed_outputs=[]) + assert outputs.b is nib.Undefined + + # Test that only those in needed_outputs are returned + outputs = TestInterface().aggregate_outputs( + needed_outputs=['b', 'd']) + assert outputs.c is nib.Undefined + assert outputs.b == b_value + assert Path(outputs.b).is_absolute() + assert not Path(outputs.d).is_absolute() + + # Test that traits are actually validated at aggregation + with pytest.raises(OSError): + outputs = TestInterface().aggregate_outputs( + needed_outputs=['b', 'c'], rebase_cwd='%s' % tmpdir) + + # Test that rebase_cwd actually returns relative paths + tmpdir.mkdir('sub') + outputs = TestInterface().aggregate_outputs(rebase_cwd='%s' % tmpdir) + + assert outputs.b == 'filename.txt' + assert not Path(outputs.b).is_absolute() + assert not Path(outputs.c).is_absolute() + assert not Path(outputs.d).is_absolute() + + +def test_aggregate_outputs_compounds(tmpdir): + tmpdir.chdir() + + class TestInterface(nib.BaseInterface): + class input_spec(nib.TraitedSpec): + a = nib.traits.Any() + class output_spec(nib.TraitedSpec): + b = nib.traits.Tuple(nib.File(), nib.File()) + c = nib.List(nib.File()) + d = nib.Either(nib.File(), nib.Float()) + e = nib.OutputMultiObject(nib.File()) + + def _list_outputs(self): + return {'b': b_value, 'c': c_value, 'd': './log.txt'} From 725c0a1e35adca7cd68624b6d8c6a0dd0c612bd4 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 18:40:24 -0700 Subject: [PATCH 02/13] FIX: Resolving absolute to relative paths in output Building a solution to #2944, starting from a refactor of ``aggregate_outputs`` to be robuster and perform the referrencing when requested via the new arguiment ``rebase_cwd``. --- nipype/interfaces/base/core.py | 18 ++--- nipype/interfaces/base/tests/test_core.py | 23 +++++- .../base/tests/test_traits_extension.py | 71 ++++++++++++++++++ nipype/interfaces/base/traits_extension.py | 74 +++++++++++++++++++ nipype/utils/filemanip.py | 7 +- 5 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 nipype/interfaces/base/tests/test_traits_extension.py diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index b23916de13..bd6293f7dd 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -32,13 +32,13 @@ from ... import config, logging, LooseVersion from ...utils.provenance import write_provenance from ...utils.misc import str2bool, rgetcwd -from ...utils.filemanip import (FileNotFoundError, split_filename, - which, get_dependencies, Path) +from ...utils.filemanip import (split_filename, which, get_dependencies, + FileNotFoundError) from ...utils.subprocess import run_command from ...external.due import due -from .traits_extension import traits, isdefined, BasePath +from .traits_extension import traits, isdefined, rebase_path_traits from .specs import (BaseInterfaceInputSpec, CommandLineInputSpec, StdOutCommandLineInputSpec, MpiCommandLineInputSpec, get_filecopy_info) @@ -445,8 +445,7 @@ def run(self, cwd=None, ignore_exception=None, rebase_cwd=None, **inputs): return results def _list_outputs(self): - """ List the expected outputs - """ + """List the expected outputs.""" if self.output_spec: raise NotImplementedError else: @@ -480,18 +479,15 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): val = predicted_outputs[key] # All the magic to fix #2944 resides here: - # TODO: test compounds/lists/tuples - if rebase_cwd and outputs.trait(key).is_trait_type(BasePath): - val = Path(val) - if val.is_absolute(): - val = Path(val).relative_to(rebase_cwd) + if rebase_cwd: + val = rebase_path_traits(outputs, key, val, rebase_cwd) try: setattr(outputs, key, val) except TraitError as error: if 'an existing' in getattr(error, 'info', 'default'): msg = "No such file or directory for output '%s' of a %s interface" % \ (key, self.__class__.__name__) - raise OSError(2, msg, val) + raise FileNotFoundError(val, message=msg) raise error return outputs diff --git a/nipype/interfaces/base/tests/test_core.py b/nipype/interfaces/base/tests/test_core.py index c95d46f6e5..d27843edd6 100644 --- a/nipype/interfaces/base/tests/test_core.py +++ b/nipype/interfaces/base/tests/test_core.py @@ -578,14 +578,29 @@ def _list_outputs(self): def test_aggregate_outputs_compounds(tmpdir): tmpdir.chdir() + outputs_dict = { + 'b': tuple(['/some/folder/f%d.txt' % d + for d in range(1, 3)]), + 'c': ['/some/folder/f%d.txt' % d for d in range(1, 5)], + 'd': 2.0, + 'e': ['/some/folder/f%d.txt' % d for d in range(1, 4)] + } + class TestInterface(nib.BaseInterface): class input_spec(nib.TraitedSpec): a = nib.traits.Any() class output_spec(nib.TraitedSpec): - b = nib.traits.Tuple(nib.File(), nib.File()) - c = nib.List(nib.File()) - d = nib.Either(nib.File(), nib.Float()) + b = nib.traits.Tuple(nib.File(extensions=['.txt']), + nib.File(extensions=['.txt'])) + c = nib.traits.List(nib.File()) + d = nib.traits.Either(nib.File(), nib.traits.Float()) e = nib.OutputMultiObject(nib.File()) def _list_outputs(self): - return {'b': b_value, 'c': c_value, 'd': './log.txt'} + return outputs_dict + + outputs = TestInterface().aggregate_outputs(rebase_cwd='%s' % tmpdir) + assert outputs.d == 2.0 + assert outputs.b == ('f1.txt', 'f2.txt') + assert outputs.e == ['f%d.txt' % (d + 1) + for d in range(len(outputs_dict['e']))] diff --git a/nipype/interfaces/base/tests/test_traits_extension.py b/nipype/interfaces/base/tests/test_traits_extension.py new file mode 100644 index 0000000000..df1c69d47d --- /dev/null +++ b/nipype/interfaces/base/tests/test_traits_extension.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- +# vi: set ft=python sts=4 ts=4 sw=4 et: +from __future__ import print_function, unicode_literals + +from ... import base as nib +from ..traits_extension import rebase_path_traits, Path + + +class _test_spec(nib.TraitedSpec): + a = nib.traits.File() + b = nib.traits.Tuple(nib.File(), + nib.File()) + c = nib.traits.List(nib.File()) + d = nib.traits.Either(nib.File(), nib.traits.Float()) + e = nib.OutputMultiObject(nib.File()) + f = nib.traits.Dict(nib.Str, nib.File()) + g = nib.traits.Either(nib.File, nib.Str) + h = nib.Str + + +def test_rebase_path_traits(): + """Check rebase_path_traits.""" + spec = _test_spec() + + a = rebase_path_traits( + spec.trait('a'), '/some/path/f1.txt', '/some/path') + assert '%s' % a == 'f1.txt' + + b = rebase_path_traits( + spec.trait('b'), ('/some/path/f1.txt', '/some/path/f2.txt'), '/some/path') + assert b == (Path('f1.txt'), Path('f2.txt')) + + c = rebase_path_traits( + spec.trait('c'), ['/some/path/f1.txt', '/some/path/f2.txt', '/some/path/f3.txt'], + '/some/path') + assert c == [Path('f1.txt'), Path('f2.txt'), Path('f3.txt')] + + e = rebase_path_traits( + spec.trait('e'), ['/some/path/f1.txt', '/some/path/f2.txt', '/some/path/f3.txt'], + '/some/path') + assert e == [Path('f1.txt'), Path('f2.txt'), Path('f3.txt')] + + f = rebase_path_traits( + spec.trait('f'), {'1': '/some/path/f1.txt'}, '/some/path') + assert f == {'1': Path('f1.txt')} + + d = rebase_path_traits( + spec.trait('d'), 2.0, '/some/path') + assert d == 2.0 + + d = rebase_path_traits( + spec.trait('d'), '/some/path/either.txt', '/some/path') + assert '%s' % d == 'either.txt' + + g = rebase_path_traits( + spec.trait('g'), 'some/path/either.txt', '/some/path') + assert '%s' % g == 'some/path/either.txt' + + g = rebase_path_traits( + spec.trait('g'), '/some/path/either.txt', '/some') + assert '%s' % g == 'path/either.txt' + + g = rebase_path_traits(spec.trait('g'), 'string', '/some') + assert '%s' % g == 'string' + + g = rebase_path_traits(spec.trait('g'), '2', '/some/path') + assert g == '2' # You dont want this one to be a Path + + h = rebase_path_traits(spec.trait('h'), '2', '/some/path') + assert h == '2' diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index de215beb96..82496438a3 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -30,6 +30,7 @@ import traits.api as traits from traits.trait_handlers import TraitType, NoDefaultSpecified from traits.trait_base import _Undefined +from traits.traits import _TraitMaker, trait_from from traits.api import Unicode from future import standard_library @@ -304,6 +305,11 @@ def validate(self, objekt, name, value, return_pathlike=False): return value +# Patch in traits these two new +traits.File = File +traits.Directory = Directory + + class ImageFile(File): """Defines a trait whose value must be a known neuroimaging file.""" @@ -465,3 +471,71 @@ class InputMultiObject(MultiObject): InputMultiPath = InputMultiObject OutputMultiPath = OutputMultiObject + + +class Tuple(traits.BaseTuple): + """Defines a new type of Tuple trait that reports inner types.""" + + def init_fast_validator(self, *args): + """Set up the C-level fast validator.""" + super(Tuple, self).init_fast_validator(*args) + self.fast_validate = args + + def inner_traits(self): + """Return the *inner trait* (or traits) for this trait.""" + return self.types + + +class PatchedEither(TraitType): + """Defines a trait whose value can be any of of a specified list of traits.""" + + def __init__(self, *traits, **metadata): + """Create a trait whose value can be any of of a specified list of traits.""" + metadata['alternatives'] = tuple(trait_from(t) for t in traits) + self.trait_maker = _TraitMaker( + metadata.pop("default", None), *traits, **metadata) + + def as_ctrait(self): + """Return a CTrait corresponding to the trait defined by this class.""" + return self.trait_maker.as_ctrait() + + +traits.Tuple = Tuple +traits.Either = PatchedEither + + +def rebase_path_traits(thistrait, value, cwd): + """Rebase a BasePath-derived trait given an interface spec.""" + if thistrait.is_trait_type(BasePath): + try: + value = Path(value) + except TypeError: + pass + else: + try: + value = Path(value).relative_to(cwd) + except ValueError: + pass + elif thistrait.is_trait_type(traits.List): + innertrait, = thistrait.inner_traits + if not isinstance(value, (list, tuple)): + value = rebase_path_traits(innertrait, value, cwd) + else: + value = [rebase_path_traits(innertrait, v, cwd) + for v in value] + elif thistrait.is_trait_type(traits.Dict): + _, innertrait = thistrait.inner_traits + value = {k: rebase_path_traits(innertrait, v, cwd) + for k, v in value.items()} + elif thistrait.is_trait_type(Tuple): + value = tuple([rebase_path_traits(subtrait, v, cwd) + for subtrait, v in zip(thistrait.inner_traits, value)]) + elif thistrait.alternatives: + is_str = [f.is_trait_type((traits.String, traits.BaseStr, traits.BaseBytes, Str)) + for f in thistrait.alternatives] + print(is_str) + if any(is_str) and isinstance(value, (bytes, str)) and not value.startswith('/'): + return value + for subtrait in thistrait.alternatives: + value = rebase_path_traits(subtrait, value, cwd) + return value diff --git a/nipype/utils/filemanip.py b/nipype/utils/filemanip.py index e19563ed54..4e649ca805 100644 --- a/nipype/utils/filemanip.py +++ b/nipype/utils/filemanip.py @@ -43,10 +43,11 @@ class FileNotFoundError(OSError): # noqa """Defines the exception for Python 2.""" - def __init__(self, path): + def __init__(self, path, errno=2, message=None): """Initialize the exception.""" - super(FileNotFoundError, self).__init__( - 2, 'No such file or directory', '%s' % path) + message = message or 'No such file or directory' + + super(FileNotFoundError, self).__init__(errno, message, '%s' % path) USING_PATHLIB2 = False From ce63d846bf583ad6b310dd92c96129e9bb6c9757 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 18:44:07 -0700 Subject: [PATCH 03/13] fix: rebase_path_traits correct arguments --- nipype/interfaces/base/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index bd6293f7dd..83c38d2198 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -480,7 +480,7 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): # All the magic to fix #2944 resides here: if rebase_cwd: - val = rebase_path_traits(outputs, key, val, rebase_cwd) + val = rebase_path_traits(outputs.trait(key), val, rebase_cwd) try: setattr(outputs, key, val) except TraitError as error: From fb684fb5844c0c736beb09552619b03da506638c Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 20:22:55 -0700 Subject: [PATCH 04/13] tst: add @effigies' new test --- nipype/pipeline/engine/tests/test_utils.py | 59 +++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index 4f4383f169..cca084fd09 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -16,7 +16,7 @@ from ....interfaces import base as nib from ....interfaces import utility as niu from .... import config -from ..utils import clean_working_directory, write_workflow_prov +from ..utils import clean_working_directory, write_workflow_prov, modify_paths class InputSpec(nib.TraitedSpec): @@ -224,3 +224,60 @@ def test_mapnode_crash3(tmpdir): wf.config["execution"]["crashdump_dir"] = os.getcwd() with pytest.raises(RuntimeError): wf.run(plugin='Linear') + + +class StrPathConfuser(nib.SimpleInterface): + class input_spec(nib.TraitedSpec): + in_str = nib.traits.String() + + class output_spec(nib.TraitedSpec): + out_tuple = nib.traits.Tuple(nib.File, nib.traits.String) + out_dict_path = nib.traits.Dict(nib.traits.String, nib.File(exists=True)) + out_dict_str = nib.traits.DictStrStr() + out_list = nib.traits.List(nib.traits.String) + out_str = nib.traits.String() + out_path = nib.File(exists=True) + + def _run_interface(self, runtime): + out_path = os.path.abspath(os.path.basename(self.inputs.in_str) + '_path') + open(out_path, 'w').close() + + self._results['out_str'] = self.inputs.in_str + self._results['out_path'] = out_path + self._results['out_tuple'] = (out_path, self.inputs.in_str) + self._results['out_dict_path'] = {self.inputs.in_str: out_path} + self._results['out_dict_str'] = {self.inputs.in_str: self.inputs.in_str} + self._results['out_list'] = [self.inputs.in_str] * 2 + return runtime + + +def test_modify_paths_bug(tmpdir): + """ + There was a bug in which, if the current working directory contained a file with the name + of an output String, the string would get transformed into a path, and generally wreak havoc. + + This attempts to replicate that condition, using an object with strings and paths in various + trait configurations, to ensure that the guards added resolve the issue. + + Please see https://github.com/nipy/nipype/issues/2944 for more details. + """ + tmpdir.chdir() + + spc = pe.Node(StrPathConfuser(in_str='2'), name='spc') + + open('2', 'w').close() + + results = spc.run() + + # Basic check that string was not manipulated and path exists + out_path = results.outputs.out_path + out_str = results.outputs.out_str + assert out_str == '2' + assert os.path.basename(out_path) == '2_path' + assert os.path.isfile(out_path) + + # Assert data structures pass through correctly + assert results.outputs.out_tuple == (out_path, out_str) + assert results.outputs.out_dict_path == {out_str: out_path} + assert results.outputs.out_dict_str == {out_str: out_str} + assert results.outputs.out_list == [out_str] * 2 From 6a8dcbbb5d4b6880a83a069dc04733ffd3f82d27 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 20:42:09 -0700 Subject: [PATCH 05/13] fix: rebase_cwd path in ``test_aggregate_outputs_compounds`` --- nipype/interfaces/base/tests/test_core.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/base/tests/test_core.py b/nipype/interfaces/base/tests/test_core.py index d27843edd6..80a9b2a8a7 100644 --- a/nipype/interfaces/base/tests/test_core.py +++ b/nipype/interfaces/base/tests/test_core.py @@ -575,9 +575,7 @@ def _list_outputs(self): assert not Path(outputs.d).is_absolute() -def test_aggregate_outputs_compounds(tmpdir): - tmpdir.chdir() - +def test_aggregate_outputs_compounds(): outputs_dict = { 'b': tuple(['/some/folder/f%d.txt' % d for d in range(1, 3)]), @@ -599,7 +597,7 @@ class output_spec(nib.TraitedSpec): def _list_outputs(self): return outputs_dict - outputs = TestInterface().aggregate_outputs(rebase_cwd='%s' % tmpdir) + outputs = TestInterface().aggregate_outputs(rebase_cwd='/some/folder') assert outputs.d == 2.0 assert outputs.b == ('f1.txt', 'f2.txt') assert outputs.e == ['f%d.txt' % (d + 1) From d038dccd832b6c9f686ed018cc9cf58efbf4fc12 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 21:07:01 -0700 Subject: [PATCH 06/13] fix: update all aggregate_outputs --- nipype/interfaces/afni/preprocess.py | 20 +++++++------------- nipype/interfaces/afni/utils.py | 8 ++++---- nipype/interfaces/ants/registration.py | 2 +- nipype/interfaces/freesurfer/preprocess.py | 4 ++-- nipype/interfaces/freesurfer/utils.py | 5 +++-- nipype/interfaces/fsl/model.py | 2 +- nipype/interfaces/fsl/preprocess.py | 4 ++-- nipype/interfaces/fsl/utils.py | 5 +++-- nipype/interfaces/niftyseg/label_fusion.py | 7 ++++++- nipype/interfaces/spm/model.py | 11 +++++------ 10 files changed, 34 insertions(+), 34 deletions(-) diff --git a/nipype/interfaces/afni/preprocess.py b/nipype/interfaces/afni/preprocess.py index ee81ee7e2f..a9775fb763 100644 --- a/nipype/interfaces/afni/preprocess.py +++ b/nipype/interfaces/afni/preprocess.py @@ -1000,7 +1000,7 @@ class ClipLevel(AFNICommandBase): input_spec = ClipLevelInputSpec output_spec = ClipLevelOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() @@ -2115,21 +2115,15 @@ class Seg(AFNICommandBase): input_spec = SegInputSpec output_spec = AFNICommandOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): - - import glob - - outputs = self._outputs() + def _list_outputs(self): + from nipype.utils.filemanip import Path + prefix = 'Segsy' if isdefined(self.inputs.prefix): - outfile = os.path.join(os.getcwd(), self.inputs.prefix, - 'Classes+*.BRIK') - else: - outfile = os.path.join(os.getcwd(), 'Segsy', 'Classes+*.BRIK') - - outputs.out_file = glob.glob(outfile)[0] + prefix = self.inputs.prefix - return outputs + return {'out_file': str( + sorted(Path() / prefix).glob('Classes+*.BRIK')[0])} class SkullStripInputSpec(AFNICommandInputSpec): diff --git a/nipype/interfaces/afni/utils.py b/nipype/interfaces/afni/utils.py index e16e1bbd79..125d2fc2d1 100644 --- a/nipype/interfaces/afni/utils.py +++ b/nipype/interfaces/afni/utils.py @@ -210,9 +210,9 @@ class Autobox(AFNICommand): input_spec = AutoboxInputSpec output_spec = AutoboxOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = super(Autobox, self).aggregate_outputs( - runtime, needed_outputs) + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(Autobox, self).aggregate_outputs(runtime, needed_outputs, + rebase_cwd=rebase_cwd) pattern = 'x=(?P-?\d+)\.\.(?P-?\d+) '\ 'y=(?P-?\d+)\.\.(?P-?\d+) '\ 'z=(?P-?\d+)\.\.(?P-?\d+)' @@ -286,7 +286,7 @@ class BrickStat(AFNICommandBase): input_spec = BrickStatInputSpec output_spec = BrickStatOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() diff --git a/nipype/interfaces/ants/registration.py b/nipype/interfaces/ants/registration.py index c08c7646a9..0cb8bcba10 100644 --- a/nipype/interfaces/ants/registration.py +++ b/nipype/interfaces/ants/registration.py @@ -1487,7 +1487,7 @@ def _format_arg(self, opt, spec, val): return self._mask_constructor() return super(MeasureImageSimilarity, self)._format_arg(opt, spec, val) - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() stdout = runtime.stdout.split('\n') outputs.similarity = float(stdout[0]) diff --git a/nipype/interfaces/freesurfer/preprocess.py b/nipype/interfaces/freesurfer/preprocess.py index 2941968f85..6577b1547e 100644 --- a/nipype/interfaces/freesurfer/preprocess.py +++ b/nipype/interfaces/freesurfer/preprocess.py @@ -2815,7 +2815,7 @@ def run(self, **inputs): copy2subjdir(self, originalfile, folder='mri') return super(SegmentCC, self).run(**inputs) - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): # it is necessary to find the output files and move # them to the correct loacation predicted_outputs = self._list_outputs() @@ -2842,7 +2842,7 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None): os.makedirs(os.path.dirname(out_tmp)) shutil.move(out_tmp, out_file) return super(SegmentCC, self).aggregate_outputs( - runtime, needed_outputs) + runtime, needed_outputs, rebase_cwd) class SegmentWMInputSpec(FSTraitedSpec): diff --git a/nipype/interfaces/freesurfer/utils.py b/nipype/interfaces/freesurfer/utils.py index 55e38576bb..8cae2cc15b 100644 --- a/nipype/interfaces/freesurfer/utils.py +++ b/nipype/interfaces/freesurfer/utils.py @@ -1031,8 +1031,9 @@ def info_regexp(self, info, field, delim="\n"): else: return None - def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = self._outputs() + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(ImageInfo, self).aggregate_outputs( + runtime, needed_outputs, rebase_cwd) info = runtime.stdout outputs.info = info diff --git a/nipype/interfaces/fsl/model.py b/nipype/interfaces/fsl/model.py index 113f785120..cb9d0c445a 100644 --- a/nipype/interfaces/fsl/model.py +++ b/nipype/interfaces/fsl/model.py @@ -1820,7 +1820,7 @@ class SmoothEstimate(FSLCommand): output_spec = SmoothEstimateOutputSpec _cmd = 'smoothest' - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() stdout = runtime.stdout.split('\n') outputs.dlh = float(stdout[0].split()[1]) diff --git a/nipype/interfaces/fsl/preprocess.py b/nipype/interfaces/fsl/preprocess.py index 66e3c5904d..6383f8fffd 100644 --- a/nipype/interfaces/fsl/preprocess.py +++ b/nipype/interfaces/fsl/preprocess.py @@ -661,9 +661,9 @@ class FLIRT(FSLCommand): output_spec = FLIRTOutputSpec _log_written = False - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = super(FLIRT, self).aggregate_outputs( - runtime=runtime, needed_outputs=needed_outputs) + runtime=runtime, needed_outputs=needed_outputs, rebase_cwd=rebase_cwd) if self.inputs.save_log and not self._log_written: with open(outputs.out_log, "a") as text_file: text_file.write(runtime.stdout + '\n') diff --git a/nipype/interfaces/fsl/utils.py b/nipype/interfaces/fsl/utils.py index f4ef73c0e9..494aba86cc 100644 --- a/nipype/interfaces/fsl/utils.py +++ b/nipype/interfaces/fsl/utils.py @@ -777,8 +777,9 @@ def _format_arg(self, name, trait_spec, value): '-k %s option in op_string requires mask_file') return super(ImageStats, self)._format_arg(name, trait_spec, value) - def aggregate_outputs(self, runtime=None, needed_outputs=None): - outputs = self._outputs() + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(ImageStats, self).aggregate_outputs( + runtime, needed_outputs, rebase_cwd) # local caching for backward compatibility outfile = os.path.join(os.getcwd(), 'stat_result.json') if runtime is None: diff --git a/nipype/interfaces/niftyseg/label_fusion.py b/nipype/interfaces/niftyseg/label_fusion.py index 1b0237d37c..e48481ea72 100644 --- a/nipype/interfaces/niftyseg/label_fusion.py +++ b/nipype/interfaces/niftyseg/label_fusion.py @@ -11,6 +11,7 @@ from ..base import (TraitedSpec, File, traits, isdefined, CommandLineInputSpec, NipypeInterfaceError) +from ..base.trats_extension import rebase_path_traits from .base import NiftySegCommand from ..niftyreg.base import get_custom_path from ...utils.filemanip import load_json, save_json, split_filename @@ -314,7 +315,7 @@ class CalcTopNCC(NiftySegCommand): input_spec = CalcTopNCCInputSpec output_spec = CalcTopNCCOutputSpec - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() # local caching for backward compatibility outfile = os.path.join(os.getcwd(), 'CalcTopNCC.json') @@ -335,5 +336,9 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None): if len(out_files) == 1: out_files = out_files[0] save_json(outfile, dict(files=out_files)) + + if rebase_cwd: + out_files = rebase_path_traits(outputs.trait('out_files'), + out_files, rebase_cwd) outputs.out_files = out_files return outputs diff --git a/nipype/interfaces/spm/model.py b/nipype/interfaces/spm/model.py index 4fb1c51e08..204c016aa1 100644 --- a/nipype/interfaces/spm/model.py +++ b/nipype/interfaces/spm/model.py @@ -717,11 +717,10 @@ def _make_matlab_command(self, _): return script - def aggregate_outputs(self, runtime=None): - outputs = self._outputs() - setattr(outputs, 'thresholded_map', - self._gen_thresholded_map_filename()) - setattr(outputs, 'pre_topo_fdr_map', self._gen_pre_topo_map_filename()) + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + outputs = super(Threshold, self).aggregate_outputs( + runtime, needed_outputs, rebase_cwd) + for line in runtime.stdout.split('\n'): if line.startswith("activation_forced = "): setattr(outputs, 'activation_forced', @@ -842,7 +841,7 @@ def _make_matlab_command(self, _): """ return script - def aggregate_outputs(self, runtime=None, needed_outputs=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): outputs = self._outputs() cur_output = "" for line in runtime.stdout.split('\n'): From 915a74477f247b7c2f6b957bb8f1eb640eea997f Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 22:50:26 -0700 Subject: [PATCH 07/13] fix: add ``alternatives`` to ignored metadata by make specs --- tools/checkspecs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/checkspecs.py b/tools/checkspecs.py index e282728c8e..da5dad1854 100644 --- a/tools/checkspecs.py +++ b/tools/checkspecs.py @@ -198,7 +198,7 @@ def test_specs(self, uri): ] in_built = [ 'type', 'copy', 'parent', 'instance_handler', 'comparison_mode', - 'array', 'default', 'editor' + 'array', 'default', 'editor', 'alternatives' ] bad_specs = [] for c in classes: From 8fa72ccdfe3acaffee1e05c14ad7d286cf814833 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 22:51:58 -0700 Subject: [PATCH 08/13] fix: typo in import --- nipype/interfaces/niftyseg/label_fusion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipype/interfaces/niftyseg/label_fusion.py b/nipype/interfaces/niftyseg/label_fusion.py index e48481ea72..9d3342657b 100644 --- a/nipype/interfaces/niftyseg/label_fusion.py +++ b/nipype/interfaces/niftyseg/label_fusion.py @@ -11,7 +11,7 @@ from ..base import (TraitedSpec, File, traits, isdefined, CommandLineInputSpec, NipypeInterfaceError) -from ..base.trats_extension import rebase_path_traits +from ..base.traits_extension import rebase_path_traits from .base import NiftySegCommand from ..niftyreg.base import get_custom_path from ...utils.filemanip import load_json, save_json, split_filename From 510ef2daa0288a2e1aae1618106aa64ed9c7e0e2 Mon Sep 17 00:00:00 2001 From: oesteban Date: Wed, 17 Jul 2019 22:55:33 -0700 Subject: [PATCH 09/13] tst: make specs --- .../algorithms/tests/test_auto_AddCSVRow.py | 5 +- .../afni/tests/test_auto_Allineate.py | 1 + .../afni/tests/test_auto_ClipLevel.py | 1 + .../afni/tests/test_auto_LocalBistat.py | 7 ++- .../afni/tests/test_auto_Localstat.py | 6 +- .../afni/tests/test_auto_NwarpApply.py | 5 +- .../afni/tests/test_auto_OneDToolPy.py | 1 + .../interfaces/afni/tests/test_auto_Qwarp.py | 5 +- .../afni/tests/test_auto_QwarpPlusMinus.py | 5 +- .../afni/tests/test_auto_ROIStats.py | 5 +- .../interfaces/afni/tests/test_auto_ReHo.py | 6 +- .../afni/tests/test_auto_Remlfit.py | 1 + .../afni/tests/test_auto_Resample.py | 5 +- .../afni/tests/test_auto_TCorrMap.py | 5 +- .../afni/tests/test_auto_Zeropad.py | 1 + .../base/tests/test_auto_StdOutCommandLine.py | 1 + .../camino/tests/test_auto_AnalyzeHeader.py | 1 + .../tests/test_auto_ComputeEigensystem.py | 1 + .../test_auto_ComputeFractionalAnisotropy.py | 1 + .../tests/test_auto_ComputeTensorTrace.py | 1 + .../camino/tests/test_auto_DTIFit.py | 1 + .../camino/tests/test_auto_DTLUTGen.py | 1 + .../camino/tests/test_auto_FSL2Scheme.py | 1 + .../camino/tests/test_auto_Image2Voxel.py | 1 + .../camino/tests/test_auto_LinRecon.py | 1 + .../interfaces/camino/tests/test_auto_MESD.py | 1 + .../camino/tests/test_auto_ModelFit.py | 1 + .../camino/tests/test_auto_NIfTIDT2Camino.py | 1 + .../camino/tests/test_auto_PicoPDFs.py | 1 + .../camino/tests/test_auto_ProcStreamlines.py | 1 + .../camino/tests/test_auto_QBallMX.py | 1 + .../camino/tests/test_auto_SFLUTGen.py | 1 + .../camino/tests/test_auto_SFPICOCalibData.py | 1 + .../camino/tests/test_auto_SFPeaks.py | 1 + .../camino/tests/test_auto_Shredder.py | 1 + .../camino/tests/test_auto_TractShredder.py | 1 + .../camino/tests/test_auto_VtkStreamlines.py | 1 + .../freesurfer/tests/test_auto_CALabel.py | 10 +++- .../freesurfer/tests/test_auto_CANormalize.py | 2 +- .../freesurfer/tests/test_auto_CARegister.py | 2 +- .../test_auto_CheckTalairachAlignment.py | 2 +- .../tests/test_auto_ConcatenateLTA.py | 2 + .../freesurfer/tests/test_auto_Contrast.py | 10 +++- .../freesurfer/tests/test_auto_FixTopology.py | 5 +- .../freesurfer/tests/test_auto_MRIsCALabel.py | 10 +++- .../freesurfer/tests/test_auto_MS_LDA.py | 12 +++- .../freesurfer/tests/test_auto_Normalize.py | 2 +- .../tests/test_auto_ParcellationStats.py | 9 ++- .../tests/test_auto_RegisterAVItoTalairach.py | 2 +- .../tests/test_auto_SegStatsReconAll.py | 5 +- .../tests/test_auto_SurfaceSnapshots.py | 1 + .../tests/test_auto_TalairachAVI.py | 6 +- .../interfaces/fsl/tests/test_auto_Cleaner.py | 3 + .../interfaces/fsl/tests/test_auto_Cluster.py | 5 +- .../minc/tests/test_auto_Average.py | 6 +- .../interfaces/minc/tests/test_auto_BBox.py | 1 + .../interfaces/minc/tests/test_auto_Beast.py | 7 ++- .../interfaces/minc/tests/test_auto_Calc.py | 2 + .../interfaces/minc/tests/test_auto_Dump.py | 1 + .../minc/tests/test_auto_Extract.py | 1 + .../interfaces/minc/tests/test_auto_Math.py | 1 + .../interfaces/minc/tests/test_auto_Norm.py | 6 +- .../minc/tests/test_auto_Resample.py | 10 +++- .../minc/tests/test_auto_Reshape.py | 1 + .../interfaces/minc/tests/test_auto_ToRaw.py | 1 + .../minc/tests/test_auto_VolSymm.py | 2 + .../minc/tests/test_auto_XfmInvert.py | 1 + .../niftyfit/tests/test_auto_DwiTool.py | 26 ++++++--- .../niftyfit/tests/test_auto_FitAsl.py | 41 +++++++++++--- .../niftyfit/tests/test_auto_FitDwi.py | 56 ++++++++++++++----- .../niftyfit/tests/test_auto_FitQt1.py | 25 +++++++-- .../nipy/tests/test_auto_EstimateContrast.py | 2 +- .../interfaces/nipy/tests/test_auto_FitGLM.py | 4 +- .../utility/tests/test_auto_Rename.py | 2 +- 74 files changed, 290 insertions(+), 75 deletions(-) diff --git a/nipype/algorithms/tests/test_auto_AddCSVRow.py b/nipype/algorithms/tests/test_auto_AddCSVRow.py index 3090c8b6a9..a8c4467cbf 100644 --- a/nipype/algorithms/tests/test_auto_AddCSVRow.py +++ b/nipype/algorithms/tests/test_auto_AddCSVRow.py @@ -6,7 +6,10 @@ def test_AddCSVRow_inputs(): input_map = dict( _outputs=dict(usedefault=True, ), - in_file=dict(mandatory=True, ), + in_file=dict( + extensions=None, + mandatory=True, + ), ) inputs = AddCSVRow.input_spec() diff --git a/nipype/interfaces/afni/tests/test_auto_Allineate.py b/nipype/interfaces/afni/tests/test_auto_Allineate.py index 0b110e669a..e1565376bd 100644 --- a/nipype/interfaces/afni/tests/test_auto_Allineate.py +++ b/nipype/interfaces/afni/tests/test_auto_Allineate.py @@ -86,6 +86,7 @@ def test_Allineate_inputs(): ), out_weight_file=dict( argstr='-wtprefix %s', + extensions=None, xor=['allcostx'], ), outputtype=dict(), diff --git a/nipype/interfaces/afni/tests/test_auto_ClipLevel.py b/nipype/interfaces/afni/tests/test_auto_ClipLevel.py index f9b3dbf705..96f928a809 100644 --- a/nipype/interfaces/afni/tests/test_auto_ClipLevel.py +++ b/nipype/interfaces/afni/tests/test_auto_ClipLevel.py @@ -17,6 +17,7 @@ def test_ClipLevel_inputs(): ), grad=dict( argstr='-grad %s', + extensions=None, position=3, xor='doall', ), diff --git a/nipype/interfaces/afni/tests/test_auto_LocalBistat.py b/nipype/interfaces/afni/tests/test_auto_LocalBistat.py index ed3e61d74d..4632e4cf7d 100644 --- a/nipype/interfaces/afni/tests/test_auto_LocalBistat.py +++ b/nipype/interfaces/afni/tests/test_auto_LocalBistat.py @@ -26,7 +26,10 @@ def test_LocalBistat_inputs(): mandatory=True, position=-1, ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), neighborhood=dict( argstr="-nbhd '%s(%s)'", mandatory=True, @@ -37,6 +40,7 @@ def test_LocalBistat_inputs(): ), out_file=dict( argstr='-prefix %s', + extensions=None, keep_extension=True, name_source='in_file1', name_template='%s_bistat', @@ -49,6 +53,7 @@ def test_LocalBistat_inputs(): ), weight_file=dict( argstr='-weight %s', + extensions=None, xor=['automask'], ), ) diff --git a/nipype/interfaces/afni/tests/test_auto_Localstat.py b/nipype/interfaces/afni/tests/test_auto_Localstat.py index 011ce44da8..62dc800941 100644 --- a/nipype/interfaces/afni/tests/test_auto_Localstat.py +++ b/nipype/interfaces/afni/tests/test_auto_Localstat.py @@ -21,7 +21,10 @@ def test_Localstat_inputs(): mandatory=True, position=-1, ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), neighborhood=dict( argstr="-nbhd '%s(%s)'", mandatory=True, @@ -33,6 +36,7 @@ def test_Localstat_inputs(): ), out_file=dict( argstr='-prefix %s', + extensions=None, keep_extension=True, name_source='in_file', name_template='%s_localstat', diff --git a/nipype/interfaces/afni/tests/test_auto_NwarpApply.py b/nipype/interfaces/afni/tests/test_auto_NwarpApply.py index e00457f4f3..87388c65ec 100644 --- a/nipype/interfaces/afni/tests/test_auto_NwarpApply.py +++ b/nipype/interfaces/afni/tests/test_auto_NwarpApply.py @@ -20,7 +20,10 @@ def test_NwarpApply_inputs(): usedefault=True, ), inv_warp=dict(argstr='-iwarp', ), - master=dict(argstr='-master %s', ), + master=dict( + argstr='-master %s', + extensions=None, + ), out_file=dict( argstr='-prefix %s', extensions=None, diff --git a/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py b/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py index f8e664a727..114fe53fba 100644 --- a/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py +++ b/nipype/interfaces/afni/tests/test_auto_OneDToolPy.py @@ -30,6 +30,7 @@ def test_OneDToolPy_inputs(): show_censor_count=dict(argstr='-show_censor_count', ), show_cormat_warnings=dict( argstr='-show_cormat_warnings |& tee %s', + extensions=None, position=-1, xor=['out_file'], ), diff --git a/nipype/interfaces/afni/tests/test_auto_Qwarp.py b/nipype/interfaces/afni/tests/test_auto_Qwarp.py index 3ef8c2e9b2..0b8a9e38ec 100644 --- a/nipype/interfaces/afni/tests/test_auto_Qwarp.py +++ b/nipype/interfaces/afni/tests/test_auto_Qwarp.py @@ -125,7 +125,10 @@ def test_Qwarp_inputs(): name_source=['in_file'], name_template='ppp_%s', ), - out_weight_file=dict(argstr='-wtprefix %s', ), + out_weight_file=dict( + argstr='-wtprefix %s', + extensions=None, + ), outputtype=dict(), overwrite=dict(argstr='-overwrite', ), pblur=dict(argstr='-pblur %s', ), diff --git a/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py b/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py index ca27a0d682..e282d0d0a5 100644 --- a/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py +++ b/nipype/interfaces/afni/tests/test_auto_QwarpPlusMinus.py @@ -125,7 +125,10 @@ def test_QwarpPlusMinus_inputs(): position=0, usedefault=True, ), - out_weight_file=dict(argstr='-wtprefix %s', ), + out_weight_file=dict( + argstr='-wtprefix %s', + extensions=None, + ), outputtype=dict(), overwrite=dict(argstr='-overwrite', ), pblur=dict(argstr='-pblur %s', ), diff --git a/nipype/interfaces/afni/tests/test_auto_ROIStats.py b/nipype/interfaces/afni/tests/test_auto_ROIStats.py index d3c956f7c5..c7fc517ccc 100644 --- a/nipype/interfaces/afni/tests/test_auto_ROIStats.py +++ b/nipype/interfaces/afni/tests/test_auto_ROIStats.py @@ -49,7 +49,10 @@ def test_ROIStats_inputs(): position=-1, ), quiet=dict(argstr='-quiet', ), - roisel=dict(argstr='-roisel %s', ), + roisel=dict( + argstr='-roisel %s', + extensions=None, + ), stat=dict(argstr='%s...', ), zerofill=dict( argstr='-zerofill %s', diff --git a/nipype/interfaces/afni/tests/test_auto_ReHo.py b/nipype/interfaces/afni/tests/test_auto_ReHo.py index 0edcedcdaf..cf3e468159 100644 --- a/nipype/interfaces/afni/tests/test_auto_ReHo.py +++ b/nipype/interfaces/afni/tests/test_auto_ReHo.py @@ -25,13 +25,17 @@ def test_ReHo_inputs(): argstr='-in_rois %s', extensions=None, ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), neighborhood=dict( argstr='-nneigh %s', xor=['sphere', 'ellipsoid'], ), out_file=dict( argstr='-prefix %s', + extensions=None, keep_extension=True, name_source='in_file', name_template='%s_reho', diff --git a/nipype/interfaces/afni/tests/test_auto_Remlfit.py b/nipype/interfaces/afni/tests/test_auto_Remlfit.py index f2e6703bf6..05ee75210e 100644 --- a/nipype/interfaces/afni/tests/test_auto_Remlfit.py +++ b/nipype/interfaces/afni/tests/test_auto_Remlfit.py @@ -58,6 +58,7 @@ def test_Remlfit_inputs(): ), matim=dict( argstr='-matim %s', + extensions=None, xor=['matrix'], ), matrix=dict( diff --git a/nipype/interfaces/afni/tests/test_auto_Resample.py b/nipype/interfaces/afni/tests/test_auto_Resample.py index 560d883d75..f63e1347f5 100644 --- a/nipype/interfaces/afni/tests/test_auto_Resample.py +++ b/nipype/interfaces/afni/tests/test_auto_Resample.py @@ -17,7 +17,10 @@ def test_Resample_inputs(): mandatory=True, position=-1, ), - master=dict(argstr='-master %s', ), + master=dict( + argstr='-master %s', + extensions=None, + ), num_threads=dict( nohash=True, usedefault=True, diff --git a/nipype/interfaces/afni/tests/test_auto_TCorrMap.py b/nipype/interfaces/afni/tests/test_auto_TCorrMap.py index 1ea4c0790e..209473fe6f 100644 --- a/nipype/interfaces/afni/tests/test_auto_TCorrMap.py +++ b/nipype/interfaces/afni/tests/test_auto_TCorrMap.py @@ -93,7 +93,10 @@ def test_TCorrMap_inputs(): name_source='in_file', suffix='_qmean', ), - regress_out_timeseries=dict(argstr='-ort %s', ), + regress_out_timeseries=dict( + argstr='-ort %s', + extensions=None, + ), seeds=dict( argstr='-seed %s', extensions=None, diff --git a/nipype/interfaces/afni/tests/test_auto_Zeropad.py b/nipype/interfaces/afni/tests/test_auto_Zeropad.py index abeceda432..1bd80cfad8 100644 --- a/nipype/interfaces/afni/tests/test_auto_Zeropad.py +++ b/nipype/interfaces/afni/tests/test_auto_Zeropad.py @@ -55,6 +55,7 @@ def test_Zeropad_inputs(): ), master=dict( argstr='-master %s', + extensions=None, xor=['I', 'S', 'A', 'P', 'L', 'R', 'z', 'RL', 'AP', 'IS', 'mm'], ), mm=dict( diff --git a/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py b/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py index e39dc3acaa..95afcd3216 100644 --- a/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py +++ b/nipype/interfaces/base/tests/test_auto_StdOutCommandLine.py @@ -12,6 +12,7 @@ def test_StdOutCommandLine_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py b/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py index 28170946ac..c40082d836 100644 --- a/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py +++ b/nipype/interfaces/camino/tests/test_auto_AnalyzeHeader.py @@ -50,6 +50,7 @@ def test_AnalyzeHeader_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py b/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py index 66ac282175..edf38864fa 100644 --- a/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py +++ b/nipype/interfaces/camino/tests/test_auto_ComputeEigensystem.py @@ -24,6 +24,7 @@ def test_ComputeEigensystem_inputs(): maxcomponents=dict(argstr='-maxcomponents %d', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py b/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py index 9cfae77b2f..75604df01b 100644 --- a/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py +++ b/nipype/interfaces/camino/tests/test_auto_ComputeFractionalAnisotropy.py @@ -20,6 +20,7 @@ def test_ComputeFractionalAnisotropy_inputs(): inputmodel=dict(argstr='-inputmodel %s', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py b/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py index 1443a253bd..4d31fa884c 100644 --- a/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py +++ b/nipype/interfaces/camino/tests/test_auto_ComputeTensorTrace.py @@ -20,6 +20,7 @@ def test_ComputeTensorTrace_inputs(): inputmodel=dict(argstr='-inputmodel %s', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_DTIFit.py b/nipype/interfaces/camino/tests/test_auto_DTIFit.py index 26d27d57d2..0870a77752 100644 --- a/nipype/interfaces/camino/tests/test_auto_DTIFit.py +++ b/nipype/interfaces/camino/tests/test_auto_DTIFit.py @@ -26,6 +26,7 @@ def test_DTIFit_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py b/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py index f7caf77ef1..3242163c3a 100644 --- a/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py +++ b/nipype/interfaces/camino/tests/test_auto_DTLUTGen.py @@ -28,6 +28,7 @@ def test_DTLUTGen_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py b/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py index 350c604c99..9b8a74be6a 100644 --- a/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py +++ b/nipype/interfaces/camino/tests/test_auto_FSL2Scheme.py @@ -40,6 +40,7 @@ def test_FSL2Scheme_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py b/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py index 29864bef82..2c013cf216 100644 --- a/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py +++ b/nipype/interfaces/camino/tests/test_auto_Image2Voxel.py @@ -18,6 +18,7 @@ def test_Image2Voxel_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_LinRecon.py b/nipype/interfaces/camino/tests/test_auto_LinRecon.py index 996d8f4b99..147cfaab5e 100644 --- a/nipype/interfaces/camino/tests/test_auto_LinRecon.py +++ b/nipype/interfaces/camino/tests/test_auto_LinRecon.py @@ -24,6 +24,7 @@ def test_LinRecon_inputs(): normalize=dict(argstr='-normalize', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_MESD.py b/nipype/interfaces/camino/tests/test_auto_MESD.py index 57dbbf3b28..f7dfecfdb9 100644 --- a/nipype/interfaces/camino/tests/test_auto_MESD.py +++ b/nipype/interfaces/camino/tests/test_auto_MESD.py @@ -42,6 +42,7 @@ def test_MESD_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ModelFit.py b/nipype/interfaces/camino/tests/test_auto_ModelFit.py index e5c16ec975..6d49969eb6 100644 --- a/nipype/interfaces/camino/tests/test_auto_ModelFit.py +++ b/nipype/interfaces/camino/tests/test_auto_ModelFit.py @@ -34,6 +34,7 @@ def test_ModelFit_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py b/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py index 0a5583b03e..65e0210268 100644 --- a/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py +++ b/nipype/interfaces/camino/tests/test_auto_NIfTIDT2Camino.py @@ -26,6 +26,7 @@ def test_NIfTIDT2Camino_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py b/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py index c918a372f1..e5cdea5cae 100644 --- a/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py +++ b/nipype/interfaces/camino/tests/test_auto_PicoPDFs.py @@ -36,6 +36,7 @@ def test_PicoPDFs_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py b/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py index c30ef08d8c..0386e0d54f 100644 --- a/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py +++ b/nipype/interfaces/camino/tests/test_auto_ProcStreamlines.py @@ -62,6 +62,7 @@ def test_ProcStreamlines_inputs(): noresample=dict(argstr='-noresample', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_QBallMX.py b/nipype/interfaces/camino/tests/test_auto_QBallMX.py index 7f7d0bc99f..f452f26350 100644 --- a/nipype/interfaces/camino/tests/test_auto_QBallMX.py +++ b/nipype/interfaces/camino/tests/test_auto_QBallMX.py @@ -20,6 +20,7 @@ def test_QBallMX_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py b/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py index 795138ea89..220e116255 100644 --- a/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py +++ b/nipype/interfaces/camino/tests/test_auto_SFLUTGen.py @@ -35,6 +35,7 @@ def test_SFLUTGen_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py b/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py index 80222fce4c..1b71553676 100644 --- a/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py +++ b/nipype/interfaces/camino/tests/test_auto_SFPICOCalibData.py @@ -27,6 +27,7 @@ def test_SFPICOCalibData_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_SFPeaks.py b/nipype/interfaces/camino/tests/test_auto_SFPeaks.py index 8db250c58c..49ac58aa06 100644 --- a/nipype/interfaces/camino/tests/test_auto_SFPeaks.py +++ b/nipype/interfaces/camino/tests/test_auto_SFPeaks.py @@ -38,6 +38,7 @@ def test_SFPeaks_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_Shredder.py b/nipype/interfaces/camino/tests/test_auto_Shredder.py index 594b35fc60..2d8ec43589 100644 --- a/nipype/interfaces/camino/tests/test_auto_Shredder.py +++ b/nipype/interfaces/camino/tests/test_auto_Shredder.py @@ -28,6 +28,7 @@ def test_Shredder_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_TractShredder.py b/nipype/interfaces/camino/tests/test_auto_TractShredder.py index eeed244533..e4df010c60 100644 --- a/nipype/interfaces/camino/tests/test_auto_TractShredder.py +++ b/nipype/interfaces/camino/tests/test_auto_TractShredder.py @@ -28,6 +28,7 @@ def test_TractShredder_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py b/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py index 72b6f106d8..bd8d295572 100644 --- a/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py +++ b/nipype/interfaces/camino/tests/test_auto_VtkStreamlines.py @@ -25,6 +25,7 @@ def test_VtkStreamlines_inputs(): interpolatescalars=dict(argstr='-interpolatescalars', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py b/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py index f3bfd5ad62..4d56d217c7 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CALabel.py @@ -7,7 +7,10 @@ def test_CALabel_inputs(): input_map = dict( align=dict(argstr='-align', ), args=dict(argstr='%s', ), - aseg=dict(argstr='-aseg %s', ), + aseg=dict( + argstr='-aseg %s', + extensions=None, + ), environ=dict( nohash=True, usedefault=True, @@ -26,7 +29,10 @@ def test_CALabel_inputs(): argstr='-r %s', extensions=None, ), - label=dict(argstr='-l %s', ), + label=dict( + argstr='-l %s', + extensions=None, + ), no_big_ventricles=dict(argstr='-nobigventricles', ), num_threads=dict(), out_file=dict( diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py b/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py index 1f6546ae3a..5700103e84 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CANormalize.py @@ -59,7 +59,7 @@ def test_CANormalize_inputs(): def test_CANormalize_outputs(): output_map = dict( control_points=dict(extensions=None, ), - out_file=dict(), + out_file=dict(extensions=None, ), ) outputs = CANormalize.output_spec() diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py b/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py index 83f669b218..dcf5aa84a8 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CARegister.py @@ -53,7 +53,7 @@ def test_CARegister_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_CARegister_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = CARegister.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py b/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py index 68a5a98e66..fe9c1a3121 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_CheckTalairachAlignment.py @@ -35,7 +35,7 @@ def test_CheckTalairachAlignment_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_CheckTalairachAlignment_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = CheckTalairachAlignment.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py b/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py index 8acab945c1..ee1d3ae7f3 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_ConcatenateLTA.py @@ -38,11 +38,13 @@ def test_ConcatenateLTA_inputs(): subjects_dir=dict(), tal_source_file=dict( argstr='-tal %s', + extensions=None, position=-5, requires=['tal_template_file'], ), tal_template_file=dict( argstr='%s', + extensions=None, position=-4, requires=['tal_source_file'], ), diff --git a/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py b/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py index 3c4e5aa484..3cd62e8ee7 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_Contrast.py @@ -5,10 +5,16 @@ def test_Contrast_inputs(): input_map = dict( - annotation=dict(mandatory=True, ), + annotation=dict( + extensions=None, + mandatory=True, + ), args=dict(argstr='%s', ), copy_inputs=dict(), - cortex=dict(mandatory=True, ), + cortex=dict( + extensions=None, + mandatory=True, + ), environ=dict( nohash=True, usedefault=True, diff --git a/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py b/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py index 4ba8442b14..83427dcd20 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_FixTopology.py @@ -35,7 +35,10 @@ def test_FixTopology_inputs(): ), mgz=dict(argstr='-mgz', ), seed=dict(argstr='-seed %d', ), - sphere=dict(argstr='-sphere %s', ), + sphere=dict( + argstr='-sphere %s', + extensions=None, + ), subject_id=dict( argstr='%s', mandatory=True, diff --git a/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py b/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py index 10c7af6832..eaf91bc1e8 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_MRIsCALabel.py @@ -6,7 +6,10 @@ def test_MRIsCALabel_inputs(): input_map = dict( args=dict(argstr='%s', ), - aseg=dict(argstr='-aseg %s', ), + aseg=dict( + argstr='-aseg %s', + extensions=None, + ), canonsurf=dict( argstr='%s', extensions=None, @@ -33,7 +36,10 @@ def test_MRIsCALabel_inputs(): mandatory=True, position=-4, ), - label=dict(argstr='-l %s', ), + label=dict( + argstr='-l %s', + extensions=None, + ), num_threads=dict(), out_file=dict( argstr='%s', diff --git a/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py b/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py index badb3b4f0c..3d3f5cde11 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_MS_LDA.py @@ -17,22 +17,30 @@ def test_MS_LDA_inputs(): mandatory=True, position=-1, ), - label_file=dict(argstr='-label %s', ), + label_file=dict( + argstr='-label %s', + extensions=None, + ), lda_labels=dict( argstr='-lda %s', mandatory=True, sep=' ', ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), shift=dict(argstr='-shift %d', ), subjects_dir=dict(), use_weights=dict(argstr='-W', ), vol_synth_file=dict( argstr='-synth %s', + extensions=None, mandatory=True, ), weight_file=dict( argstr='-weight %s', + extensions=None, mandatory=True, ), ) diff --git a/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py b/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py index 2384adbb2b..baf03f026c 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_Normalize.py @@ -43,7 +43,7 @@ def test_Normalize_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_Normalize_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = Normalize.output_spec() for key, metadata in list(output_map.items()): diff --git a/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py b/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py index be19b2bd37..8b4ae45d67 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_ParcellationStats.py @@ -27,11 +27,16 @@ def test_ParcellationStats_inputs(): ), in_annotation=dict( argstr='-a %s', + extensions=None, xor=['in_label'], ), - in_cortex=dict(argstr='-cortex %s', ), + in_cortex=dict( + argstr='-cortex %s', + extensions=None, + ), in_label=dict( argstr='-l %s', + extensions=None, xor=['in_annotatoin', 'out_color'], ), lh_pial=dict( @@ -45,11 +50,13 @@ def test_ParcellationStats_inputs(): mgz=dict(argstr='-mgz', ), out_color=dict( argstr='-c %s', + extensions=None, genfile=True, xor=['in_label'], ), out_table=dict( argstr='-f %s', + extensions=None, genfile=True, requires=['tabular_output'], ), diff --git a/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py b/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py index f3406d41fc..5a609f586e 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_RegisterAVItoTalairach.py @@ -47,7 +47,7 @@ def test_RegisterAVItoTalairach_outputs(): extensions=None, usedefault=True, ), - out_file=dict(), + out_file=dict(extensions=None, ), ) outputs = RegisterAVItoTalairach.output_spec() diff --git a/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py b/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py index 3cf47c71ce..09da0d001d 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_SegStatsReconAll.py @@ -100,7 +100,10 @@ def test_SegStatsReconAll_inputs(): extensions=None, mandatory=True, ), - ribbon=dict(mandatory=True, ), + ribbon=dict( + extensions=None, + mandatory=True, + ), segment_id=dict(argstr='--id %s...', ), segmentation_file=dict( argstr='--seg %s', diff --git a/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py b/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py index 26dcbe3458..968041ff67 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_SurfaceSnapshots.py @@ -59,6 +59,7 @@ def test_SurfaceSnapshots_inputs(): overlay_range_offset=dict(argstr='-foffset %.3f', ), overlay_reg=dict( argstr='-overlay-reg %s', + extensions=None, xor=['overlay_reg', 'identity_reg', 'mni152_reg'], ), patch_file=dict( diff --git a/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py b/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py index 9d248c1a7d..2a28765d67 100644 --- a/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py +++ b/nipype/interfaces/freesurfer/tests/test_auto_TalairachAVI.py @@ -30,9 +30,9 @@ def test_TalairachAVI_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_TalairachAVI_outputs(): output_map = dict( - out_file=dict(), - out_log=dict(), - out_txt=dict(), + out_file=dict(extensions=None, ), + out_log=dict(extensions=None, ), + out_txt=dict(extensions=None, ), ) outputs = TalairachAVI.output_spec() diff --git a/nipype/interfaces/fsl/tests/test_auto_Cleaner.py b/nipype/interfaces/fsl/tests/test_auto_Cleaner.py index a75df99db5..597dc88d5e 100644 --- a/nipype/interfaces/fsl/tests/test_auto_Cleaner.py +++ b/nipype/interfaces/fsl/tests/test_auto_Cleaner.py @@ -22,14 +22,17 @@ def test_Cleaner_inputs(): ), confound_file=dict( argstr='-x %s', + extensions=None, position=4, ), confound_file_1=dict( argstr='-x %s', + extensions=None, position=5, ), confound_file_2=dict( argstr='-x %s', + extensions=None, position=6, ), environ=dict( diff --git a/nipype/interfaces/fsl/tests/test_auto_Cluster.py b/nipype/interfaces/fsl/tests/test_auto_Cluster.py index 6ed34ab816..19340b9383 100644 --- a/nipype/interfaces/fsl/tests/test_auto_Cluster.py +++ b/nipype/interfaces/fsl/tests/test_auto_Cluster.py @@ -7,7 +7,10 @@ def test_Cluster_inputs(): input_map = dict( args=dict(argstr='%s', ), connectivity=dict(argstr='--connectivity=%d', ), - cope_file=dict(argstr='--cope=%s', ), + cope_file=dict( + argstr='--cope=%s', + extensions=None, + ), dlh=dict(argstr='--dlh=%.10f', ), environ=dict( nohash=True, diff --git a/nipype/interfaces/minc/tests/test_auto_Average.py b/nipype/interfaces/minc/tests/test_auto_Average.py index 678ab93a52..6f0e84d144 100644 --- a/nipype/interfaces/minc/tests/test_auto_Average.py +++ b/nipype/interfaces/minc/tests/test_auto_Average.py @@ -29,6 +29,7 @@ def test_Average_inputs(): ), filelist=dict( argstr='-filelist %s', + extensions=None, mandatory=True, xor=('input_files', 'filelist'), ), @@ -126,7 +127,10 @@ def test_Average_inputs(): argstr='-quiet', xor=('verbose', 'quiet'), ), - sdfile=dict(argstr='-sdfile %s', ), + sdfile=dict( + argstr='-sdfile %s', + extensions=None, + ), two=dict(argstr='-2', ), verbose=dict( argstr='-verbose', diff --git a/nipype/interfaces/minc/tests/test_auto_BBox.py b/nipype/interfaces/minc/tests/test_auto_BBox.py index c1b3515cea..7bdf35f03d 100644 --- a/nipype/interfaces/minc/tests/test_auto_BBox.py +++ b/nipype/interfaces/minc/tests/test_auto_BBox.py @@ -25,6 +25,7 @@ def test_BBox_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_Beast.py b/nipype/interfaces/minc/tests/test_auto_Beast.py index bc4705db2e..3a44b436b0 100644 --- a/nipype/interfaces/minc/tests/test_auto_Beast.py +++ b/nipype/interfaces/minc/tests/test_auto_Beast.py @@ -18,7 +18,10 @@ def test_Beast_inputs(): argstr='-alpha %s', usedefault=True, ), - configuration_file=dict(argstr='-configuration %s', ), + configuration_file=dict( + argstr='-configuration %s', + extensions=None, + ), environ=dict( nohash=True, usedefault=True, @@ -27,6 +30,7 @@ def test_Beast_inputs(): flip_images=dict(argstr='-flip', ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-2, ), @@ -44,6 +48,7 @@ def test_Beast_inputs(): ), output_file=dict( argstr='%s', + extensions=None, hash_files=False, name_source=['input_file'], name_template='%s_beast_mask.mnc', diff --git a/nipype/interfaces/minc/tests/test_auto_Calc.py b/nipype/interfaces/minc/tests/test_auto_Calc.py index e9df677150..d5fb316e39 100644 --- a/nipype/interfaces/minc/tests/test_auto_Calc.py +++ b/nipype/interfaces/minc/tests/test_auto_Calc.py @@ -26,6 +26,7 @@ def test_Calc_inputs(): eval_width=dict(argstr='-eval_width %s', ), expfile=dict( argstr='-expfile %s', + extensions=None, mandatory=True, xor=('expression', 'expfile'), ), @@ -36,6 +37,7 @@ def test_Calc_inputs(): ), filelist=dict( argstr='-filelist %s', + extensions=None, mandatory=True, xor=('input_files', 'filelist'), ), diff --git a/nipype/interfaces/minc/tests/test_auto_Dump.py b/nipype/interfaces/minc/tests/test_auto_Dump.py index 19c299dac8..fbd33f5a46 100644 --- a/nipype/interfaces/minc/tests/test_auto_Dump.py +++ b/nipype/interfaces/minc/tests/test_auto_Dump.py @@ -36,6 +36,7 @@ def test_Dump_inputs(): netcdf_name=dict(argstr='-n %s', ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_Extract.py b/nipype/interfaces/minc/tests/test_auto_Extract.py index 35f6162c7f..fbd0a84729 100644 --- a/nipype/interfaces/minc/tests/test_auto_Extract.py +++ b/nipype/interfaces/minc/tests/test_auto_Extract.py @@ -84,6 +84,7 @@ def test_Extract_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_Math.py b/nipype/interfaces/minc/tests/test_auto_Math.py index 6bc142b15d..41d75379d4 100644 --- a/nipype/interfaces/minc/tests/test_auto_Math.py +++ b/nipype/interfaces/minc/tests/test_auto_Math.py @@ -36,6 +36,7 @@ def test_Math_inputs(): exp=dict(argstr='-exp -const2 %s %s', ), filelist=dict( argstr='-filelist %s', + extensions=None, mandatory=True, xor=('input_files', 'filelist'), ), diff --git a/nipype/interfaces/minc/tests/test_auto_Norm.py b/nipype/interfaces/minc/tests/test_auto_Norm.py index 9fb0d3c5ba..cdd4a3db7c 100644 --- a/nipype/interfaces/minc/tests/test_auto_Norm.py +++ b/nipype/interfaces/minc/tests/test_auto_Norm.py @@ -26,7 +26,10 @@ def test_Norm_inputs(): position=-2, ), lower=dict(argstr='-lower %s', ), - mask=dict(argstr='-mask %s', ), + mask=dict( + argstr='-mask %s', + extensions=None, + ), out_ceil=dict(argstr='-out_ceil %s', ), out_floor=dict(argstr='-out_floor %s', ), output_file=dict( @@ -40,6 +43,7 @@ def test_Norm_inputs(): ), output_threshold_mask=dict( argstr='-threshold_mask %s', + extensions=None, hash_files=False, name_source=['input_file'], name_template='%s_norm_threshold_mask.mnc', diff --git a/nipype/interfaces/minc/tests/test_auto_Resample.py b/nipype/interfaces/minc/tests/test_auto_Resample.py index 8d4b24ff41..da512d9f62 100644 --- a/nipype/interfaces/minc/tests/test_auto_Resample.py +++ b/nipype/interfaces/minc/tests/test_auto_Resample.py @@ -94,7 +94,10 @@ def test_Resample_inputs(): argstr='-keep_real_range', xor=('keep_real_range', 'nokeep_real_range'), ), - like=dict(argstr='-like %s', ), + like=dict( + argstr='-like %s', + extensions=None, + ), nearest_neighbour_interpolation=dict( argstr='-nearest_neighbour', xor=('trilinear_interpolation', 'tricubic_interpolation', @@ -157,7 +160,10 @@ def test_Resample_inputs(): xor=('nelements', 'nelements_x_y_or_z'), ), talairach=dict(argstr='-talairach', ), - transformation=dict(argstr='-transformation %s', ), + transformation=dict( + argstr='-transformation %s', + extensions=None, + ), transverse_slices=dict( argstr='-transverse', xor=('transverse', 'sagittal', 'coronal'), diff --git a/nipype/interfaces/minc/tests/test_auto_Reshape.py b/nipype/interfaces/minc/tests/test_auto_Reshape.py index 669425da95..d80f9a377b 100644 --- a/nipype/interfaces/minc/tests/test_auto_Reshape.py +++ b/nipype/interfaces/minc/tests/test_auto_Reshape.py @@ -16,6 +16,7 @@ def test_Reshape_inputs(): ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-2, ), diff --git a/nipype/interfaces/minc/tests/test_auto_ToRaw.py b/nipype/interfaces/minc/tests/test_auto_ToRaw.py index aeda687c97..42ba72f145 100644 --- a/nipype/interfaces/minc/tests/test_auto_ToRaw.py +++ b/nipype/interfaces/minc/tests/test_auto_ToRaw.py @@ -26,6 +26,7 @@ def test_ToRaw_inputs(): ), out_file=dict( argstr='> %s', + extensions=None, genfile=True, position=-1, ), diff --git a/nipype/interfaces/minc/tests/test_auto_VolSymm.py b/nipype/interfaces/minc/tests/test_auto_VolSymm.py index b710d59543..ae3332b7f2 100644 --- a/nipype/interfaces/minc/tests/test_auto_VolSymm.py +++ b/nipype/interfaces/minc/tests/test_auto_VolSymm.py @@ -22,6 +22,7 @@ def test_VolSymm_inputs(): fit_nonlinear=dict(argstr='-nonlinear', ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-3, ), @@ -38,6 +39,7 @@ def test_VolSymm_inputs(): ), trans_file=dict( argstr='%s', + extensions=None, genfile=True, hash_files=False, keep_extension=False, diff --git a/nipype/interfaces/minc/tests/test_auto_XfmInvert.py b/nipype/interfaces/minc/tests/test_auto_XfmInvert.py index 9e242300da..69c455f875 100644 --- a/nipype/interfaces/minc/tests/test_auto_XfmInvert.py +++ b/nipype/interfaces/minc/tests/test_auto_XfmInvert.py @@ -16,6 +16,7 @@ def test_XfmInvert_inputs(): ), input_file=dict( argstr='%s', + extensions=None, mandatory=True, position=-2, ), diff --git a/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py b/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py index 20995e806e..2a58b39d57 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_DwiTool.py @@ -8,6 +8,7 @@ def test_DwiTool_inputs(): args=dict(argstr='%s', ), b0_file=dict( argstr='-b0 %s', + extensions=None, position=4, ), ball_flag=dict( @@ -28,11 +29,13 @@ def test_DwiTool_inputs(): ), bval_file=dict( argstr='-bval %s', + extensions=None, mandatory=True, position=2, ), bvec_file=dict( argstr='-bvec %s', + extensions=None, position=3, ), diso_val=dict(argstr='-diso %f', ), @@ -59,6 +62,7 @@ def test_DwiTool_inputs(): ), famap_file=dict( argstr='-famap %s', + extensions=None, name_source=['source_file'], name_template='%s_famap.nii.gz', ), @@ -72,20 +76,24 @@ def test_DwiTool_inputs(): ), logdti_file=dict( argstr='-logdti2 %s', + extensions=None, name_source=['source_file'], name_template='%s_logdti2.nii.gz', ), mask_file=dict( argstr='-mask %s', + extensions=None, position=5, ), mcmap_file=dict( argstr='-mcmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mcmap.nii.gz', ), mdmap_file=dict( argstr='-mdmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mdmap.nii.gz', ), @@ -115,22 +123,26 @@ def test_DwiTool_inputs(): ), rgbmap_file=dict( argstr='-rgbmap %s', + extensions=None, name_source=['source_file'], name_template='%s_rgbmap.nii.gz', ), source_file=dict( argstr='-source %s', + extensions=None, mandatory=True, position=1, ), syn_file=dict( argstr='-syn %s', + extensions=None, name_source=['source_file'], name_template='%s_syn.nii.gz', requires=['bvec_file', 'b0_file'], ), v1map_file=dict( argstr='-v1map %s', + extensions=None, name_source=['source_file'], name_template='%s_v1map.nii.gz', ), @@ -142,13 +154,13 @@ def test_DwiTool_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_DwiTool_outputs(): output_map = dict( - famap_file=dict(), - logdti_file=dict(), - mcmap_file=dict(), - mdmap_file=dict(), - rgbmap_file=dict(), - syn_file=dict(), - v1map_file=dict(), + famap_file=dict(extensions=None, ), + logdti_file=dict(extensions=None, ), + mcmap_file=dict(extensions=None, ), + mdmap_file=dict(extensions=None, ), + rgbmap_file=dict(extensions=None, ), + syn_file=dict(extensions=None, ), + v1map_file=dict(extensions=None, ), ) outputs = DwiTool.output_spec() diff --git a/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py b/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py index b2e1bef961..94a489a4ad 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_FitAsl.py @@ -8,6 +8,7 @@ def test_FitAsl_inputs(): args=dict(argstr='%s', ), cbf_file=dict( argstr='-cbf %s', + extensions=None, name_source=['source_file'], name_template='%s_cbf.nii.gz', ), @@ -20,19 +21,33 @@ def test_FitAsl_inputs(): ), error_file=dict( argstr='-error %s', + extensions=None, name_source=['source_file'], name_template='%s_error.nii.gz', ), gm_plasma=dict(argstr='-gmL %f', ), gm_t1=dict(argstr='-gmT1 %f', ), gm_ttt=dict(argstr='-gmTTT %f', ), - ir_output=dict(argstr='-IRoutput %s', ), - ir_volume=dict(argstr='-IRvolume %s', ), + ir_output=dict( + argstr='-IRoutput %s', + extensions=None, + ), + ir_volume=dict( + argstr='-IRvolume %s', + extensions=None, + ), ldd=dict(argstr='-LDD %f', ), - m0map=dict(argstr='-m0map %s', ), - m0mape=dict(argstr='-m0mape %s', ), + m0map=dict( + argstr='-m0map %s', + extensions=None, + ), + m0mape=dict( + argstr='-m0mape %s', + extensions=None, + ), mask=dict( argstr='-mask %s', + extensions=None, position=2, ), mul=dict(argstr='-mul %f', ), @@ -46,21 +61,29 @@ def test_FitAsl_inputs(): pv2=dict(argstr='-pv2 %d', ), pv3=dict(argstr='-pv3 %d %d %d', ), pv_threshold=dict(argstr='-pvthreshold', ), - seg=dict(argstr='-seg %s', ), + seg=dict( + argstr='-seg %s', + extensions=None, + ), segstyle=dict(argstr='-segstyle', ), sig=dict(argstr='-sig', ), source_file=dict( argstr='-source %s', + extensions=None, mandatory=True, position=1, ), syn_file=dict( argstr='-syn %s', + extensions=None, name_source=['source_file'], name_template='%s_syn.nii.gz', ), t1_art_cmp=dict(argstr='-T1a %f', ), - t1map=dict(argstr='-t1map %s', ), + t1map=dict( + argstr='-t1map %s', + extensions=None, + ), t_inv1=dict(argstr='-Tinv1 %f', ), t_inv2=dict(argstr='-Tinv2 %f', ), wm_plasma=dict(argstr='-wmL %f', ), @@ -74,9 +97,9 @@ def test_FitAsl_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_FitAsl_outputs(): output_map = dict( - cbf_file=dict(), - error_file=dict(), - syn_file=dict(), + cbf_file=dict(extensions=None, ), + error_file=dict(extensions=None, ), + syn_file=dict(extensions=None, ), ) outputs = FitAsl.output_spec() diff --git a/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py b/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py index 700d9a31c4..2086d12a2b 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_FitDwi.py @@ -25,15 +25,20 @@ def test_FitDwi_inputs(): ), bval_file=dict( argstr='-bval %s', + extensions=None, mandatory=True, position=2, ), bvec_file=dict( argstr='-bvec %s', + extensions=None, mandatory=True, position=3, ), - cov_file=dict(argstr='-cov %s', ), + cov_file=dict( + argstr='-cov %s', + extensions=None, + ), csf_t2_val=dict(argstr='-csfT2 %f', ), diso_val=dict(argstr='-diso %f', ), dpr_val=dict(argstr='-dpr %f', ), @@ -51,11 +56,13 @@ def test_FitDwi_inputs(): ), error_file=dict( argstr='-error %s', + extensions=None, name_source=['source_file'], name_template='%s_error.nii.gz', ), famap_file=dict( argstr='-famap %s', + extensions=None, name_source=['source_file'], name_template='%s_famap.nii.gz', ), @@ -75,13 +82,17 @@ def test_FitDwi_inputs(): argstr='-lm %f %f', requires=['gn_flag'], ), - mask_file=dict(argstr='-mask %s', ), + mask_file=dict( + argstr='-mask %s', + extensions=None, + ), maxit_val=dict( argstr='-maxit %d', requires=['gn_flag'], ), mcmap_file=dict( argstr='-mcmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mcmap.nii.gz', requires=['nodv_flag'], @@ -89,12 +100,14 @@ def test_FitDwi_inputs(): mcmaxit=dict(argstr='-mcmaxit %d', ), mcout=dict( argstr='-mcout %s', + extensions=None, name_source=['source_file'], name_template='%s_mcout.txt', ), mcsamples=dict(argstr='-mcsamples %d', ), mdmap_file=dict( argstr='-mdmap %s', + extensions=None, name_source=['source_file'], name_template='%s_mdmap.nii.gz', ), @@ -116,6 +129,7 @@ def test_FitDwi_inputs(): ), nodiff_file=dict( argstr='-nodiff %s', + extensions=None, name_source=['source_file'], name_template='%s_no_diff.nii.gz', ), @@ -128,14 +142,19 @@ def test_FitDwi_inputs(): ], ), perf_thr=dict(argstr='-perfthreshold %f', ), - prior_file=dict(argstr='-prior %s', ), + prior_file=dict( + argstr='-prior %s', + extensions=None, + ), res_file=dict( argstr='-res %s', + extensions=None, name_source=['source_file'], name_template='%s_resmap.nii.gz', ), rgbmap_file=dict( argstr='-rgbmap %s', + extensions=None, name_source=['source_file'], name_template='%s_rgbmap.nii.gz', requires=['dti_flag'], @@ -144,38 +163,45 @@ def test_FitDwi_inputs(): slice_no=dict(argstr='-slice %d', ), source_file=dict( argstr='-source %s', + extensions=None, mandatory=True, position=1, ), swls_val=dict(argstr='-swls %f', ), syn_file=dict( argstr='-syn %s', + extensions=None, name_source=['source_file'], name_template='%s_syn.nii.gz', ), te_file=dict( argstr='-TE %s', + extensions=None, xor=['te_file'], ), te_value=dict( argstr='-TE %s', + extensions=None, xor=['te_file'], ), ten_type=dict(usedefault=True, ), tenmap2_file=dict( argstr='-tenmap2 %s', + extensions=None, name_source=['source_file'], name_template='%s_tenmap2.nii.gz', requires=['dti_flag'], ), tenmap_file=dict( argstr='-tenmap %s', + extensions=None, name_source=['source_file'], name_template='%s_tenmap.nii.gz', requires=['dti_flag'], ), v1map_file=dict( argstr='-v1map %s', + extensions=None, name_source=['source_file'], name_template='%s_v1map.nii.gz', ), @@ -194,18 +220,18 @@ def test_FitDwi_inputs(): assert getattr(inputs.traits()[key], metakey) == value def test_FitDwi_outputs(): output_map = dict( - error_file=dict(), - famap_file=dict(), - mcmap_file=dict(), - mcout=dict(), - mdmap_file=dict(), - nodiff_file=dict(), - res_file=dict(), - rgbmap_file=dict(), - syn_file=dict(), - tenmap2_file=dict(), - tenmap_file=dict(), - v1map_file=dict(), + error_file=dict(extensions=None, ), + famap_file=dict(extensions=None, ), + mcmap_file=dict(extensions=None, ), + mcout=dict(extensions=None, ), + mdmap_file=dict(extensions=None, ), + nodiff_file=dict(extensions=None, ), + res_file=dict(extensions=None, ), + rgbmap_file=dict(extensions=None, ), + syn_file=dict(extensions=None, ), + tenmap2_file=dict(extensions=None, ), + tenmap_file=dict(extensions=None, ), + v1map_file=dict(extensions=None, ), ) outputs = FitDwi.output_spec() diff --git a/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py b/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py index 392654fd5c..b184e448cd 100644 --- a/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py +++ b/nipype/interfaces/niftyfit/tests/test_auto_FitQt1.py @@ -7,7 +7,10 @@ def test_FitQt1_inputs(): input_map = dict( acceptance=dict(argstr='-acceptance %f', ), args=dict(argstr='%s', ), - b1map=dict(argstr='-b1map %s', ), + b1map=dict( + argstr='-b1map %s', + extensions=None, + ), comp_file=dict( argstr='-comp %s', extensions=None, @@ -28,7 +31,10 @@ def test_FitQt1_inputs(): argstr='-flips %s', sep=' ', ), - flips_list=dict(argstr='-fliplist %s', ), + flips_list=dict( + argstr='-fliplist %s', + extensions=None, + ), gn_flag=dict( argstr='-gn', position=8, @@ -63,7 +69,10 @@ def test_FitQt1_inputs(): name_template='%s_mcmap.nii.gz', ), mcmaxit=dict(argstr='-mcmaxit %d', ), - mcout=dict(argstr='-mcout %s', ), + mcout=dict( + argstr='-mcout %s', + extensions=None, + ), mcsamples=dict(argstr='-mcsamples %d', ), nb_comp=dict( argstr='-nc %d', @@ -101,7 +110,10 @@ def test_FitQt1_inputs(): name_source=['source_file'], name_template='%s_syn.nii.gz', ), - t1_list=dict(argstr='-T1list %s', ), + t1_list=dict( + argstr='-T1list %s', + extensions=None, + ), t1map_file=dict( argstr='-t1map %s', extensions=None, @@ -119,7 +131,10 @@ def test_FitQt1_inputs(): position=14, sep=' ', ), - tis_list=dict(argstr='-TIlist %s', ), + tis_list=dict( + argstr='-TIlist %s', + extensions=None, + ), tr_value=dict( argstr='-TR %f', position=5, diff --git a/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py b/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py index 3fe17160db..9a9b5c421c 100644 --- a/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py +++ b/nipype/interfaces/nipy/tests/test_auto_EstimateContrast.py @@ -13,7 +13,7 @@ def test_EstimateContrast_inputs(): constants=dict(mandatory=True, ), contrasts=dict(mandatory=True, ), dof=dict(mandatory=True, ), - mask=dict(), + mask=dict(extensions=None, ), nvbeta=dict(mandatory=True, ), reg_names=dict(mandatory=True, ), s2=dict( diff --git a/nipype/interfaces/nipy/tests/test_auto_FitGLM.py b/nipype/interfaces/nipy/tests/test_auto_FitGLM.py index a700c18d43..71ca221efc 100644 --- a/nipype/interfaces/nipy/tests/test_auto_FitGLM.py +++ b/nipype/interfaces/nipy/tests/test_auto_FitGLM.py @@ -8,7 +8,7 @@ def test_FitGLM_inputs(): TR=dict(mandatory=True, ), drift_model=dict(usedefault=True, ), hrf_model=dict(usedefault=True, ), - mask=dict(), + mask=dict(extensions=None, ), method=dict(usedefault=True, ), model=dict(usedefault=True, ), normalize_design_matrix=dict(usedefault=True, ), @@ -30,7 +30,7 @@ def test_FitGLM_outputs(): dof=dict(), nvbeta=dict(), reg_names=dict(), - residuals=dict(), + residuals=dict(extensions=None, ), s2=dict(extensions=None, ), ) outputs = FitGLM.output_spec() diff --git a/nipype/interfaces/utility/tests/test_auto_Rename.py b/nipype/interfaces/utility/tests/test_auto_Rename.py index 6b56badd6f..70ced0ecdd 100644 --- a/nipype/interfaces/utility/tests/test_auto_Rename.py +++ b/nipype/interfaces/utility/tests/test_auto_Rename.py @@ -20,7 +20,7 @@ def test_Rename_inputs(): for metakey, value in list(metadata.items()): assert getattr(inputs.traits()[key], metakey) == value def test_Rename_outputs(): - output_map = dict(out_file=dict(), ) + output_map = dict(out_file=dict(extensions=None, ), ) outputs = Rename.output_spec() for key, metadata in list(output_map.items()): From 6cf22ce968e1389b4a5851bec7ee6c87a261e5e2 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 18 Jul 2019 00:43:35 -0700 Subject: [PATCH 10/13] fix: dealing with #2968 --- .../base/tests/test_traits_extension.py | 27 +++++++++---- nipype/interfaces/base/traits_extension.py | 27 ++++++++----- nipype/pipeline/engine/nodes.py | 2 +- nipype/pipeline/engine/tests/test_nodes.py | 14 ++++--- nipype/pipeline/engine/tests/test_utils.py | 2 +- nipype/pipeline/engine/utils.py | 40 +++++++------------ 6 files changed, 61 insertions(+), 51 deletions(-) diff --git a/nipype/interfaces/base/tests/test_traits_extension.py b/nipype/interfaces/base/tests/test_traits_extension.py index df1c69d47d..5150c46a8e 100644 --- a/nipype/interfaces/base/tests/test_traits_extension.py +++ b/nipype/interfaces/base/tests/test_traits_extension.py @@ -17,6 +17,7 @@ class _test_spec(nib.TraitedSpec): f = nib.traits.Dict(nib.Str, nib.File()) g = nib.traits.Either(nib.File, nib.Str) h = nib.Str + ee = nib.OutputMultiObject(nib.Str) def test_rebase_path_traits(): @@ -36,23 +37,28 @@ def test_rebase_path_traits(): '/some/path') assert c == [Path('f1.txt'), Path('f2.txt'), Path('f3.txt')] + d = rebase_path_traits( + spec.trait('d'), 2.0, '/some/path') + assert d == 2.0 + + d = rebase_path_traits( + spec.trait('d'), '/some/path/either.txt', '/some/path') + assert '%s' % d == 'either.txt' + e = rebase_path_traits( spec.trait('e'), ['/some/path/f1.txt', '/some/path/f2.txt', '/some/path/f3.txt'], '/some/path') assert e == [Path('f1.txt'), Path('f2.txt'), Path('f3.txt')] + e = rebase_path_traits( + spec.trait('e'), [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]], + '/some/path') + assert e == [[Path('f1.txt'), Path('f2.txt')], [[Path('f3.txt')]]] + f = rebase_path_traits( spec.trait('f'), {'1': '/some/path/f1.txt'}, '/some/path') assert f == {'1': Path('f1.txt')} - d = rebase_path_traits( - spec.trait('d'), 2.0, '/some/path') - assert d == 2.0 - - d = rebase_path_traits( - spec.trait('d'), '/some/path/either.txt', '/some/path') - assert '%s' % d == 'either.txt' - g = rebase_path_traits( spec.trait('g'), 'some/path/either.txt', '/some/path') assert '%s' % g == 'some/path/either.txt' @@ -69,3 +75,8 @@ def test_rebase_path_traits(): h = rebase_path_traits(spec.trait('h'), '2', '/some/path') assert h == '2' + + ee = rebase_path_traits( + spec.trait('ee'), [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]], + '/some/path') + assert ee == [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]] diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index 82496438a3..cc64b74148 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -504,18 +504,26 @@ def as_ctrait(self): traits.Either = PatchedEither +def _rebase_path(value, cwd): + if isinstance(value, list): + return [_rebase_path(v, cwd) for v in value] + + try: + value = Path(value) + except TypeError: + pass + else: + try: + value = Path(value).relative_to(cwd) + except ValueError: + pass + return value + + def rebase_path_traits(thistrait, value, cwd): """Rebase a BasePath-derived trait given an interface spec.""" if thistrait.is_trait_type(BasePath): - try: - value = Path(value) - except TypeError: - pass - else: - try: - value = Path(value).relative_to(cwd) - except ValueError: - pass + value = _rebase_path(value, cwd) elif thistrait.is_trait_type(traits.List): innertrait, = thistrait.inner_traits if not isinstance(value, (list, tuple)): @@ -533,7 +541,6 @@ def rebase_path_traits(thistrait, value, cwd): elif thistrait.alternatives: is_str = [f.is_trait_type((traits.String, traits.BaseStr, traits.BaseBytes, Str)) for f in thistrait.alternatives] - print(is_str) if any(is_str) and isinstance(value, (bytes, str)) and not value.startswith('/'): return value for subtrait in thistrait.alternatives: diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 006558b296..6dcb5b98b7 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -640,7 +640,7 @@ def _run_command(self, execute, copyfiles=True): message += ', a CommandLine Interface with command:\n{}'.format(cmd) logger.info(message) try: - result = self._interface.run(cwd=outdir) + result = self._interface.run(cwd=outdir, rebase_cwd=True) except Exception as msg: result.runtime.stderr = '%s\n\n%s'.format( getattr(result.runtime, 'stderr', ''), msg) diff --git a/nipype/pipeline/engine/tests/test_nodes.py b/nipype/pipeline/engine/tests/test_nodes.py index ea03fe69ae..d6349459f6 100644 --- a/nipype/pipeline/engine/tests/test_nodes.py +++ b/nipype/pipeline/engine/tests/test_nodes.py @@ -295,13 +295,17 @@ def test_inputs_removal(tmpdir): def test_outputmultipath_collapse(tmpdir): """Test an OutputMultiPath whose initial value is ``[[x]]`` to ensure that it is returned as ``[x]``, regardless of how accessed.""" - select_if = niu.Select(inlist=[[1, 2, 3], [4]], index=1) - select_nd = pe.Node(niu.Select(inlist=[[1, 2, 3], [4]], index=1), + select_if = niu.Select(inlist=[[1, 2, 3], [4], [5]], index=[0, 1]) + select_nd = pe.Node(niu.Select(inlist=[[1, 2, 3], [4], [5]], index=[0, 1]), name='select_nd') ifres = select_if.run() + ifbase = select_if.run(rebase_cwd=True) # If flattening happens could be the rebase ndres = select_nd.run() - assert ifres.outputs.out == [4] - assert ndres.outputs.out == [4] - assert select_nd.result.outputs.out == [4] + assert ifres.outputs.out == [[1, 2, 3], [4]] + assert ifbase.outputs.out == [[1, 2, 3], [4]] + assert ndres.outputs.out == [[1, 2, 3], [4]] + assert select_nd.result.outputs.out == [[1, 2, 3], [4]] + + # Inconsistent behavior of OutputMultiObject, see #2968 diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index cca084fd09..d7a65c67e4 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -16,7 +16,7 @@ from ....interfaces import base as nib from ....interfaces import utility as niu from .... import config -from ..utils import clean_working_directory, write_workflow_prov, modify_paths +from ..utils import clean_working_directory, write_workflow_prov class InputSpec(nib.TraitedSpec): diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 3d961126d5..838be5d4ca 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -287,25 +287,23 @@ def _protect_collapses(hastraits): def save_resultfile(result, cwd, name): """Save a result pklz file to ``cwd``""" resultsfile = os.path.join(cwd, 'result_%s.pklz' % name) - if result.outputs: - try: - collapsed = _identify_collapses(result.outputs) - outputs = _uncollapse(result.outputs.trait_get(), collapsed) - # Double-protect tosave so that the original, uncollapsed trait - # is saved in the pickle file. Thus, when the loading process - # collapses, the original correct value is loaded. - tosave = _uncollapse(outputs.copy(), collapsed) - except AttributeError: - tosave = outputs = result.outputs.dictcopy() # outputs was a bunch - for k, v in list(modify_paths(tosave, relative=True, basedir=cwd).items()): - setattr(result.outputs, k, v) + # if result.outputs: + # try: + # collapsed = _identify_collapses(result.outputs) + # outputs = _uncollapse(result.outputs.trait_get(), collapsed) + # # Double-protect tosave so that the original, uncollapsed trait + # # is saved in the pickle file. Thus, when the loading process + # # collapses, the original correct value is loaded. + # tosave = _uncollapse(outputs.copy(), collapsed) + # except AttributeError: + # tosave = outputs = result.outputs.dictcopy() # outputs was a bunch savepkl(resultsfile, result) logger.debug('saved results in %s', resultsfile) - if result.outputs: - for k, v in list(outputs.items()): - setattr(result.outputs, k, v) + # if result.outputs: + # for k, v in list(outputs.items()): + # setattr(result.outputs, k, v) def load_resultfile(path, name): @@ -350,17 +348,7 @@ def load_resultfile(path, name): 'some file does not exist. hence trait cannot be set') else: if result.outputs: - try: - outputs = _protect_collapses(result.outputs) - except AttributeError: - outputs = result.outputs.dictcopy() # outputs == Bunch - try: - for k, v in list(modify_paths(outputs, relative=False, - basedir=path).items()): - setattr(result.outputs, k, v) - except FileNotFoundError: - logger.debug('conversion to full path results in ' - 'non existent file') + pass # TODO: resolve BasePath traits aggregate = False pkl_file.close() logger.debug('Aggregate: %s', aggregate) From a34e9ade59138f4345a4b5bd15d21d757780e0f6 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 18 Jul 2019 09:26:08 -0700 Subject: [PATCH 11/13] fix: new ``resolve_path_traits`` utility, running tests --- .../base/tests/test_traits_extension.py | 72 ++++++++++++++++++- nipype/interfaces/base/traits_extension.py | 42 +++++++++++ nipype/pipeline/engine/tests/test_utils.py | 18 ++--- nipype/pipeline/engine/utils.py | 37 +++++++--- 4 files changed, 151 insertions(+), 18 deletions(-) diff --git a/nipype/interfaces/base/tests/test_traits_extension.py b/nipype/interfaces/base/tests/test_traits_extension.py index 5150c46a8e..4a2b884921 100644 --- a/nipype/interfaces/base/tests/test_traits_extension.py +++ b/nipype/interfaces/base/tests/test_traits_extension.py @@ -4,7 +4,7 @@ from __future__ import print_function, unicode_literals from ... import base as nib -from ..traits_extension import rebase_path_traits, Path +from ..traits_extension import rebase_path_traits, resolve_path_traits, Path class _test_spec(nib.TraitedSpec): @@ -80,3 +80,73 @@ def test_rebase_path_traits(): spec.trait('ee'), [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]], '/some/path') assert ee == [['/some/path/f1.txt', '/some/path/f2.txt'], [['/some/path/f3.txt']]] + + +def test_resolve_path_traits(): + """Check resolve_path_traits.""" + spec = _test_spec() + + a = resolve_path_traits( + spec.trait('a'), 'f1.txt', '/some/path') + assert a == Path('/some/path/f1.txt') + + b = resolve_path_traits( + spec.trait('b'), ('f1.txt', 'f2.txt'), '/some/path') + assert b == (Path('/some/path/f1.txt'), Path('/some/path/f2.txt')) + + c = resolve_path_traits( + spec.trait('c'), ['f1.txt', 'f2.txt', 'f3.txt'], + '/some/path') + assert c == [Path('/some/path/f1.txt'), Path('/some/path/f2.txt'), Path('/some/path/f3.txt')] + + d = resolve_path_traits( + spec.trait('d'), 2.0, '/some/path') + assert d == 2.0 + + d = resolve_path_traits( + spec.trait('d'), 'either.txt', '/some/path') + assert '%s' % d == '/some/path/either.txt' + + e = resolve_path_traits( + spec.trait('e'), ['f1.txt', 'f2.txt', 'f3.txt'], + '/some/path') + assert e == [Path('/some/path/f1.txt'), Path('/some/path/f2.txt'), Path('/some/path/f3.txt')] + + e = resolve_path_traits( + spec.trait('e'), [['f1.txt', 'f2.txt'], [['f3.txt']]], + '/some/path') + assert e == [[Path('/some/path/f1.txt'), Path('/some/path/f2.txt')], + [[Path('/some/path/f3.txt')]]] + + f = resolve_path_traits( + spec.trait('f'), {'1': 'path/f1.txt'}, '/some') + assert f == {'1': Path('/some/path/f1.txt')} + + g = resolve_path_traits( + spec.trait('g'), '/either.txt', '/some/path') + assert g == Path('/either.txt') + + # This is a problematic case, it is impossible to know whether this + # was meant to be a string or a file. + # Commented out because in this implementation, strings take precedence + # g = resolve_path_traits( + # spec.trait('g'), 'path/either.txt', '/some') + # assert g == Path('/some/path/either.txt') + + # This is a problematic case, it is impossible to know whether this + # was meant to be a string or a file. + g = resolve_path_traits(spec.trait('g'), 'string', '/some') + assert g == 'string' + + # This is a problematic case, it is impossible to know whether this + # was meant to be a string or a file. + g = resolve_path_traits(spec.trait('g'), '2', '/some/path') + assert g == '2' # You dont want this one to be a Path + + h = resolve_path_traits(spec.trait('h'), '2', '/some/path') + assert h == '2' + + ee = resolve_path_traits( + spec.trait('ee'), [['f1.txt', 'f2.txt'], [['f3.txt']]], + '/some/path') + assert ee == [['f1.txt', 'f2.txt'], [['f3.txt']]] diff --git a/nipype/interfaces/base/traits_extension.py b/nipype/interfaces/base/traits_extension.py index cc64b74148..e0a8705ad1 100644 --- a/nipype/interfaces/base/traits_extension.py +++ b/nipype/interfaces/base/traits_extension.py @@ -546,3 +546,45 @@ def rebase_path_traits(thistrait, value, cwd): for subtrait in thistrait.alternatives: value = rebase_path_traits(subtrait, value, cwd) return value + + +def _resolve_path(value, cwd): + if isinstance(value, list): + return [_resolve_path(v, cwd) for v in value] + + try: + value = Path(value) + except TypeError: + pass + else: + if not value.is_absolute(): + value = Path(cwd) / value + return value + + +def resolve_path_traits(thistrait, value, cwd): + """Resolve a BasePath-derived trait given an interface spec.""" + if thistrait.is_trait_type(BasePath): + value = _resolve_path(value, cwd) + elif thistrait.is_trait_type(traits.List): + innertrait, = thistrait.inner_traits + if not isinstance(value, (list, tuple)): + value = resolve_path_traits(innertrait, value, cwd) + else: + value = [resolve_path_traits(innertrait, v, cwd) + for v in value] + elif thistrait.is_trait_type(traits.Dict): + _, innertrait = thistrait.inner_traits + value = {k: resolve_path_traits(innertrait, v, cwd) + for k, v in value.items()} + elif thistrait.is_trait_type(Tuple): + value = tuple([resolve_path_traits(subtrait, v, cwd) + for subtrait, v in zip(thistrait.inner_traits, value)]) + elif thistrait.alternatives: + is_str = [f.is_trait_type((traits.String, traits.BaseStr, traits.BaseBytes, Str)) + for f in thistrait.alternatives] + if any(is_str) and isinstance(value, (bytes, str)) and not value.startswith('/'): + return value + for subtrait in thistrait.alternatives: + value = resolve_path_traits(subtrait, value, cwd) + return value diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index d7a65c67e4..d47e90dc0d 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -241,6 +241,7 @@ class output_spec(nib.TraitedSpec): def _run_interface(self, runtime): out_path = os.path.abspath(os.path.basename(self.inputs.in_str) + '_path') open(out_path, 'w').close() + print(out_path) self._results['out_str'] = self.inputs.in_str self._results['out_path'] = out_path @@ -267,17 +268,18 @@ def test_modify_paths_bug(tmpdir): open('2', 'w').close() - results = spc.run() + outputs = spc.run().outputs # Basic check that string was not manipulated and path exists - out_path = results.outputs.out_path - out_str = results.outputs.out_str + out_path = outputs.out_path + print(out_path) + out_str = outputs.out_str assert out_str == '2' assert os.path.basename(out_path) == '2_path' - assert os.path.isfile(out_path) # Assert data structures pass through correctly - assert results.outputs.out_tuple == (out_path, out_str) - assert results.outputs.out_dict_path == {out_str: out_path} - assert results.outputs.out_dict_str == {out_str: out_str} - assert results.outputs.out_list == [out_str] * 2 + assert outputs.out_tuple == (out_path, out_str) + assert outputs.out_dict_path == {out_str: out_path} + assert outputs.out_dict_str == {out_str: out_str} + assert outputs.out_list == [out_str] * 2 + assert out_path.is_file() diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 838be5d4ca..99050157e8 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -40,6 +40,7 @@ ) from ...utils.misc import str2bool from ...utils.functions import create_function_from_source +from ...interfaces.base.traits_extension import resolve_path_traits from ...interfaces.base import (Bunch, CommandLine, isdefined, Undefined, InterfaceResult, traits) from ...interfaces.utility import IdentityInterface @@ -146,16 +147,24 @@ def write_report(node, report_type=None, is_mapnode=False): write_rst_dict(node.inputs.trait_get()), ] - result = node.result # Locally cache result - outputs = result.outputs - - if outputs is None: + # Preexec reports should stop here + if report_type == 'preexec': with open(report_file, 'at') as fp: fp.write('\n'.join(lines)) return lines.append(write_rst_header('Execution Outputs', level=1)) + result = node.result # Locally cache result + + # Premature end of report + if result is None: + logger.critical('[Node] Recovered InterfaceResult is None.') + lines += ['', 'CRITICAL - InterfaceResult is None.'] + with open(report_file, 'at') as fp: + fp.write('\n'.join(lines)) + return + outputs = result.outputs if isinstance(outputs, Bunch): lines.append(write_rst_dict(outputs.dictcopy())) elif outputs: @@ -287,6 +296,7 @@ def _protect_collapses(hastraits): def save_resultfile(result, cwd, name): """Save a result pklz file to ``cwd``""" resultsfile = os.path.join(cwd, 'result_%s.pklz' % name) + logger.debug("Saving results file: '%s'", resultsfile) # if result.outputs: # try: # collapsed = _identify_collapses(result.outputs) @@ -299,14 +309,15 @@ def save_resultfile(result, cwd, name): # tosave = outputs = result.outputs.dictcopy() # outputs was a bunch savepkl(resultsfile, result) - logger.debug('saved results in %s', resultsfile) + if result.outputs: + pass # if result.outputs: # for k, v in list(outputs.items()): # setattr(result.outputs, k, v) -def load_resultfile(path, name): +def load_resultfile(path, name, resolve=True): """ Load InterfaceResult file from path @@ -329,6 +340,8 @@ def load_resultfile(path, name): resultsoutputfile = os.path.join(path, 'result_%s.pklz' % name) result = None attribute_error = False + + logger.info("Loading results file: '%s'", resultsoutputfile) if os.path.exists(resultsoutputfile): pkl_file = gzip.open(resultsoutputfile, 'rb') try: @@ -347,9 +360,15 @@ def load_resultfile(path, name): logger.debug( 'some file does not exist. hence trait cannot be set') else: - if result.outputs: - pass # TODO: resolve BasePath traits - aggregate = False + if result is not None and result.outputs: + aggregate = False + + if resolve and not aggregate: + for name in list(result.outputs.get().keys()): + value = getattr(result.outputs, name) + if isdefined(value): + value = resolve_path_traits(result.outputs.trait(name), + value, path) pkl_file.close() logger.debug('Aggregate: %s', aggregate) return result, aggregate, attribute_error From 93f513d5a63bb248e08c385890ffad50db6097b0 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 18 Jul 2019 12:07:03 -0700 Subject: [PATCH 12/13] fix: resolved issues with saving/loading results --- nipype/interfaces/base/core.py | 9 +- nipype/interfaces/base/support.py | 4 +- nipype/interfaces/base/tests/test_core.py | 140 ++++++++++----------- nipype/pipeline/engine/nodes.py | 40 +++--- nipype/pipeline/engine/tests/test_nodes.py | 2 +- nipype/pipeline/engine/tests/test_utils.py | 34 ++--- nipype/pipeline/engine/utils.py | 128 ++++++++++--------- 7 files changed, 186 insertions(+), 171 deletions(-) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index 83c38d2198..a368986f19 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -378,7 +378,7 @@ def run(self, cwd=None, ignore_exception=None, rebase_cwd=None, **inputs): runtime = self._pre_run_hook(runtime) runtime = self._run_interface(runtime) runtime = self._post_run_hook(runtime) - outputs = self.aggregate_outputs(runtime, rebase_cwd=rebase_cwd) + outputs = self.aggregate_outputs(runtime) except Exception as e: import traceback # Retrieve the maximum info fast @@ -451,7 +451,7 @@ def _list_outputs(self): else: return None - def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): + def aggregate_outputs(self, runtime=None, needed_outputs=None): """Collate expected outputs and check for existence.""" outputs = self._outputs() # Generate an output spec object if needed_outputs is not None and not needed_outputs: @@ -477,10 +477,6 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): for key in aggregate_names: val = predicted_outputs[key] - - # All the magic to fix #2944 resides here: - if rebase_cwd: - val = rebase_path_traits(outputs.trait(key), val, rebase_cwd) try: setattr(outputs, key, val) except TraitError as error: @@ -489,7 +485,6 @@ def aggregate_outputs(self, runtime=None, needed_outputs=None, rebase_cwd=None): (key, self.__class__.__name__) raise FileNotFoundError(val, message=msg) raise error - return outputs @property diff --git a/nipype/interfaces/base/support.py b/nipype/interfaces/base/support.py index 0fd1d27674..4aa5d4a3b8 100644 --- a/nipype/interfaces/base/support.py +++ b/nipype/interfaces/base/support.py @@ -225,13 +225,15 @@ def __init__(self, runtime, inputs=None, outputs=None, - provenance=None): + provenance=None, + error=None): self._version = 2.0 self.interface = interface self.runtime = runtime self.inputs = inputs self.outputs = outputs self.provenance = provenance + self.error = error @property def version(self): diff --git a/nipype/interfaces/base/tests/test_core.py b/nipype/interfaces/base/tests/test_core.py index 80a9b2a8a7..e6baad36a2 100644 --- a/nipype/interfaces/base/tests/test_core.py +++ b/nipype/interfaces/base/tests/test_core.py @@ -532,73 +532,73 @@ def _run_interface(self, runtime): BrokenRuntime().run() -def test_aggregate_outputs(tmpdir): - tmpdir.chdir() - b_value = '%s' % tmpdir.join('filename.txt') - c_value = '%s' % tmpdir.join('sub') - - class TestInterface(nib.BaseInterface): - class input_spec(nib.TraitedSpec): - a = nib.traits.Any() - class output_spec(nib.TraitedSpec): - b = nib.File() - c = nib.Directory(exists=True) - d = nib.File() - - def _list_outputs(self): - return {'b': b_value, 'c': c_value, 'd': './log.txt'} - - # Test aggregate_outputs without needed_outputs - outputs = TestInterface().aggregate_outputs(needed_outputs=[]) - assert outputs.b is nib.Undefined - - # Test that only those in needed_outputs are returned - outputs = TestInterface().aggregate_outputs( - needed_outputs=['b', 'd']) - assert outputs.c is nib.Undefined - assert outputs.b == b_value - assert Path(outputs.b).is_absolute() - assert not Path(outputs.d).is_absolute() - - # Test that traits are actually validated at aggregation - with pytest.raises(OSError): - outputs = TestInterface().aggregate_outputs( - needed_outputs=['b', 'c'], rebase_cwd='%s' % tmpdir) - - # Test that rebase_cwd actually returns relative paths - tmpdir.mkdir('sub') - outputs = TestInterface().aggregate_outputs(rebase_cwd='%s' % tmpdir) - - assert outputs.b == 'filename.txt' - assert not Path(outputs.b).is_absolute() - assert not Path(outputs.c).is_absolute() - assert not Path(outputs.d).is_absolute() - - -def test_aggregate_outputs_compounds(): - outputs_dict = { - 'b': tuple(['/some/folder/f%d.txt' % d - for d in range(1, 3)]), - 'c': ['/some/folder/f%d.txt' % d for d in range(1, 5)], - 'd': 2.0, - 'e': ['/some/folder/f%d.txt' % d for d in range(1, 4)] - } - - class TestInterface(nib.BaseInterface): - class input_spec(nib.TraitedSpec): - a = nib.traits.Any() - class output_spec(nib.TraitedSpec): - b = nib.traits.Tuple(nib.File(extensions=['.txt']), - nib.File(extensions=['.txt'])) - c = nib.traits.List(nib.File()) - d = nib.traits.Either(nib.File(), nib.traits.Float()) - e = nib.OutputMultiObject(nib.File()) - - def _list_outputs(self): - return outputs_dict - - outputs = TestInterface().aggregate_outputs(rebase_cwd='/some/folder') - assert outputs.d == 2.0 - assert outputs.b == ('f1.txt', 'f2.txt') - assert outputs.e == ['f%d.txt' % (d + 1) - for d in range(len(outputs_dict['e']))] +# def test_aggregate_outputs(tmpdir): +# tmpdir.chdir() +# b_value = '%s' % tmpdir.join('filename.txt') +# c_value = '%s' % tmpdir.join('sub') + +# class TestInterface(nib.BaseInterface): +# class input_spec(nib.TraitedSpec): +# a = nib.traits.Any() +# class output_spec(nib.TraitedSpec): +# b = nib.File() +# c = nib.Directory(exists=True) +# d = nib.File() + +# def _list_outputs(self): +# return {'b': b_value, 'c': c_value, 'd': './log.txt'} + +# # Test aggregate_outputs without needed_outputs +# outputs = TestInterface().aggregate_outputs(needed_outputs=[]) +# assert outputs.b is nib.Undefined + +# # Test that only those in needed_outputs are returned +# outputs = TestInterface().aggregate_outputs( +# needed_outputs=['b', 'd']) +# assert outputs.c is nib.Undefined +# assert outputs.b == b_value +# assert Path(outputs.b).is_absolute() +# assert not Path(outputs.d).is_absolute() + +# # Test that traits are actually validated at aggregation +# with pytest.raises(OSError): +# outputs = TestInterface().aggregate_outputs( +# needed_outputs=['b', 'c'], rebase_cwd='%s' % tmpdir) + +# # Test that rebase_cwd actually returns relative paths +# tmpdir.mkdir('sub') +# outputs = TestInterface().aggregate_outputs(rebase_cwd='%s' % tmpdir) + +# assert outputs.b == 'filename.txt' +# assert not Path(outputs.b).is_absolute() +# assert not Path(outputs.c).is_absolute() +# assert not Path(outputs.d).is_absolute() + + +# def test_aggregate_outputs_compounds(): +# outputs_dict = { +# 'b': tuple(['/some/folder/f%d.txt' % d +# for d in range(1, 3)]), +# 'c': ['/some/folder/f%d.txt' % d for d in range(1, 5)], +# 'd': 2.0, +# 'e': ['/some/folder/f%d.txt' % d for d in range(1, 4)] +# } + +# class TestInterface(nib.BaseInterface): +# class input_spec(nib.TraitedSpec): +# a = nib.traits.Any() +# class output_spec(nib.TraitedSpec): +# b = nib.traits.Tuple(nib.File(extensions=['.txt']), +# nib.File(extensions=['.txt'])) +# c = nib.traits.List(nib.File()) +# d = nib.traits.Either(nib.File(), nib.traits.Float()) +# e = nib.OutputMultiObject(nib.File()) + +# def _list_outputs(self): +# return outputs_dict + +# outputs = TestInterface().aggregate_outputs(rebase_cwd='/some/folder') +# assert outputs.d == 2.0 +# assert outputs.b == ('f1.txt', 'f2.txt') +# assert outputs.e == ['f%d.txt' % (d + 1) +# for d in range(len(outputs_dict['e']))] diff --git a/nipype/pipeline/engine/nodes.py b/nipype/pipeline/engine/nodes.py index 6dcb5b98b7..bc81dd7890 100644 --- a/nipype/pipeline/engine/nodes.py +++ b/nipype/pipeline/engine/nodes.py @@ -194,7 +194,7 @@ def interface(self): @property def result(self): """Get result from result file (do not hold it in memory)""" - return _load_resultfile(self.output_dir(), self.name)[0] + return self._load_results() @property def inputs(self): @@ -483,6 +483,7 @@ def run(self, updatehash=False): # Tear-up after success shutil.move(hashfile_unfinished, hashfile_unfinished.replace('_unfinished', '')) + write_report( self, report_type='postexec', is_mapnode=isinstance(self, MapNode)) logger.info('[Node] Finished "%s".', self.fullname) @@ -564,14 +565,16 @@ def _run_interface(self, execute=True, updatehash=False): def _load_results(self): cwd = self.output_dir() - result, aggregate, attribute_error = _load_resultfile(cwd, self.name) + is_mapnode = isinstance(self, MapNode) + result, attribute_error = _load_resultfile( + cwd, self.name, resolve=not is_mapnode) # try aggregating first - if aggregate: + if result is None: logger.debug('aggregating results') if attribute_error: old_inputs = loadpkl(op.join(cwd, '_inputs.pklz')) self.inputs.trait_set(**old_inputs) - if not isinstance(self, MapNode): + if not is_mapnode: self._copyfiles_to_wd(linksonly=True) aggouts = self._interface.aggregate_outputs( needed_outputs=self.needed_outputs) @@ -630,6 +633,7 @@ def _run_command(self, execute, copyfiles=True): with indirectory(outdir): cmd = self._interface.cmdline except Exception as msg: + result.error = True result.runtime.stderr = '{}\n\n{}'.format( getattr(result.runtime, 'stderr', ''), msg) _save_resultfile(result, outdir, self.name) @@ -642,6 +646,7 @@ def _run_command(self, execute, copyfiles=True): try: result = self._interface.run(cwd=outdir, rebase_cwd=True) except Exception as msg: + result.error = True result.runtime.stderr = '%s\n\n%s'.format( getattr(result.runtime, 'stderr', ''), msg) _save_resultfile(result, outdir, self.name) @@ -651,15 +656,18 @@ def _run_command(self, execute, copyfiles=True): if isinstance(self, MapNode): dirs2keep = [op.join(outdir, 'mapflow')] - result.outputs = clean_working_directory( + unset_outputs = clean_working_directory( result.outputs, outdir, self._interface.inputs, self.needed_outputs, self.config, dirs2keep=dirs2keep) - _save_resultfile(result, outdir, self.name) + for name in unset_outputs: + setattr(result.outputs, name, Undefined) + + _save_resultfile(result, outdir, self.name) return result def _copyfiles_to_wd(self, execute=True, linksonly=False): @@ -1131,9 +1139,12 @@ def _collate_results(self, nodes): inputs=[], outputs=self.outputs) returncode = [] + error_msgs = [] for i, nresult, err in nodes: finalresult.runtime.insert(i, None) returncode.insert(i, err) + if err is not None: + error_msgs += ['Subnode %d failed with error:\n%s' % (i, err)] if nresult: if hasattr(nresult, 'runtime'): @@ -1171,14 +1182,10 @@ def _collate_results(self, nodes): self.iterfield[0]))) setattr(finalresult.outputs, key, values) - if returncode and any([code is not None for code in returncode]): - msg = [] - for i, code in enumerate(returncode): - if code is not None: - msg += ['Subnode %d failed' % i] - msg += ['Error: %s' % str(code)] - raise Exception('Subnodes of node: %s failed:\n%s' % - (self.name, '\n'.join(msg))) + if error_msgs: + error_msgs.insert(0, 'Subnodes of node: %s failed:' % self.name) + logger.critical('\n\t\t'.join(error_msgs)) + raise Exception('\n\t\t'.join(error_msgs)) return finalresult @@ -1229,7 +1236,8 @@ def _check_iterfield(self): "have the same length. %s") % str(self.inputs)) def _run_interface(self, execute=True, updatehash=False): - """Run the mapnode interface + """ + Run the mapnode interface. This is primarily intended for serial execution of mapnode. A parallel execution requires creation of new nodes that can be spawned @@ -1258,7 +1266,7 @@ def _run_interface(self, execute=True, updatehash=False): stop_first=str2bool( self.config['execution']['stop_on_first_crash']))) # And store results - _save_resultfile(result, cwd, self.name) + _save_resultfile(result, cwd, self.name, rebase=False) # remove any node directories no longer required dirs2remove = [] for path in glob(op.join(cwd, 'mapflow', '*')): diff --git a/nipype/pipeline/engine/tests/test_nodes.py b/nipype/pipeline/engine/tests/test_nodes.py index d6349459f6..15b464f675 100644 --- a/nipype/pipeline/engine/tests/test_nodes.py +++ b/nipype/pipeline/engine/tests/test_nodes.py @@ -153,7 +153,7 @@ def func1(in1): Function(input_names=['in1'], output_names=['out'], function=func1), iterfield=['in1'], nested=False, - name='n1') + name='n2') n2.inputs.in1 = [[1, [2]], 3, [4, 5]] with pytest.raises(Exception) as excinfo: diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index d47e90dc0d..e3620a8589 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -107,7 +107,7 @@ class InputSpec(nib.TraitedSpec): out = clean_working_directory(outputs, tmpdir.strpath, inputs, needed_outputs, deepcopy(config._sections)) assert os.path.exists(outfiles[5]) - assert out.others == outfiles[5] + assert 'others' not in out config.set('execution', 'remove_unnecessary_outputs', True) out = clean_working_directory(outputs, tmpdir.strpath, inputs, needed_outputs, deepcopy(config._sections)) @@ -115,8 +115,8 @@ class InputSpec(nib.TraitedSpec): assert os.path.exists(outfiles[3]) assert os.path.exists(outfiles[4]) assert not os.path.exists(outfiles[5]) - assert out.others == nib.Undefined - assert len(out.files) == 2 + assert 'others' in out + assert 'files' not in out config.set_default_config() @@ -222,7 +222,7 @@ def test_mapnode_crash3(tmpdir): wf.base_dir = tmpdir.strpath # changing crashdump dir to current working directory (to avoid problems with read-only systems) wf.config["execution"]["crashdump_dir"] = os.getcwd() - with pytest.raises(RuntimeError): + with pytest.raises(Exception): wf.run(plugin='Linear') @@ -241,8 +241,6 @@ class output_spec(nib.TraitedSpec): def _run_interface(self, runtime): out_path = os.path.abspath(os.path.basename(self.inputs.in_str) + '_path') open(out_path, 'w').close() - print(out_path) - self._results['out_str'] = self.inputs.in_str self._results['out_path'] = out_path self._results['out_tuple'] = (out_path, self.inputs.in_str) @@ -268,18 +266,20 @@ def test_modify_paths_bug(tmpdir): open('2', 'w').close() - outputs = spc.run().outputs + results = spc.run() - # Basic check that string was not manipulated and path exists - out_path = outputs.out_path - print(out_path) - out_str = outputs.out_str + print(results.outputs) + + # Basic check that string was not manipulated + out_str = results.outputs.out_str assert out_str == '2' - assert os.path.basename(out_path) == '2_path' + + # Check path exists and is absolute + out_path = results.outputs.out_path + assert os.path.isabs(out_path) # Assert data structures pass through correctly - assert outputs.out_tuple == (out_path, out_str) - assert outputs.out_dict_path == {out_str: out_path} - assert outputs.out_dict_str == {out_str: out_str} - assert outputs.out_list == [out_str] * 2 - assert out_path.is_file() + assert results.outputs.out_tuple == (out_path, out_str) + assert results.outputs.out_dict_path == {out_str: out_path} + assert results.outputs.out_dict_str == {out_str: out_str} + assert results.outputs.out_list == [out_str] * 2 diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 99050157e8..2245c2638f 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -25,6 +25,7 @@ from ... import logging, config, LooseVersion from ...utils.filemanip import ( + indirectory, relpath, makedirs, fname_presuffix, @@ -40,7 +41,7 @@ ) from ...utils.misc import str2bool from ...utils.functions import create_function_from_source -from ...interfaces.base.traits_extension import resolve_path_traits +from ...interfaces.base.traits_extension import rebase_path_traits, resolve_path_traits from ...interfaces.base import (Bunch, CommandLine, isdefined, Undefined, InterfaceResult, traits) from ...interfaces.utility import IdentityInterface @@ -101,7 +102,6 @@ def nodelist_runner(nodes, updatehash=False, stop_first=False): except Exception: if stop_first: raise - result = node.result err = [] if result.runtime and hasattr(result.runtime, 'traceback'): @@ -189,7 +189,7 @@ def write_report(node, report_type=None, is_mapnode=False): # Init rst dictionary of runtime stats rst_dict = { 'hostname': result.runtime.hostname, - 'duration': result.runtime.duration, + 'duration': getattr(result.runtime, 'duration', ''), 'working_dir': result.runtime.cwd, 'prev_wd': getattr(result.runtime, 'prevcwd', ''), } @@ -293,85 +293,96 @@ def _protect_collapses(hastraits): return _uncollapse(hastraits.trait_get(), collapsed) -def save_resultfile(result, cwd, name): - """Save a result pklz file to ``cwd``""" +def save_resultfile(result, cwd, name, rebase=True): + """Save a result pklz file to ``cwd``.""" + cwd = os.path.abspath(cwd) resultsfile = os.path.join(cwd, 'result_%s.pklz' % name) logger.debug("Saving results file: '%s'", resultsfile) - # if result.outputs: - # try: - # collapsed = _identify_collapses(result.outputs) - # outputs = _uncollapse(result.outputs.trait_get(), collapsed) - # # Double-protect tosave so that the original, uncollapsed trait - # # is saved in the pickle file. Thus, when the loading process - # # collapses, the original correct value is loaded. - # tosave = _uncollapse(outputs.copy(), collapsed) - # except AttributeError: - # tosave = outputs = result.outputs.dictcopy() # outputs was a bunch - - savepkl(resultsfile, result) - if result.outputs: - pass - - # if result.outputs: - # for k, v in list(outputs.items()): - # setattr(result.outputs, k, v) + + if result.outputs is None: + logger.warn('Storing result file without outputs') + savepkl(resultsfile, result) + return + + try: + outputs = result.outputs.trait_get() + except AttributeError: + logger.debug('Storing non-traited results, skipping rebase of paths') + savepkl(resultsfile, result) + return + + try: + with indirectory(cwd): + # All the magic to fix #2944 resides here: + for key, val in list(outputs.items()): + val = rebase_path_traits(result.outputs.trait(key), val, cwd) + setattr(result.outputs, key, val) + savepkl(resultsfile, result) + finally: + # Reset resolved paths from the outputs dict no matter what + for key, val in list(outputs.items()): + setattr(result.outputs, key, val) def load_resultfile(path, name, resolve=True): """ - Load InterfaceResult file from path + Load InterfaceResult file from path. Parameter --------- - path : base_dir of node name : name of node Returns ------- - result : InterfaceResult structure - aggregate : boolean indicating whether node should aggregate_outputs attribute error : boolean indicating whether there was some mismatch in versions of traits used to store result and hence node needs to rerun + """ - aggregate = True - resultsoutputfile = os.path.join(path, 'result_%s.pklz' % name) result = None attribute_error = False - logger.info("Loading results file: '%s'", resultsoutputfile) - if os.path.exists(resultsoutputfile): - pkl_file = gzip.open(resultsoutputfile, 'rb') + if not os.path.exists(os.path.join(path, 'result_%s.pklz' % name)): + logger.warning("Error loading results file, no such file: '%s'", + os.path.join(path, 'result_%s.pklz' % name)) + return result, False + + with indirectory(path): + pkl_file = gzip.open('result_%s.pklz' % name, 'rb') try: result = pickle.load(pkl_file) except UnicodeDecodeError: # Was this pickle created with Python 2.x? - pickle.load(pkl_file, fix_imports=True, encoding='utf-8') - logger.warning('Successfully loaded pkl in compatibility mode') - except (traits.TraitError, AttributeError, ImportError, - EOFError) as err: - if isinstance(err, (AttributeError, ImportError)): - attribute_error = True - logger.debug('attribute error: %s probably using ' - 'different trait pickled file', str(err)) - else: - logger.debug( - 'some file does not exist. hence trait cannot be set') - else: - if result is not None and result.outputs: - aggregate = False + result = pickle.load(pkl_file, fix_imports=True, encoding='utf-8') + logger.warning('Successfully loaded pkl in compatibility mode.') + except traits.TraitError as err: + logger.warning('Error restoring outputs while loading results.\n%s.', + err) + except (AttributeError, ImportError, EOFError) as err: + attribute_error = True + logger.warning( + 'Error loading results file (probably using different trait pickled file).\n%s', + err) + finally: + pkl_file.close() - if resolve and not aggregate: - for name in list(result.outputs.get().keys()): - value = getattr(result.outputs, name) - if isdefined(value): - value = resolve_path_traits(result.outputs.trait(name), - value, path) - pkl_file.close() - logger.debug('Aggregate: %s', aggregate) - return result, aggregate, attribute_error + if result is None: + return result, attribute_error + + if result.error: + return result, attribute_error + + if result is not None and resolve: + outputs = result.outputs + for name in list(result.outputs.get().keys()): + old_value = getattr(outputs, name) + value = resolve_path_traits(outputs.trait(name), old_value, path) + setattr(outputs, name, value) + result.outputs = outputs + + return result, attribute_error def strip_temp(files, wd): @@ -1525,10 +1536,9 @@ def clean_working_directory(outputs, logger.debug('Removing files: %s', ';'.join(files2remove)) for f in files2remove: os.remove(f) - for key in outputs.copyable_trait_names(): - if key not in outputs_to_keep: - setattr(outputs, key, Undefined) - return outputs + + unset_outputs = set(outputs.copyable_trait_names()) - set(outputs_to_keep) + return unset_outputs def merge_dict(d1, d2, merge=lambda x, y: y): From 8ed089c4e4791b63a8a73181c748e001f4a09e91 Mon Sep 17 00:00:00 2001 From: oesteban Date: Thu, 18 Jul 2019 21:16:21 -0700 Subject: [PATCH 13/13] fix(Python<3.5): new test ``test_modify_paths_bug`` raising PicklingError When ``input_spec`` and/or ``output_spec`` are defined as private classes, pickle has a hard time to find the right type and breaks with Python 2 and 3.4. This problem of pickle has been exposed by the new way of pickling InterfaceResults. I believe we can live with such a restriction - i.e., (In/Out)put specs must not be scoped within the interface (and again, only for Python<3.5). cc/ @satra --- nipype/pipeline/engine/tests/test_utils.py | 41 ++++++++++++---------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/nipype/pipeline/engine/tests/test_utils.py b/nipype/pipeline/engine/tests/test_utils.py index e3620a8589..7a6a53b300 100644 --- a/nipype/pipeline/engine/tests/test_utils.py +++ b/nipype/pipeline/engine/tests/test_utils.py @@ -226,17 +226,22 @@ def test_mapnode_crash3(tmpdir): wf.run(plugin='Linear') -class StrPathConfuser(nib.SimpleInterface): - class input_spec(nib.TraitedSpec): - in_str = nib.traits.String() +class StrPathConfuserInputSpec(nib.TraitedSpec): + in_str = nib.traits.String() + + +class StrPathConfuserOutputSpec(nib.TraitedSpec): + out_tuple = nib.traits.Tuple(nib.File, nib.traits.String) + out_dict_path = nib.traits.Dict(nib.traits.String, nib.File(exists=True)) + out_dict_str = nib.traits.DictStrStr() + out_list = nib.traits.List(nib.traits.String) + out_str = nib.traits.String() + out_path = nib.File(exists=True) - class output_spec(nib.TraitedSpec): - out_tuple = nib.traits.Tuple(nib.File, nib.traits.String) - out_dict_path = nib.traits.Dict(nib.traits.String, nib.File(exists=True)) - out_dict_str = nib.traits.DictStrStr() - out_list = nib.traits.List(nib.traits.String) - out_str = nib.traits.String() - out_path = nib.File(exists=True) + +class StrPathConfuser(nib.SimpleInterface): + input_spec = StrPathConfuserInputSpec + output_spec = StrPathConfuserOutputSpec def _run_interface(self, runtime): out_path = os.path.abspath(os.path.basename(self.inputs.in_str) + '_path') @@ -266,20 +271,18 @@ def test_modify_paths_bug(tmpdir): open('2', 'w').close() - results = spc.run() - - print(results.outputs) + outputs = spc.run().outputs # Basic check that string was not manipulated - out_str = results.outputs.out_str + out_str = outputs.out_str assert out_str == '2' # Check path exists and is absolute - out_path = results.outputs.out_path + out_path = outputs.out_path assert os.path.isabs(out_path) # Assert data structures pass through correctly - assert results.outputs.out_tuple == (out_path, out_str) - assert results.outputs.out_dict_path == {out_str: out_path} - assert results.outputs.out_dict_str == {out_str: out_str} - assert results.outputs.out_list == [out_str] * 2 + assert outputs.out_tuple == (out_path, out_str) + assert outputs.out_dict_path == {out_str: out_path} + assert outputs.out_dict_str == {out_str: out_str} + assert outputs.out_list == [out_str] * 2