Merge pull request #3061 from effigies/fix/results_file

FIX: Minimize scope for directory changes while loading results file

Author: effigies

nipype/utils/filemanip.py

@@ -699,37 +699,49 @@ def loadpkl(infile):
     infile = Path(infile)
     fmlogger.debug('Loading pkl: %s', infile)
     pklopen = gzip.open if infile.suffix == '.pklz' else open
+
+    with SoftFileLock('%s.lock' % infile):
+        with pklopen(str(infile), 'rb') as pkl_file:
+            pkl_contents = pkl_file.read()
+
     pkl_metadata = None
 
+    # Look if pkl file contains version metadata
+    idx = pkl_contents.find(b'\n')
+    if idx >= 0:
+        try:
+            pkl_metadata = json.loads(pkl_contents[:idx])
+        except (UnicodeDecodeError, json.JSONDecodeError):
+            # Could not get version info
+            pass
+        else:
+            # On success, skip JSON metadata
+            pkl_contents = pkl_contents[idx + 1:]
+
+    # Pickle files may contain relative paths that must be resolved relative
+    # to the working directory, so use indirectory while attempting to load
     unpkl = None
-    with indirectory(infile.parent):
-        with SoftFileLock('%s.lock' % infile.name):
-            with pklopen(infile.name, 'rb') as pkl_file:
-                try:  # Look if pkl file contains version file
-                    pkl_metadata_line = pkl_file.readline()
-                    pkl_metadata = json.loads(pkl_metadata_line)
-                except (UnicodeDecodeError, json.JSONDecodeError):
-                    # Could not get version info
-                    pkl_file.seek(0)
-                try:
-                    unpkl = pickle.load(pkl_file)
-                except UnicodeDecodeError:
-                    # Was this pickle created with Python 2.x?
-                    unpkl = pickle.load(pkl_file, fix_imports=True, encoding='utf-8')
-                    fmlogger.info('Successfully loaded pkl in compatibility mode.')
-                # Unpickling problems
-                except Exception as e:
-                    if pkl_metadata and 'version' in pkl_metadata:
-                        from nipype import __version__ as version
-                        if pkl_metadata['version'] != version:
-                            fmlogger.error("""\
+    try:
+        with indirectory(infile.parent):
+            unpkl = pickle.loads(pkl_contents)
+    except UnicodeDecodeError:
+        # Was this pickle created with Python 2.x?
+        with indirectory(infile.parent):
+            unpkl = pickle.loads(pkl_contents, fix_imports=True, encoding='utf-8')
+        fmlogger.info('Successfully loaded pkl in compatibility mode.')
+    # Unpickling problems
+    except Exception as e:
+        if pkl_metadata and 'version' in pkl_metadata:
+            from nipype import __version__ as version
+            if pkl_metadata['version'] != version:
+                fmlogger.error("""\
 Attempted to open a results file generated by Nipype version %s, \
 with an incompatible Nipype version (%s)""", pkl_metadata['version'], version)
-                            raise e
-                    fmlogger.warning("""\
+                raise e
+        fmlogger.warning("""\
 No metadata was found in the pkl file. Make sure you are currently using \
 the same Nipype version from the generated pkl.""")
-                    raise e
+        raise e
 
     if unpkl is None:
         raise ValueError('Loading %s resulted in None.' % infile)