Skip to content

Commit 686aef9

Browse files
committed
ENH: Lightweight node cache checking
Generating the hashvalue when outputs are not ready at cache check stage when the node's directory does not exist (or no results file is in there) leads to #3014. This PR preempts those problems by delaying the hashval calculation.
1 parent 01de656 commit 686aef9

File tree

1 file changed

+12
-9
lines changed

1 file changed

+12
-9
lines changed

nipype/pipeline/engine/nodes.py

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -293,27 +293,29 @@ def is_cached(self, rm_outdated=False):
293293
"""
294294
outdir = self.output_dir()
295295

296-
# Update hash
297-
hashed_inputs, hashvalue = self._get_hashval()
298-
299296
# The output folder does not exist: not cached
300-
if not op.exists(outdir):
301-
logger.debug('[Node] Directory not found "%s".', outdir)
297+
if not op.exists(outdir) or \
298+
not op.exists(op.join(outdir, 'result_%s.pklz' % self.name)):
299+
logger.debug('[Node] Not cached "%s".', outdir)
302300
return False, False
303301

304-
hashfile = op.join(outdir, '_0x%s.json' % hashvalue)
305-
cached = op.exists(hashfile)
306-
307-
# Check if updated
302+
# Check if there are hashfiles
308303
globhashes = glob(op.join(outdir, '_0x*.json'))
309304
unfinished = [
310305
path for path in globhashes
311306
if path.endswith('_unfinished.json')
312307
]
313308
hashfiles = list(set(globhashes) - set(unfinished))
309+
310+
# Update hash
311+
hashed_inputs, hashvalue = self._get_hashval()
312+
313+
hashfile = op.join(outdir, '_0x%s.json' % hashvalue)
314314
logger.debug('[Node] Hashes: %s, %s, %s, %s',
315315
hashed_inputs, hashvalue, hashfile, hashfiles)
316316

317+
cached = op.exists(hashfile)
318+
317319
# No previous hashfiles found, we're all set.
318320
if cached and len(hashfiles) == 1:
319321
assert(hashfile == hashfiles[0])
@@ -441,6 +443,7 @@ def run(self, updatehash=False):
441443
for outdatedhash in glob(op.join(self.output_dir(), '_0x*.json')):
442444
os.remove(outdatedhash)
443445

446+
self._get_hashval()
444447
# Hashfile while running
445448
hashfile_unfinished = op.join(
446449
outdir, '_0x%s_unfinished.json' % self._hashvalue)

0 commit comments

Comments
 (0)