Merge pull request #211 from HALFpipe/dev/fix/memory

Make memory estimation even more conservative
HALFpipe · Sep 13, 2021 · 9e9b734 · 9e9b734
2 parents eeac223 + c9580d0
commit 9e9b734
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 5 deletions.
diff --git a/halfpipe/cluster.py b/halfpipe/cluster.py
@@ -127,7 +127,7 @@ def create_example_script(workdir, graphs: OrderedDictT[str, Any], opts):
     n_cpus = 2
     mem_gb: float = max(
         node.mem_gb for graph in graphs.values() for node in graph.nodes
-    ) * 1.5 + 2.0  # two gigabytes for the python process plus the actual memory
+    ) * 1.5 + 3.0  # three gigabytes for the python process plus the actual memory
     mem_mb: int = int(ceil(mem_gb * 1024))
 
     extra_args = ""

diff --git a/halfpipe/memory.py b/halfpipe/memory.py
@@ -221,4 +221,10 @@ def memory_limit() -> float:
         if ml is not None
     ]
 
-    return min(memory_limits) * 0.9  # nipype uses 90% of max as a safety precaution
+    memory_limit = min(memory_limits) * 0.9  # nipype uses 90% of max as a safety precaution
+
+    process = psutil.Process(pid=os.getpid())
+    resident_set_size = process.memory_info().rss * ureg.bytes
+    memory_limit -= resident_set_size.m_as(ureg.gigabytes)  # subtract memory used by current process
+
+    return memory_limit
diff --git a/halfpipe/workflow/memory.py b/halfpipe/workflow/memory.py
@@ -132,8 +132,6 @@ def patch_mem_gb(node: pe.Node, memcalc: MemoryCalculator):
         name.endswith(s)
         for s in [
             "ica_aroma_wf.melodic",
-            "ica_aroma_wf.calc_bold_mean",
-            "ica_aroma_wf.calc_median_val",
         ]
     ):
         node._mem_gb = memcalc.series_std_gb
@@ -145,6 +143,8 @@ def patch_mem_gb(node: pe.Node, memcalc: MemoryCalculator):
             "bold_confounds_wf.rois_plot",
             "bold_confounds_wf.signals",
             "bold_confounds_wf.tcompcor",
+            "ica_aroma_wf.calc_bold_mean",
+            "ica_aroma_wf.calc_median_val",
         ]
     ):
         node._mem_gb = 2 * memcalc.series_std_gb

diff --git a/halfpipe/workflow/report/func.py b/halfpipe/workflow/report/func.py
@@ -109,7 +109,7 @@ def init_func_report_wf(workdir=None, name="func_report_wf", memcalc=MemoryCalcu
     workflow.connect(epi_norm_rpt, "out_report", make_resultdicts, "epi_norm_rpt")
 
     # plot the tsnr image
-    tsnr = pe.Node(nac.TSNR(), name="compute_tsnr", mem_gb=2 * memcalc.series_std_gb)
+    tsnr = pe.Node(nac.TSNR(), name="compute_tsnr", mem_gb=memcalc.series_std_gb * 2.5)
     workflow.connect(select_std, "bold_std", tsnr, "in_file")
 
     tsnr_rpt = pe.Node(PlotEpi(), name="tsnr_rpt", mem_gb=memcalc.min_gb)