Enable support for large execution reports #547

This commit render the execution report boxplots 
using the report summary data collected by nextflow 
at runtime. 

The trace data is omitted when the number of tasks 
is greater than 10’000 (default) and the related 
table is not shown

Author: pditommaso

src/main/groovy/nextflow/trace/ReportObserver.groovy

@@ -235,7 +235,7 @@ class ReportObserver implements TraceObserver {
      */
     protected String renderTasksJson() {
         final r = getRecords()
-        r.size()<=maxTasks ? renderJsonData(r.values()) : '[]'
+        r.size()<=maxTasks ? renderJsonData(r.values()) : 'null'
     }
 
     /**

src/main/groovy/nextflow/trace/ReportSummary.groovy

@@ -41,8 +41,35 @@ class ReportSummary {
         mappers.time = { TraceRecord record -> record.get('realtime') as Double }
         mappers.reads = { TraceRecord record -> record.get('read_bytes') as Double }
         mappers.writes = { TraceRecord record -> record.get('write_bytes') as Double}
+
+        mappers.cpuUsage = { TraceRecord record ->
+            final Double pcpu = record.get('%cpu') as Double
+            final int ncpu = (record.get('cpus') ?: 1) as int
+            if( !pcpu )
+                return null
+            return pcpu / ncpu as Double
+        }
+
+        mappers.memUsage = { TraceRecord record ->
+            final vmem = record.get('vmem') as Double
+            final request = record.get('memory') as Long
+            if( !vmem ) return null
+            if( !request ) return null
+            return vmem / request * 100 as Double
+        }
+
+
+        mappers.timeUsage = { TraceRecord record ->
+            final realtime = record.get('realtime') as Long
+            final request = record.get('time') as Long
+            if( !realtime ) return null
+            if( !request ) return null
+            return realtime / request * 100 as Double
+        }
     }
 
+
+
     /**
      * Hold the summary for each series ie. cpu, memory, time, disk reads, disk writes
      */
@@ -118,8 +145,6 @@ class ReportSummary {
 
         private int count
 
-        private Integer digits = 2
-
         @PackageScope Double min
 
         @PackageScope Double max
@@ -154,11 +179,8 @@ class ReportSummary {
             }
         }
 
-        private double round( double value ) {
-            if( digits == null )
-                return value
-            final x = Math.pow(10,digits)
-            Math.round( value * x ) / x
+        private BigDecimal round( double value ) {
+            Math.round( value * 100 ) / 100
         }
 
         void add( TraceRecord record ) {
@@ -208,11 +230,16 @@ class ReportSummary {
             result.q2 = round(quantile(sorted, 50))
             result.q3 = round(quantile(sorted, 75))
             result.max = round(quantile(sorted, 100))
+            // discard entry with all zero 
+            if( result.min == 0 && result.min == result.max  )
+                return null
+
             result.minLabel = minLabel
             result.maxLabel = maxLabel
             result.q1Label = q1Label
             result.q2Label = q2Label
             result.q3Label = q3Label
+
             return result
         }
 

src/main/resources/nextflow/trace/ReportTemplate.html

@@ -208,76 +208,93 @@ <h2 id="resources" style="padding-top: 80px;">Resource Usage</h2>
     <h4>CPU Usage</h4>
     <ul class="nav nav-tabs" id="cpuplot_tabs" role="tablist">
       <li class="nav-item">
-        <a class="nav-link active" id="pctcpuplot_tablink" data-toggle="tab" href="#pctcpuplot_tabpanel" role="tab" aria-controls="cpuplot_tabpanel" aria-expanded="true">
-          % Allocated
+        <a class="nav-link active" id="cpuplot_tablink" data-toggle="tab" href="#cpuplot_tabpanel" role="tab" aria-controls="pctcpuplot_tabpanel" aria-expanded="false">
+          Raw Usage
         </a>
       </li>
       <li class="nav-item">
-        <a class="nav-link" id="cpuplot_tablink" data-toggle="tab" href="#cpuplot_tabpanel" role="tab" aria-controls="pctcpuplot_tabpanel" aria-expanded="false">
-          Raw Usage
+        <a class="nav-link" id="pctcpuplot_tablink" data-toggle="tab" href="#pctcpuplot_tabpanel" role="tab" aria-controls="cpuplot_tabpanel" aria-expanded="true">
+          % Allocated
         </a>
       </li>
     </ul>
     <div class="tab-content" id="cpuplot_tabcontent">
-      <div class="tab-pane fade show active" id="pctcpuplot_tabpanel" role="tabpanel">
-        <div id="pctcpuplot"></div>
-      </div>
-      <div class="tab-pane fade" id="cpuplot_tabpanel" role="tabpanel">
+      <div class="tab-pane fade show active" id="cpuplot_tabpanel" role="tabpanel">
         <div id="cpuplot"></div>
       </div>
+      <div class="tab-pane fade" id="pctcpuplot_tabpanel" role="tabpanel">
+        <div id="pctcpuplot"></div>
+      </div>
+
     </div>
 
     <h4>Memory Usage</h4>
     <ul class="nav nav-tabs" id="memplot_tabs" role="tablist">
       <li class="nav-item">
-        <a class="nav-link active" id="pctmemplot_tablink" data-toggle="tab" href="#pctmemplot_tabpanel" role="tab" aria-controls="pctmemplot_tabpanel" aria-expanded="true">
-          % Allocated
+        <a class="nav-link active" id="memplotlot_tablink" data-toggle="tab" href="#memplotlot_tabpanel" role="tab" aria-controls="memplotlot_tabpanel" aria-expanded="false">
+          Raw Usage
         </a>
       </li>
       <li class="nav-item">
-        <a class="nav-link" id="memplotlot_tablink" data-toggle="tab" href="#memplotlot_tabpanel" role="tab" aria-controls="memplotlot_tabpanel" aria-expanded="false">
-          Raw Usage
+        <a class="nav-link" id="pctmemplot_tablink" data-toggle="tab" href="#pctmemplot_tabpanel" role="tab" aria-controls="pctmemplot_tabpanel" aria-expanded="true">
+          % Allocated
         </a>
       </li>
     </ul>
     <div class="tab-content" id="memplot_tabcontent">
-      <div class="tab-pane fade show active" id="pctmemplot_tabpanel" role="tabpanel">
-        <div id="pctmemplot"></div>
-      </div>
-      <div class="tab-pane fade" id="memplotlot_tabpanel" role="tabpanel">
+      <div class="tab-pane fade show active" id="memplotlot_tabpanel" role="tabpanel">
         <div id="memplot"></div>
       </div>
+      <div class="tab-pane fade" id="pctmemplot_tabpanel" role="tabpanel">
+        <div id="pctmemplot"></div>
+      </div>
     </div>
 
     <h4 id="timeplot_header">Job Duration</h4>
     <ul class="nav nav-tabs" id="timeplot_tabs" role="tablist">
       <li class="nav-item">
-        <a class="nav-link active" id="pcttimeplot_tablink" data-toggle="tab" href="#pcttimeplot_tabpanel" role="tab" aria-controls="pcttimeplot_tabpanel" aria-expanded="true">
-          % Allocated
+        <a class="nav-link active" id="timeplot_tablink" data-toggle="tab" href="#timeplot_tabpanel" role="tab" aria-controls="timeplot_tabpanel" aria-expanded="false">
+          Raw Usage
         </a>
       </li>
       <li class="nav-item">
-        <a class="nav-link" id="timeplot_tablink" data-toggle="tab" href="#timeplot_tabpanel" role="tab" aria-controls="timeplot_tabpanel" aria-expanded="false">
-          Raw Usage
+        <a class="nav-link" id="pcttimeplot_tablink" data-toggle="tab" href="#pcttimeplot_tabpanel" role="tab" aria-controls="pcttimeplot_tabpanel" aria-expanded="true">
+          % Allocated
         </a>
       </li>
     </ul>
     <div class="tab-content" id="timeplot_tabcontent">
-      <div class="tab-pane fade show active" id="pcttimeplot_tabpanel" role="tabpanel">
-        <div id="pcttimeplot"></div>
-      </div>
-      <div class="tab-pane fade" id="timeplot_tabpanel" role="tabpanel">
+      <div class="tab-pane fade show active" id="timeplot_tabpanel" role="tabpanel">
         <div id="timeplot"></div>
       </div>
+      <div class="tab-pane fade" id="pcttimeplot_tabpanel" role="tabpanel">
+        <div id="pcttimeplot"></div>
+      </div>
     </div>
 
-    <div id="readwriteplot_div">
-      <h4>Disk Read / Write</h4>
-      <p>Number of bytes directly read from disk added to the number of bytes the process originally
-        dirtied in the page-cache for each process.</p>
-      <div id="readwriteplot"></div>
+    <h4 id="readwriteplot_header">Disk I/O</h4>
+    <ul class="nav nav-tabs" id="readwriteplot_tabs" role="tablist">
+      <li class="nav-item">
+        <a class="nav-link active" id="readplot_tablink" data-toggle="tab" href="#readplot_tabpanel" role="tab" aria-controls="readplot_tabpanel" aria-expanded="true">
+          Read
+        </a>
+      </li>
+      <li class="nav-item">
+        <a class="nav-link" id="writeplot_tablink" data-toggle="tab" href="#writeplot_tabpanel" role="tab" aria-controls="writeplot_tabpanel" aria-expanded="false">
+          Write
+        </a>
+      </li>
+    </ul>
+    <div class="tab-content" id="readwriteplot_tabcontent">
+      <div class="tab-pane fade show active" id="readplot_tabpanel" role="tabpanel">
+        <div id="readplot"></div>
+      </div>
+      <div class="tab-pane fade" id="writeplot_tabpanel" role="tabpanel">
+        <div id="writeplot"></div>
+      </div>
     </div>
 
+    <div id="table-container">
     <h2 id="tasks" style="padding-top: 80px;">Tasks</h2>
     <p>This table shows information about each task in the workflow. Use the search box on the right
       to filter rows for specific values. Clicking headers will sort the table by that value and
@@ -291,49 +308,11 @@ <h2 id="tasks" style="padding-top: 80px;">Tasks</h2>
     </form>
   </div>
   <div class="container-fluid">
-    <table class="table small table-striped" id="tasks_table">
-      <thead>
-        <tr>
-          <th>task_id</th>
-          <th>process</th>
-          <th>tag</th>
-          <th>status</th>
-          <th>hash</th>
-          <th>allocated cpus</th>
-          <th>%cpu</th>
-          <th>allocated memory (bytes)</th>
-          <th>%mem</th>
-          <th>vmem</th>
-          <th>rss</th>
-          <th>peak_vmem</th>
-          <th>peak_rss</th>
-          <th>allocated time</th>
-          <th>duration</th>
-          <th>realtime</th>
-          <th>script</th>
-          <th>exit</th>
-          <th>submit</th>
-          <th>start</th>
-          <th>complete</th>
-          <th>rchar</th>
-          <th>wchar</th>
-          <th>syscr</th>
-          <th>syscw</th>
-          <th>read_bytes</th>
-          <th>write_bytes</th>
-          <th>native_id</th>
-          <th>name</th>
-          <th>module</th>
-          <th>container</th>
-          <th>disk</th>
-          <th>attempt</th>
-          <th>scratch</th>
-          <th>workdir</th>
-        </tr>
-      </thead>
-      <tbody></tbody>
-    </table>
-
+    <table class="table small table-striped" id="tasks_table"></table>
+  </div>
+  </div>
+  <div id="no-table-container" class="container">
+    (tasks table omitted because the dataset is too big)
   </div>
 
   <footer>