From 1d6ce20cded62eb6d02ee5ccf5398e5d938b40e3 Mon Sep 17 00:00:00 2001
From: Phil Miller <unmobile+gh@gmail.com>
Date: Sat, 2 Apr 2022 23:12:53 -0700
Subject: Allow process signals to trigger checkpoint or break behavior (#2896)

* Beginnings of signal handling machinery

* Add tentative logic to make checkpoint call

* Adapt formatting slightly

* Add calls to read signals and set up signal handlers

* Initialize signal flag array

* Add parsing of signal names, and fix some whitespace issues

* Skip signal setup on Windows

* added checkpoint and break signal inputs to picmi.py

* Address initial review requests

* Correct comment to match changed code

* Convert maximum signal number to a symbolic name

* Always parse signal input, and error out on Windows or wherever it may be unsupported

* Typo fix

* Add missing reset of checkpoint signal flag

* Add reset of break signal, in support of Python or library usage

* Test for a configured checkpoint diag when asked to checkpoint on a signal

* Fix typo in Linux code path

* Clean up MPI support

* Use symbolic name for maximum signal number

* Fix unused variable in the no-MPI case

* Add missing header inclusions

* Switch signal parsing to an enumerated table

* Test signal handling for Linux, not GNU C library

* Avoid another magic number

* Update MPI_Ibcast call to match symbolic array length

* Update loop over signal flags to use symbolic limit

* Match #includes to usage

* Add omitted C++ std <atomic> header include

* Guard entire set of signal definitions as *nix-only, not for Windows

* Broaden Windows exclusion to avoid zero-length array that displeases MSVC++

* Check return value from sigaction()

* Convert conditional calls to Abort() to assertions

* Move check for platform support to input parsing

* Shift signal handling code over toward ABLASTR to share with ImpactX and Hipace++

* Minor cleanup

* A bit more cleanup

* Fix formatting nits

* Add AMReX error handling on MPI calls

* Add ABLASTR signal handling code to GNU makefile too

* Document new input parameters

* Use ABLASTR assertion macros in ABLASTR code

* Convert requests limit value to a requests array size

* Generalize signal handling to an arbitrary set of potential actions

* Rename class to match usage and file name

* Stick stuff in ABLASTR namespace

* Indent conditional includes as requested

Co-authored-by: Roelof <roelof.groenewald@modernelectron.com>
---
 Source/Evolve/WarpXEvolve.cpp | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

(limited to 'Source/Evolve/WarpXEvolve.cpp')

diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp
index 049806059..78c4713da 100644
--- a/Source/Evolve/WarpXEvolve.cpp
+++ b/Source/Evolve/WarpXEvolve.cpp
@@ -33,6 +33,8 @@
 #include "Utils/WarpXProfilerWrapper.H"
 #include "Utils/WarpXUtil.H"
 
+#include <ablastr/utils/SignalHandling.H>
+
 #include <AMReX.H>
 #include <AMReX_Array.H>
 #include <AMReX_BLassert.H>
@@ -53,6 +55,7 @@
 #include <vector>
 
 using namespace amrex;
+using ablastr::utils::SignalHandling;
 
 void
 WarpX::Evolve (int numsteps)
@@ -79,6 +82,8 @@ WarpX::Evolve (int numsteps)
         WARPX_PROFILE("WarpX::Evolve::step");
         Real evolve_time_beg_step = amrex::second();
 
+        CheckSignals();
+
         multi_diags->NewIteration();
 
         // Start loop on time steps
@@ -342,6 +347,8 @@ WarpX::Evolve (int numsteps)
         Real evolve_time_end_step = amrex::second();
         evolve_time += evolve_time_end_step - evolve_time_beg_step;
 
+        HandleSignals();
+
         if (verbose) {
             amrex::Print()<< "STEP " << step+1 << " ends." << " TIME = " << cur_time
                         << " DT = " << dt[0] << "\n";
@@ -350,7 +357,7 @@ WarpX::Evolve (int numsteps)
                       << " s; Avg. per step = " << evolve_time/(step-step_begin+1) << " s\n";
         }
 
-        if (cur_time >= stop_time - 1.e-3*dt[0]) {
+        if (cur_time >= stop_time - 1.e-3*dt[0] || SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK)) {
             break;
         }
 
@@ -949,3 +956,21 @@ WarpX::applyMirrors(Real time){
         }
     }
 }
+
+void
+WarpX::CheckSignals()
+{
+    SignalHandling::CheckSignals();
+}
+
+void
+WarpX::HandleSignals()
+{
+    SignalHandling::WaitSignals();
+
+    // SIGNAL_REQUESTS_BREAK is handled directly in WarpX::Evolve
+
+    if (SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_CHECKPOINT)) {
+        multi_diags->FilterComputePackFlushLastTimestep( istep[0] );
+    }
+}
-- 
cgit v1.2.3