#!/bin/bash
# RemoteInstantSCRIPT:makeMPITemplate
# SubmissionScripts/Distributor/Instant/SCRIPT/mpi

trap cleanup HUP INT QUIT ABRT TERM

cleanup()
{
   hosts=$(cat @@{NODEFILE})
   for host in ${hosts} ; do
      slavePids=$(ssh ${host} pgrep -P 1 -u ${USER} -f "@@{TIME_RESULTS}" 2> /dev/null)
      for slavePid in ${slavePids} ; do
         ssh ${host} pkill -TERM -P ${slavePid} 2> /dev/null
      done
   done
   sleep 2

   for host in ${hosts} ; do
      slavePids=$(ssh ${host} pgrep -P 1 -u ${USER} -f "@@{TIME_RESULTS}" 2> /dev/null)
      for slavePid in ${slavePids} ; do
         ssh ${host} pkill -KILL -P ${slavePid} 2> /dev/null
      done
   done

   if [ ! -s @@{TS_FINISH} ] ; then
      date +"%s" > @@{TS_FINISH}
   fi
   touch @@{TIME_RESULTS}-0
   cat @@{TIME_RESULTS}-[0-9]* >> @@{TIME_RESULTS}
   rm -f @@{TIME_RESULTS}-[0-9]*
}

exitStatus=0
# Change to directory where job was submitted.
cd @@{WORKINGDIRECTORY}
export PATH=@@{WORKINGDIRECTORY}:${PATH}

date +"%s" > @@{TS_TRANSFERRED}
date +"%s" > @@{TS_START}

@@{PREMANAGERCOMMANDS}
@@{MANAGERCOMMAND} `pwd`/@@{APPSCRIPTNAME} &
wait %1
exitStatus=$?
@@{POSTMANAGERCOMMANDS}

date +"%s" > @@{TS_FINISH}

touch @@{TIME_RESULTS}-0
cat @@{TIME_RESULTS}-[0-9]* >> @@{TIME_RESULTS}
rm -f @@{TIME_RESULTS}-[0-9]*

exit ${exitStatus}
