source: util/dynamicANOVA.sh.in@ c9b32a

#!/bin/sh
#
# Performs an molecular dynamics simulation with the BOSSANOVA method

MPIRUN="/opt/packages/mpichgm-1.2.7..15/bin/mpirun.ch_gm"
#MPIRUN="/usr/bin/mpirun.mpich"
exec_prefix="@prefix@"
MOLECUILDER="@bindir@/molecuilder"
JOINER="@bindir@/joiner"
PCP="@bindir@/pcp"

function check {
        if [ ! $? -eq 0 ]; then
                echo "An error occured." | tee -a dynamic.log
                exit 1;
        fi
}

function RunSim {
        # 1 is the config file
        # 2 is the number of nodes
        # 3 further command line option
        # 4 and argument
        # set the maximum number of nodes
        MaxNodes=`cat $2 | awk 'END{print NR}'`
        gamma=`grep ProcPEGamma $1 | awk -F"\t" {'print $2'}`
        psi=`grep ProcPEPsi $1 | awk -F"\t" {'print $2'}`
        let nodes=$gamma*$psi
        if [ $nodes -gt $MaxNodes ]; then
                echo "Process $1 needs too many nodes! Breaking." | tee -a dynamic.log
                exit 1
        fi
        ${MPIRUN} -machinefile $2 -np $nodes ${PCP} -a 86000 $3 $4 $1 2>/dev/null >/dev/null
        check
}

function MultiRunSim {
        # 1 is config file
        # 2 is the number of groups
        
        # find the next free proc group
        DIR=`dirname $1`
        started=0
        while [ $started -eq 0 ]; do
                groupnr=1
                while [ $groupnr -le $2 ]; do
                        if [ ! -e "${DIR}/ProcRuns${groupnr}" ]; then
                                MaxNodes=`cat ${DIR}/ProcGroup${groupnr} | awk 'END{print NR}'`
                                gamma=`grep ProcPEGamma $1 | awk -F"\t" {'print $2'}`
                                psi=`grep ProcPEPsi $1 | awk -F"\t" {'print $2'}`
                                let nodes=$gamma*$psi
                                if [ $nodes -gt $MaxNodes ]; then
                                        echo "Process $1 needs too many nodes! Breaking." | tee -a dynamic.log
                                        exit 1
                                fi
                                echo "touch ${DIR}/ProcRuns${groupnr}" >"${DIR}/ProcBatch${groupnr}"
                                echo "${MPIRUN} -machinefile ${DIR}/ProcGroup${groupnr} -np $nodes ${PCP} -a 86000 $1 2>/dev/null >/dev/null" >>"${DIR}/ProcBatch${groupnr}"
                                echo "rm -f ${DIR}/ProcRuns${groupnr}" >>"${DIR}/ProcBatch${groupnr}"
                                /bin/sh "${DIR}/ProcBatch${groupnr}" &
                                started=1
                                let groupnr=${2}+1
                        else
                                let groupnr=$groupnr+1
                        fi
                done
                # wait a few seconds
                sleep 5
        done
}

#PBS_NODEFILE="${DIR}/machines"

# get command line options
if [ -z $4 ]; then
        echo "Usage: $0 <config file> <Order> <max. bond distance> <MaxNodes> [MaxMDsteps]"
        echo -e "\t<config file> the pcp config file of the total molecule"
        echo -e "\t<Order> the highest bond order (i.e. the cutoff number in ANOVA series expansion)"
        echo -e "\t<max. bond distance> maximum distance to look for bonds (bonds are associated by element covalent radii criterion)"
        echo -e "\t[MaxMDSteps] overrides given MaxOuterStep in config file"
        exit 1;
else
        arg=$1
        mainname=`grep mainname $arg | awk -F"\t" {'print $2'}`
        order=$2
        distance=$3
        MaxNodes=$4
        if [ -z $5 ]; then
                MaxSteps=`grep MaxOuterStep $arg | awk -F"\t" {'print $2'}`
        else
                MaxSteps=$5
        fi
        echo "Going to run for a total of $MaxSteps steps, bond order $order and maximum distance $distance of config file $arg with a total of $MaxNodes nodes." | tee -a dynamic.log
fi


# get the directory
DIR=`dirname $arg`
if [ -z "`grep $DIR $arg`" ]; then
        echo "Cannot find the directory $DIR in the config file." | tee -a dynamic.log
        exit 1;
else
        echo "Using $DIR as directory." | tee -a dynamic.log
fi


# delete old processor group files
rm ${DIR}/ProcGroup* -f
rm ${DIR}/ProcRuns* -f
rm ${DIR}/ProcBatch* -f

# put nodes into groups
gamma=`grep ProcPEGamma $arg | awk -F"\t" {'print $2'}`
psi=`grep ProcPEPsi $arg | awk -F"\t" {'print $2'}`
let nodes=$gamma*$psi
let divisor=$MaxNodes/$nodes
echo "Using $divisor processor groups." | tee -a dynamic.log
nodenr=0
groupnr=1
for node in `cat <$PBS_NODEFILE`; do
        let nodenr=$nodenr+1
        #echo "Current node $nodenr is $node." | tee -a dynamic.log
        let currentgrouplimit=$groupnr*$nodes
        if [ $currentgrouplimit -lt $nodenr ]; then
                let groupnr=$groupnr+1
        fi
        #echo "Putting into group $groupnr." | tee -a dynamic.log
        echo "$node" >>"${DIR}/ProcGroup${groupnr}"
done
i=0
while [ $i -lt $groupnr ]; do
        let i=$i+1
        echo "Group nr. $i" | tee -a dynamic.log
        echo "===========" | tee -a dynamic.log
        cat <"${DIR}/ProcGroup${i}"
        cat <"${DIR}/ProcGroup${i}" >>dynamic.log
        echo -e "\n" | tee -a dynamic.log
done

# copy first conf
cp $arg ${arg}.MD
cp $arg ${arg}.MD.MD

# create fake pcp.energy.all by parsing a bit of the config file
MaxLevel=`grep MaxLevel $arg | awk -F"\t" '{print $2'}`
echo -e "Time\tTotal\tKinetic\tNonLocal\tCorrelation\tExchange\tPseudo\tHartree\t-Gauss\tEwald\tIonKin\tETotal\t" >${DIR}/pcp.full.energy.all
j=0
while [ $j -lt $MaxLevel ]; do
  let j=$j+1
  i=0
  while [ $i -lt 12 ]; do
    let i=$i+1
    printf "%e\t" 0 >>${DIR}/pcp.full.energy.all
  done
  echo -n -e "\n" >>${DIR}/pcp.full.energy.all
done
echo "$MaxLevel lines created in ${DIR}/pcp.full.energy.all."

# create fake pcp.forces.all by parsing a bit of the config file
MaxTypes=`grep MaxTypes $arg | awk -F"\t" '{print $2'}`
MaxIons=0
i=0
echo -e "Type\tNo\tPos0\tPos1\tPos2\tTotal0\tTotal1\tTotal2\tLocal0\tLocal1\tLocal2\tNLocal0\tNLocal1\tNLocal2\tMagnetic0\tMagnetic1\tMagnetic2\tEwald0\tEwald1\tEwald2" >${DIR}/pcp.full.forces.all
while [ $i -lt $MaxTypes ]; do
  let type=$i+1
  j=`grep -E "^Ion_Type${type}[^_]+" $arg | awk -F"\t" '{print $2'}`
  echo "Ion type ${type} has $j ions."
  ionnr=0
  while [ $ionnr -lt $j ]; do
    echo -n -e "$i\t$ionnr\t" >>${DIR}/pcp.full.forces.all
    k=0
    while [ $k -lt 18 ]; do
      printf "%e\t" 0 >>${DIR}/pcp.full.forces.all
      let k=$k+1
    done
    echo -n -e "\n" >>${DIR}/pcp.full.forces.all
    let ionnr=$ionnr+1
  done
  let MaxIons=$MaxIons+$j
  let i=$i+1
done
echo -e "MeanForce:\t0." >>${DIR}/pcp.full.forces.all
echo "$MaxIons lines created in ${DIR}/pcp.full.forces.all."

i=1;
while [ $i -le $MaxSteps ]; do
# break down the molecule with molecuilder
        sed -i -e "s#MaxOuterStep.*\##MaxOuterStep\t0\t\##" $arg.MD.MD
        echo -n "Fragmenting ... " | tee -a dynamic.log
        PWD=`pwd`
        cd `dirname ${MOLECUILDER}`
  ./`basename ${MOLECUILDER}` ${arg}.MD.MD -f $distance $order A
        check
        cd $PWD
        echo "done." | tee -a dynamic.log

# get the number of digits of the fragment count
        digits=1
        while [ ! -e ${DIR}/BondFragment`printf "%0${digits}d" 0`.conf ]; do
                let digits=$digits+1
        done
        echo "Found $digits digits for the fragment number." | tee -a dynamic.log

# get the fragment count
        frag=0
        while [ -e ${DIR}/BondFragment`printf "%0${digits}d" $frag`.conf ]; do
                # unset MaxOuterStep in config file
                sed -i -e "s#MaxOuterStep.*\##MaxOuterStep\t0\t\##" ${DIR}/BondFragment`printf "%0${digits}d" $frag`.conf
                mkdir -p ${DIR}/BondFragment`printf "%0${digits}d" $frag`
                let frag=$frag+1
        done
        echo "There are $frag fragments." | tee -a dynamic.log

# evaluate each fragment
  j=0;
  while [ $j -lt $frag ]; do
                number=`printf "%0${digits}d" $j`
                echo -n  "Starting calculation of Fragment $number at step $i ... " | tee -a dynamic.log
                MultiRunSim ${DIR}/BondFragment${number}.conf $divisor
                echo "done." | tee -a dynamic.log
    let j=$j+1
  done

# wait till all ProcRuns files are gone
        echo "Waiting for all running jobs at step $i to end ... " | tee -a dynamic.log
        while [ ! -z "`ls ${DIR}/ProcRuns*`" ]; do
                sleep 15
        done
        echo "done." | tee -a dynamic.log

# join the resulting forces into a single file
        cp ${DIR}/pcp.full.energy.all ${DIR}/pcp.energy.all
        cp ${DIR}/pcp.full.forces.all ${DIR}/pcp.forces.all
        echo -n "Joining fragment energies ... " | tee -a dynamic.log
  ${JOINER} ${DIR}/ $mainname >/dev/null 2>/dev/null
        check
        echo "done." | tee -a dynamic.log

# move the ions by calling pcp with this force file
        sed -e "s#MaxOuterStep.*\##MaxOuterStep\t$i\t\##" ${arg}.MD.MD >${arg}.MD
        echo -n "Moving ions with obtained forces at step $i ... " | tee -a dynamic.log
        RunSim ${arg}.MD $PBS_NODEFILE -F "${DIR}/pcp.Order${order}.forces.all"
        echo "done" | tee -a dynamic.log

# last of all, put "joined" energy and forces under this step
        cp ${DIR}/pcp.Order${order}.energy.all ${DIR}/pcp.step${i}.energy.all
        cp ${DIR}/pcp.Order${order}.forces.all ${DIR}/pcp.step${i}.forces.all
  
# next step
        let i=$i+1
done

# draw densities of each step
sed -e "s#DoOutVis.*\##DoOutVis\t2\t\##" ${arg}.MD.MD >${arg}.MD
echo -n "Calling simulation to draw final densities of all steps ... " | tee -a dynamic.log
RunSim ${arg}.MD $PBS_NODEFILE
echo "done." | tee -a dynamic.log

exit 0