Gem nemo coupled launch

necessary extra components

• r.run_in_parallel (new mpi launcher)
• u.make_node_map (needed by r.run_in_parallel)
• ${HOME}/CPL_EXEC must point to a directory NOT IN THE HOME FILESYSTEM

modified components

• Um_drive.ksh
  • add sourcing gemdm_couplage_start.dot
  • add sourcing /gemdm_couplage_end.dot
• Um_model.ksh
  • add '-instances gemdm -inorder -tag @@' to call to ${TASK_BIN}/r.mpirun

modification to Um_drive.ksh (in generated model job)
##################################################################
# gemdm_couplage.dot gemdm_tiles gemdm_openmp instance_directory
#   gemdm_tiles and gemdm_openmp from gem script
##################################################################
if [[ -r ${REPCFG}/gemdm_couplage_start.dot ]] ; then
export REPCFG=${REPCFG}
cd ${HOME}/CPL_EXEC || echo "===== WARNING: parallel TMPDIR in HOME filesystem ====="
. ${REPCFG}/gemdm_couplage_start.dot ${_cpus} ${_npeOMP} ${REPCFG}/instances
cd ${HOME}
fi
. . . . . . . . .
if [[ -r ${REPCFG}/gemdm_couplage_end.dot ]] ; then
. ${REPCFG}/gemdm_couplage_end.dot
fi

before Um_lance / Chunk_lance

 export SOUMET_EXTRAS='-q hb'            (or appropriate queue)
 export GEMDM_JOB_EXTRAS='-cpus Mx1'     (M is model cpus + nemo cpus + 1)

• scripts that must be found in configuration directory
  • lance_nemo (to execute nemo)
  • gemdm_couplage_start.dot (used to activate the compound MPI launch)
  • gemdm_couplage_end.dot (things to do at termination)

 #####  a typical example for gemdm_couplage_start.dot #############
 echo "configuration directory is : '${REPCFG}' "
 s.use r.run_in_parallel as r.mpirun2
 s.use r.run_in_parallel as r.mpirun
 [[ -x ${REPCFG}/lance_nemo ]] || return
 echo "########################################################"
 echo "#        preparing for GEM/NEMO coupled run            #"
 echo "########################################################"
 set -x
 #
 #  define instances control directory
 #
 export PARALLEL_INSTANCES_DIR="${3:-${REPCFG}/instances}"
 #
 # check that instances dir exists or can be created
 mkdir -p $PARALLEL_INSTANCES_DIR && touch $PARALLEL_INSTANCES_DIR/.flag
 [[ ! -w $PARALLEL_INSTANCES_DIR/.flag ]] && echo "instances directory $PARALLEL_INSTANCES_DIR not writable, aborting" && exit 1
 rm -f $PARALLEL_INSTANCES_DIR/.flag
 #
 ((JOB_CPUS=BATCH_MPI_CPUS*OMP_NUM_THREADS))
 GEM_CPUS=${1}
 GEM_OMP=${2}
 #set value of NEMO_CPUS
 ((NEMO_CPUS=JOB_CPUS-GEM_CPUS*GEM_OMP-1))
 #NEMO_CPUS=nnn
 #
 # master launcher in background nodemap may have to be adjusted
 #
 echo "node map : $(u.make_node_map 0 ${JOB_CPUS} ${GEM_CPUS}x${GEM_OMP} ${NEMO_CPUS}x1 1x1 )"
 r.run_in_parallel -inorder -tag -instances gemdm nemo  -npex 1 \
      -nodemap $(u.make_node_map 0 ${JOB_CPUS} ${GEM_CPUS}x${GEM_OMP} ${NEMO_CPUS}x1 1x1 ) \
      -pes $((GEM_CPUS+NEMO_CPUS+1)) -pgm ${REPCFG}/Single.Abs &
 #
 # launch nemo in background with appropriate number of cpus
 # set RUN_IN_PARALLEL_EXTRAS here or use -instances nemo in nemo launcher
 #
 export RUN_IN_PARALLEL_EXTRAS='-instances nemo'
 export BATCH_MPI_CPUS=${NEMO_CPUS}
 export OMP_NUM_THREADS=1
 ${REPCFG}/lance_nemo &
 unset RUN_IN_PARALLEL_EXTRAS
 #
 # back to gemdm script
 #
 export RUN_IN_PARALLEL_EXTRAS='-instances gemdm'
 export BATCH_MPI_CPUS=${GEM_CPUS}
 export OMP_NUM_THREADS=${GEM_OMP}
 set +x

#####  a typical example for gemdm_couplage_end.dot #############
#wait for background tasks to terminate
wait

##### dummy example of lance_nemo
${TASK_BIN}/r.mpirun -pgm ${REPCFG}/Nemo.Abs -instances nemo