Gem nemo coupled launch : Différence entre versions
m |
(→modified components) |
||
| (5 révisions intermédiaires par 2 utilisateurs non affichées) | |||
| Ligne 3: | Ligne 3: | ||
*r.run_in_parallel (new mpi launcher) | *r.run_in_parallel (new mpi launcher) | ||
*u.make_node_map (needed by r.run_in_parallel) | *u.make_node_map (needed by r.run_in_parallel) | ||
| − | + | *${HOME}/CPL_EXEC must point to a directory NOT IN THE HOME FILESYSTEM | |
== modified components == | == modified components == | ||
| Ligne 25: | Ligne 25: | ||
fi | fi | ||
. . . . . . . . . | . . . . . . . . . | ||
| − | if [[ -r ${REPCFG}/gemdm_couplage_end.dot ]] | + | if [[ -r ${REPCFG}/gemdm_couplage_end.dot ]] ; then |
. ${REPCFG}/gemdm_couplage_end.dot | . ${REPCFG}/gemdm_couplage_end.dot | ||
fi | fi | ||
| Ligne 39: | Ligne 39: | ||
**gemdm_couplage_end.dot (things to do at termination) | **gemdm_couplage_end.dot (things to do at termination) | ||
| + | <pre> | ||
##### a typical example for gemdm_couplage_start.dot ############# | ##### a typical example for gemdm_couplage_start.dot ############# | ||
echo "configuration directory is : '${REPCFG}' " | echo "configuration directory is : '${REPCFG}' " | ||
| Ligne 87: | Ligne 88: | ||
export OMP_NUM_THREADS=${GEM_OMP} | export OMP_NUM_THREADS=${GEM_OMP} | ||
set +x | set +x | ||
| + | </pre> | ||
##### a typical example for gemdm_couplage_end.dot ############# | ##### a typical example for gemdm_couplage_end.dot ############# | ||
Version actuelle datée du 11 d'avril 2015 à 23:03
necessary extra components
- r.run_in_parallel (new mpi launcher)
- u.make_node_map (needed by r.run_in_parallel)
- ${HOME}/CPL_EXEC must point to a directory NOT IN THE HOME FILESYSTEM
modified components
- Um_drive.ksh
- add sourcing gemdm_couplage_start.dot
- add sourcing /gemdm_couplage_end.dot
- Um_model.ksh
- add '-instances gemdm -inorder -tag @@' to call to ${TASK_BIN}/r.mpirun
modification to Um_drive.ksh (in generated model job)
##################################################################
# gemdm_couplage.dot gemdm_tiles gemdm_openmp instance_directory
# gemdm_tiles and gemdm_openmp from gem script
##################################################################
if [[ -r ${REPCFG}/gemdm_couplage_start.dot ]] ; then
export REPCFG=${REPCFG}
cd ${HOME}/CPL_EXEC || echo "===== WARNING: parallel TMPDIR in HOME filesystem ====="
. ${REPCFG}/gemdm_couplage_start.dot ${_cpus} ${_npeOMP} ${REPCFG}/instances
cd ${HOME}
fi
. . . . . . . . .
if [[ -r ${REPCFG}/gemdm_couplage_end.dot ]] ; then
. ${REPCFG}/gemdm_couplage_end.dot
fi
before Um_lance / Chunk_lance
export SOUMET_EXTRAS='-q hb' (or appropriate queue) export GEMDM_JOB_EXTRAS='-cpus Mx1' (M is model cpus + nemo cpus + 1)
- scripts that must be found in configuration directory
- lance_nemo (to execute nemo)
- gemdm_couplage_start.dot (used to activate the compound MPI launch)
- gemdm_couplage_end.dot (things to do at termination)
##### a typical example for gemdm_couplage_start.dot #############
echo "configuration directory is : '${REPCFG}' "
s.use r.run_in_parallel as r.mpirun2
s.use r.run_in_parallel as r.mpirun
[[ -x ${REPCFG}/lance_nemo ]] || return
echo "########################################################"
echo "# preparing for GEM/NEMO coupled run #"
echo "########################################################"
set -x
#
# define instances control directory
#
export PARALLEL_INSTANCES_DIR="${3:-${REPCFG}/instances}"
#
# check that instances dir exists or can be created
mkdir -p $PARALLEL_INSTANCES_DIR && touch $PARALLEL_INSTANCES_DIR/.flag
[[ ! -w $PARALLEL_INSTANCES_DIR/.flag ]] && echo "instances directory $PARALLEL_INSTANCES_DIR not writable, aborting" && exit 1
rm -f $PARALLEL_INSTANCES_DIR/.flag
#
((JOB_CPUS=BATCH_MPI_CPUS*OMP_NUM_THREADS))
GEM_CPUS=${1}
GEM_OMP=${2}
#set value of NEMO_CPUS
((NEMO_CPUS=JOB_CPUS-GEM_CPUS*GEM_OMP-1))
#NEMO_CPUS=nnn
#
# master launcher in background nodemap may have to be adjusted
#
echo "node map : $(u.make_node_map 0 ${JOB_CPUS} ${GEM_CPUS}x${GEM_OMP} ${NEMO_CPUS}x1 1x1 )"
r.run_in_parallel -inorder -tag -instances gemdm nemo -npex 1 \
-nodemap $(u.make_node_map 0 ${JOB_CPUS} ${GEM_CPUS}x${GEM_OMP} ${NEMO_CPUS}x1 1x1 ) \
-pes $((GEM_CPUS+NEMO_CPUS+1)) -pgm ${REPCFG}/Single.Abs &
#
# launch nemo in background with appropriate number of cpus
# set RUN_IN_PARALLEL_EXTRAS here or use -instances nemo in nemo launcher
#
export RUN_IN_PARALLEL_EXTRAS='-instances nemo'
export BATCH_MPI_CPUS=${NEMO_CPUS}
export OMP_NUM_THREADS=1
${REPCFG}/lance_nemo &
unset RUN_IN_PARALLEL_EXTRAS
#
# back to gemdm script
#
export RUN_IN_PARALLEL_EXTRAS='-instances gemdm'
export BATCH_MPI_CPUS=${GEM_CPUS}
export OMP_NUM_THREADS=${GEM_OMP}
set +x
##### a typical example for gemdm_couplage_end.dot ############# #wait for background tasks to terminate wait
##### dummy example of lance_nemo
${TASK_BIN}/r.mpirun -pgm ${REPCFG}/Nemo.Abs -instances nemo
