[112] | 1 | #!/bin/bash |
---|
| 2 | |
---|
| 3 | #----------------------------------------------------------- |
---|
| 4 | # $Id: distexe.sh 138 2010-05-12 17:34:01Z rosiere $ |
---|
| 5 | #----------------------------------------------------------- |
---|
| 6 | |
---|
[138] | 7 | #-----[ global variable ]----------------------------------- |
---|
| 8 | declare VERSION="1.3" |
---|
| 9 | declare -i SLEEP=2; |
---|
| 10 | declare -i TRAP=0; |
---|
[124] | 11 | |
---|
[138] | 12 | # Need : test, echo, cd, dirname, basename, ssh, ps aux, ls, wc, sed |
---|
| 13 | # lock.sh, unlock.sh, date.sh, nb_cpu.sh, distexe_execute_n.sh, distexe_execute.sh |
---|
[112] | 14 | |
---|
| 15 | #-----[ distexe_usage ]------------------------------------- |
---|
| 16 | function distexe_usage () |
---|
| 17 | { |
---|
| 18 | echo "Usage : ${0} file [ dir ]"; |
---|
| 19 | echo "Arguments : "; |
---|
| 20 | echo " * file : list of command"; |
---|
| 21 | echo " * dir : work directory (default is current directory)."; |
---|
| 22 | # echo " * nb_process : number of process (default (and maximum) is the number of processor)"; |
---|
| 23 | echo ""; |
---|
| 24 | echo "Note : "; |
---|
| 25 | echo " * File content list of command. Each line is execute by an host."; |
---|
| 26 | echo " A line can content many shell command."; |
---|
| 27 | echo " Don't forgot the final end of line (else the last command is not executed)"; |
---|
| 28 | echo " * For each command, a directory (in work directory) is created : Task_X (X is the number of line in command file)."; |
---|
[138] | 29 | echo " The command is execute in this directory. (Warning, set your environment ! but we source .bashrc)."; |
---|
| 30 | echo " Three file is generate :"; |
---|
| 31 | echo " - \"distexe.info\" content information as host, pid ..."; |
---|
| 32 | echo " - \"distexe.output\" is the output (standard and error) of the execution"; |
---|
| 33 | echo " - \"distexe.command\" is the command launch."; |
---|
[112] | 34 | # echo " * A command empty (no command on a line of file) is a synchronisation with all process" |
---|
| 35 | echo " * The environment variable DISTEXE_HOSTS list hosts to execute command"; |
---|
| 36 | echo " After each host, a number is to the number of process. It's optionnal, the default value is the number of processor."; |
---|
| 37 | echo " * All hosts must have network file systems (per example nfs or samba)"; |
---|
| 38 | echo " Work directory must be in this network file systems !"; |
---|
| 39 | echo ""; |
---|
[138] | 40 | echo "Example : "; |
---|
| 41 | echo " # Create command file"; |
---|
| 42 | echo ' echo "export PATH=${PATH}:${HOME}/my_appli/bin; my_appli 13" > command_file.txt'; |
---|
| 43 | echo ' echo "export PATH=${PATH}:${HOME}/my_appli/bin; my_appli 21" >> command_file.txt'; |
---|
| 44 | echo ' echo "export PATH=${PATH}:${HOME}/my_appli/bin; my_appli 7" >> command_file.txt'; |
---|
| 45 | echo ' echo "" >> command_file.txt'; |
---|
| 46 | echo " # Set hosts list variable"; |
---|
| 47 | echo ' export DISTEXE_HOSTS="localhost/1 nod/4 gdi/2"'; |
---|
| 48 | echo " # Execute distexe"; |
---|
| 49 | echo " ${0} command_file.txt;"; |
---|
| 50 | |
---|
[112] | 51 | exit; |
---|
| 52 | } |
---|
| 53 | |
---|
| 54 | #-----[ distexe_test_usage ]-------------------------------- |
---|
| 55 | function distexe_test_usage () |
---|
| 56 | { |
---|
| 57 | if test ${#} -ne 1 -a ${#} -ne 2; then |
---|
| 58 | distexe_usage; |
---|
| 59 | fi; |
---|
| 60 | |
---|
| 61 | if test -z "${MORPHEO_SCRIPT}"; then |
---|
| 62 | echo "Environment variable MORPHEO_SCRIPT is not set"; |
---|
| 63 | distexe_usage; |
---|
| 64 | fi; |
---|
| 65 | |
---|
| 66 | if test ! -f ${1}; then |
---|
| 67 | echo "File \"${1}\" is invalid"; |
---|
| 68 | distexe_usage; |
---|
| 69 | fi; |
---|
| 70 | |
---|
| 71 | if test ! -s ${1}; then |
---|
| 72 | echo "File \"${1}\" is empty."; |
---|
| 73 | distexe_usage; |
---|
| 74 | fi; |
---|
| 75 | |
---|
| 76 | if test ${#} -eq 2; then |
---|
| 77 | if test ! -d ${2}; then |
---|
| 78 | echo "Directory \"${2}\" is invalid."; |
---|
| 79 | distexe_usage; |
---|
| 80 | fi; |
---|
| 81 | fi; |
---|
[124] | 82 | } |
---|
[112] | 83 | |
---|
[124] | 84 | #-----[ header ]-------------------------------------------- |
---|
| 85 | function header () |
---|
| 86 | { |
---|
[138] | 87 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> =========================================="; |
---|
| 88 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> distexe ${VERSION}"; |
---|
| 89 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> =========================================="; |
---|
[112] | 90 | } |
---|
| 91 | |
---|
[138] | 92 | #-----[ distexe_wait_end ]---------------------------------- |
---|
| 93 | function distexe_wait_end() |
---|
| 94 | { |
---|
| 95 | while true; do |
---|
| 96 | |
---|
| 97 | nb_file=$(ls -1 ${PREFIX_FILE_HOST}* 2>/dev/null| wc -l); |
---|
| 98 | |
---|
| 99 | if test ${nb_file} -eq 0; then |
---|
| 100 | break; |
---|
| 101 | fi; |
---|
| 102 | |
---|
| 103 | # wait (to not have 100% cpu) |
---|
| 104 | sleep ${SLEEP}; |
---|
| 105 | done; |
---|
| 106 | |
---|
| 107 | rm -f ${FILE_CPT}; |
---|
| 108 | }; |
---|
| 109 | |
---|
| 110 | #-----[ distexe_no_trap ]----------------------------------- |
---|
| 111 | function distexe_no_trap() |
---|
| 112 | { |
---|
| 113 | TRAP=1; |
---|
| 114 | } |
---|
| 115 | |
---|
| 116 | #-----[ distexe_trap ]-------------------------------------- |
---|
| 117 | function distexe_trap() |
---|
| 118 | { |
---|
| 119 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> Trap signal detected"; |
---|
| 120 | # echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> !!!!!!!!!!!!!!!! Trap signal detected !!!!!!!!!!!!!!!!"; |
---|
| 121 | |
---|
| 122 | local -a files_host=($(ls ${PREFIX_FILE_HOST}* 2>/dev/null)); |
---|
| 123 | local -i nb_files_host=${#files_host[*]}; |
---|
| 124 | |
---|
| 125 | if test ${nb_files_host} -ne 0; then |
---|
| 126 | for file_host in ${files_host[*]}; do |
---|
| 127 | |
---|
| 128 | local host=$(basename ${file_host} |sed s/$(basename ${PREFIX_FILE_HOST})//); |
---|
| 129 | |
---|
| 130 | # ssh can not set yet pid in file_host |
---|
| 131 | local -i nb_word=0; |
---|
| 132 | while test ${nb_word} -eq 0; do |
---|
| 133 | nb_word=$(echo ${file_host} |wc -m); |
---|
| 134 | done; |
---|
| 135 | |
---|
| 136 | ${MORPHEO_SCRIPT}/lock.sh ${LOCK_HOST}; |
---|
| 137 | |
---|
| 138 | if test -f ${file_host}; then |
---|
| 139 | local cmd="kill -s SIGINT $(cat ${file_host})"; |
---|
| 140 | ssh ${host} ${cmd}; |
---|
| 141 | fi; |
---|
| 142 | |
---|
| 143 | ${MORPHEO_SCRIPT}/unlock.sh ${LOCK_HOST}; |
---|
| 144 | done; |
---|
| 145 | fi; |
---|
| 146 | |
---|
| 147 | # echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> >>>>>>>>>>>>>>>> Trap signal detected >>>>>>>>>>>>>>>>"; |
---|
| 148 | |
---|
| 149 | distexe_wait_end; |
---|
| 150 | |
---|
| 151 | # echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> ################ Trap signal detected ################"; |
---|
| 152 | |
---|
| 153 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> an error occure"; |
---|
| 154 | exit 1; |
---|
| 155 | } |
---|
| 156 | |
---|
[112] | 157 | #-----[ distexe ]------------------------------------------- |
---|
| 158 | function distexe () |
---|
| 159 | { |
---|
[138] | 160 | trap distexe_no_trap INT TERM; |
---|
| 161 | |
---|
[112] | 162 | distexe_test_usage ${*}; |
---|
| 163 | |
---|
| 164 | # construct file command with absolute path |
---|
| 165 | cd $(dirname ${1}); |
---|
| 166 | local FILE_CMD=${PWD}/$(basename ${1}); |
---|
| 167 | cd -; |
---|
| 168 | |
---|
[124] | 169 | # Absolute path of work directory |
---|
| 170 | local PATH_EXE; |
---|
[112] | 171 | if test ${#} -eq 2; then |
---|
[124] | 172 | cd ${2} &> /dev/null; |
---|
| 173 | PATH_EXE=${PWD}; |
---|
| 174 | cd - &> /dev/null; |
---|
[112] | 175 | else |
---|
[124] | 176 | PATH_EXE=${PWD}; |
---|
[112] | 177 | fi; |
---|
| 178 | |
---|
[138] | 179 | local FILE_CPT="${PATH_EXE}/distexe-control-"$(basename ${FILE_CMD}); |
---|
| 180 | local LOCK_HOST="${PATH_EXE}/distexe-host-lock"; |
---|
| 181 | local PREFIX_FILE_HOST="${PATH_EXE}/distexe-host-"; |
---|
| 182 | local PREFIX_FILE_PROCESS="${PATH_EXE}/distexe-process-"; |
---|
| 183 | local hosts="${DISTEXE_HOSTS}"; |
---|
| 184 | |
---|
[124] | 185 | header; |
---|
| 186 | |
---|
[138] | 187 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> file : ${FILE_CMD}"; |
---|
| 188 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> path : ${PATH_EXE}"; |
---|
| 189 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> sleep : ${SLEEP}"; |
---|
| 190 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> hosts : ${hosts}"; |
---|
[128] | 191 | |
---|
[112] | 192 | for line in ${hosts}; do |
---|
| 193 | local host=$(echo ${line} | cut -d/ -f1); |
---|
| 194 | local -i nb_process=$(echo ${line} | cut -d/ -f2); |
---|
| 195 | |
---|
| 196 | # lunch service |
---|
[138] | 197 | local FILE_HOST=${PREFIX_FILE_HOST}'${HOSTNAME}'; |
---|
[126] | 198 | |
---|
[138] | 199 | local valid=$(ssh ${host} "if test -f ${FILE_HOST}; then echo \"ko\"; else touch ${FILE_HOST}; echo \"ok\"; fi"); |
---|
[124] | 200 | |
---|
[138] | 201 | if test ${valid} = "ok"; then |
---|
| 202 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> station : ${host} (${nb_process} process(es))"; |
---|
[124] | 203 | |
---|
[138] | 204 | local cmd="export MORPHEO_SCRIPT=${MORPHEO_SCRIPT};. ${MORPHEO_SCRIPT}/distexe_execute_n.sh ${PATH_EXE} ${FILE_CMD} ${FILE_CPT} ${FILE_HOST} ${LOCK_HOST} ${PREFIX_FILE_PROCESS} ${SLEEP} ${nb_process};"; |
---|
[126] | 205 | |
---|
[138] | 206 | ssh ${host} ${cmd} & |
---|
| 207 | else |
---|
| 208 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> station \"${host}\" is already used"; |
---|
| 209 | fi; |
---|
| 210 | done; |
---|
[128] | 211 | |
---|
[138] | 212 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> all hosts working"; |
---|
[128] | 213 | |
---|
[138] | 214 | trap distexe_trap INT TERM; |
---|
[126] | 215 | |
---|
[138] | 216 | if test ${TRAP} -eq 1; then |
---|
| 217 | distexe_trap; |
---|
| 218 | fi; |
---|
[126] | 219 | |
---|
[138] | 220 | # Wait end ok all Task |
---|
| 221 | distexe_wait_end; |
---|
| 222 | |
---|
| 223 | echo " * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> all hosts is done"; |
---|
[112] | 224 | } |
---|
| 225 | |
---|
| 226 | #-----[ Corps ]--------------------------------------------- |
---|
| 227 | distexe ${*}; |
---|