#!/bin/bash

#-----------------------------------------------------------
# $Id: distexe.sh 138 2010-05-12 17:34:01Z rosiere $
#-----------------------------------------------------------

#-----[ global variable ]-----------------------------------
declare    VERSION="1.3"
declare -i SLEEP=2;
declare -i TRAP=0;

# Need : test, echo, cd, dirname, basename, ssh, ps aux, ls, wc, sed
# lock.sh, unlock.sh, date.sh, nb_cpu.sh, distexe_execute_n.sh, distexe_execute.sh

#-----[ distexe_usage ]-------------------------------------
function distexe_usage ()
{
    echo "Usage     : ${0} file [ dir ]";
    echo "Arguments : ";
    echo " * file       : list of command";
    echo " * dir        : work directory (default is current directory).";
#   echo " * nb_process : number of process (default (and maximum) is the number of processor)";
    echo "";
    echo "Note      : ";
    echo " * File content list of command. Each line is execute by an host.";
    echo "   A line can content many shell command.";
    echo "   Don't forgot the final end of line (else the last command is not executed)";
    echo " * For each command, a directory (in work directory) is created : Task_X (X is the number of line in command file).";
    echo "   The command is execute in this directory. (Warning, set your environment ! but we source .bashrc).";
    echo "   Three file is generate :";
    echo "    - \"distexe.info\"    content information as host, pid ...";
    echo "    - \"distexe.output\"  is the output (standard and error) of the execution";
    echo "    - \"distexe.command\" is the command launch.";
#   echo " * A command empty (no command on a line of file) is a synchronisation with all process"
    echo " * The environment variable DISTEXE_HOSTS list hosts to execute command";
    echo "   After each host, a number is to the number of process. It's optionnal, the default value is the number of processor.";
    echo " * All hosts must have network file systems (per example nfs or samba)";
    echo "   Work directory must be in this network file systems !";
    echo "";
    echo "Example   : ";
    echo "   # Create command file";
    echo '   echo "export PATH=${PATH}:${HOME}/my_appli/bin; my_appli 13"  > command_file.txt';
    echo '   echo "export PATH=${PATH}:${HOME}/my_appli/bin; my_appli 21" >> command_file.txt';
    echo '   echo "export PATH=${PATH}:${HOME}/my_appli/bin; my_appli  7" >> command_file.txt';
    echo '   echo ""                                                      >> command_file.txt';
    echo "   # Set hosts list variable";
    echo '   export DISTEXE_HOSTS="localhost/1 nod/4 gdi/2"';
    echo "   # Execute distexe";
    echo "   ${0} command_file.txt;";

    exit;
}

#-----[ distexe_test_usage ]--------------------------------
function distexe_test_usage ()
{
    if test ${#} -ne 1 -a ${#} -ne 2; then
	distexe_usage;
    fi;

    if test -z "${MORPHEO_SCRIPT}"; then
        echo "Environment variable MORPHEO_SCRIPT is not set";
        distexe_usage;
    fi;

    if test ! -f ${1}; then
	echo "File \"${1}\" is invalid";
	distexe_usage;
    fi;

    if test ! -s ${1}; then
	echo "File \"${1}\" is empty.";
	distexe_usage;
    fi;

    if test ${#} -eq 2; then
        if test ! -d ${2}; then
	    echo "Directory \"${2}\" is invalid.";
	    distexe_usage;
        fi;
    fi;
}

#-----[ header ]--------------------------------------------
function header ()
{
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> ==========================================";
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> distexe ${VERSION}";
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> ==========================================";
}

#-----[ distexe_wait_end ]----------------------------------
function distexe_wait_end()
{
    while true; do

        nb_file=$(ls -1  ${PREFIX_FILE_HOST}* 2>/dev/null| wc -l);

        if test ${nb_file} -eq 0; then
            break;
        fi;

        # wait (to not have 100% cpu)
        sleep ${SLEEP};
    done;

    rm -f ${FILE_CPT};
};

#-----[ distexe_no_trap ]-----------------------------------
function distexe_no_trap()
{
    TRAP=1;
}

#-----[ distexe_trap ]--------------------------------------
function distexe_trap()
{
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> Trap signal detected";
#    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> !!!!!!!!!!!!!!!! Trap signal detected !!!!!!!!!!!!!!!!";

    local -a files_host=($(ls ${PREFIX_FILE_HOST}* 2>/dev/null));
    local -i nb_files_host=${#files_host[*]};
    
    if test ${nb_files_host} -ne 0; then
        for file_host in ${files_host[*]}; do
            
            local host=$(basename ${file_host} |sed s/$(basename ${PREFIX_FILE_HOST})//);

                # ssh can not set yet pid in file_host
            local -i nb_word=0;
            while test ${nb_word} -eq 0; do
                nb_word=$(echo ${file_host} |wc -m);
            done;
            
            ${MORPHEO_SCRIPT}/lock.sh   ${LOCK_HOST};
            
            if test -f ${file_host}; then
                local cmd="kill -s SIGINT $(cat ${file_host})";
                ssh ${host} ${cmd};
            fi;
            
            ${MORPHEO_SCRIPT}/unlock.sh   ${LOCK_HOST};
        done;
    fi;

#    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> >>>>>>>>>>>>>>>> Trap signal detected >>>>>>>>>>>>>>>>";

    distexe_wait_end;

#    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> ################ Trap signal detected ################";

    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> an error occure";
    exit 1;
}

#-----[ distexe ]-------------------------------------------
function distexe ()
{
    trap distexe_no_trap INT TERM;

    distexe_test_usage ${*};

    # construct file command with absolute path
    cd $(dirname ${1});
    local FILE_CMD=${PWD}/$(basename ${1});
    cd -;

    # Absolute path of work directory
    local    PATH_EXE;
    if test ${#} -eq 2; then
        cd ${2} &> /dev/null;
        PATH_EXE=${PWD};
        cd -  &> /dev/null;
    else
        PATH_EXE=${PWD};
    fi;

    local FILE_CPT="${PATH_EXE}/distexe-control-"$(basename ${FILE_CMD});
    local LOCK_HOST="${PATH_EXE}/distexe-host-lock";
    local PREFIX_FILE_HOST="${PATH_EXE}/distexe-host-";
    local PREFIX_FILE_PROCESS="${PATH_EXE}/distexe-process-";
    local hosts="${DISTEXE_HOSTS}";

    header;

    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> file    : ${FILE_CMD}";
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> path    : ${PATH_EXE}";
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> sleep   : ${SLEEP}";
    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> hosts   : ${hosts}";

    for line in ${hosts}; do
        local    host=$(echo ${line} | cut -d/ -f1);
        local -i nb_process=$(echo ${line} | cut -d/ -f2);

        # lunch service
        local FILE_HOST=${PREFIX_FILE_HOST}'${HOSTNAME}';

        local    valid=$(ssh ${host} "if test -f ${FILE_HOST}; then echo \"ko\"; else touch ${FILE_HOST}; echo \"ok\"; fi");

        if test ${valid} = "ok"; then
            echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> station : ${host} (${nb_process} process(es))";

            local cmd="export MORPHEO_SCRIPT=${MORPHEO_SCRIPT};. ${MORPHEO_SCRIPT}/distexe_execute_n.sh ${PATH_EXE} ${FILE_CMD} ${FILE_CPT} ${FILE_HOST} ${LOCK_HOST} ${PREFIX_FILE_PROCESS} ${SLEEP} ${nb_process};";

            ssh ${host} ${cmd} &
        else
            echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> station \"${host}\" is already used";
        fi;
    done;

    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> all hosts working";

    trap distexe_trap INT TERM;

    if test ${TRAP} -eq 1; then
        distexe_trap;
    fi;

    # Wait end ok all Task
    distexe_wait_end;

    echo "  * {"$(${MORPHEO_SCRIPT}/date.sh)"} <${HOSTNAME}> all hosts is done";
}

#-----[ Corps ]---------------------------------------------
distexe ${*};