source: soft/giet_vm/applications/rosenfeld/scripts/run_simus.py @ 824

#!/usr/bin/python
# -*- coding: utf-8 -*-

# Note: This code should stay compatible with
# python 2 as it will execute on computation servers

# TODO:
#       Can we do something about assert in perf eval?

from __future__ import print_function
import os
import sys
import re
import filecmp
import shutil
import random

from common import *

# Directories and files
top_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))
scripts_path  = os.path.join(top_path, "scripts")
pyconf_file   = os.path.join(scripts_path, "config.py")
config_file   = os.path.join(top_path, "include/config.h")
images_dir    = os.path.join(top_path, "../../images")
binary_file   = os.path.join(top_path, "appli.elf")

base_output_dir   = "output" # For output images and stat files (ref + exec)
base_logs_dir     = "logs"   # Log of execution, i.e. what is printed on screen
base_data_dir     = "data"   # For timing information extracted
base_rand_img_dir = "random_images" # For storing random images if using dsk



#images = ['boulons.pgm', 'cadastre.pgm', 'alea1.pgm', 'alea2.pgm', 'alea3.pgm']
images = ['cadastre.pgm']

images = map(lambda x:os.path.join(images_dir, x), images)

# Parameters
# - with eval_perf, num_internal_runs should be used, as this allows to mitigate the cost of the extra
#     run required to have the correct "ne" value (number of labels), and only the times from last application run are taken
# - With check_results, num_app_runs should be used, so as to have a number of checks equals to the number of runs,
#     because only one check per application run is performed
num_app_runs = 1        # Number of times the application is launched per configuration
num_internal_runs = 20  # Number of times the image is processed inside the application
check_results = False
eval_perf = True
use_valgrind = False
use_rand_images = True
threads = [1, 2, 4, 8, 16, 32, 64]
#threads = [1, 2]
use_dsk = True
# Using dsk will store generated random images, otherwise they are re-generated at each run to save disk space

# Configurations
configs = [
        #{'SLOW':'1', 'FAST':'0', 'FEATURES':'0', 'PARMERGE':'0', 'ARSP':'0'},
        #{'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'0', 'ARSP':'0'},
        #{'SLOW':'1', 'FAST':'0', 'FEATURES':'1', 'PARMERGE':'0', 'ARSP':'0'},
        #{'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'0', 'ARSP':'0'},
        {'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'0'},
        #{'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'1', 'ARSP':'0'},
        #{'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'1'},
]


# Other parameters which shouldn't be changed
rand_seed = 7
granularity = 1 # constant for now
img_size = 2048

check_pyconf_file(pyconf_file)

# Loading config file
exec(file(pyconf_file))

if use_dsk:
    try:
        dsk_dir
        if not os.path.exists(dsk_dir):
            print("mkdir %s", dsk_dir)
            os.mkdir(dsk_dir)
    except NameError:
        print("*** Warning: variable dsk_dir is not defined in %s file; using current directory for storing output files" % (short_path(pyconf_file)))
        use_dsk = False
    except OSError:
        print("*** Warning: Impossible to create directory %s; using current directory for storing output files" % (dsk_dir))
        use_dsk = False


# Updating output directories
if use_dsk:
    output_dir   = os.path.join(dsk_dir, base_output_dir)
    logs_dir     = os.path.join(dsk_dir, base_logs_dir)
    data_dir     = os.path.join(dsk_dir, base_data_dir)
    rand_img_dir = os.path.join(dsk_dir, base_rand_img_dir)
else:
    output_dir = os.path.join(scripts_path, base_output_dir)
    logs_dir   = os.path.join(scripts_path, base_logs_dir)
    data_dir   = os.path.join(scripts_path, base_data_dir)


if check_results and eval_perf:
    print("*** Warning: check_results and eval_perf modes are both set\n")
if eval_perf and use_valgrind:
    print("*** Warning: using valgrind while eval_perf mode is set\n")
if eval_perf and num_app_runs != 1:
    print("*** Warning: using eval_perf with num_app_runs != 1\n")
if check_results and num_internal_runs != 1:
    print("*** Warning: using check_results with num_internal_runs != 1\n")



def gen_random_image(filename, x, y, granularity, density, seed):
    random.seed(seed)
    img = [[0 for a in range(x)] for b in range(y)] 
    for i in range(0, x, granularity):
        for j in range(0, y, granularity):
            r = random.random();
            if r < density:
                px = 255
            else:
                px = 0

            for di in range(0, granularity):
                for dj in range(0, granularity):
                    if i + di < x and j + dj < y:
                        img[i + di][j + dj] = px;

    f = open(filename, 'wb')
    f.write("P5\n%d %d\n255\n" % (x, y))
    for j in range(0, y):
        bimg = bytearray(img[j])
        f.write(bimg)
    f.close()



def update_config_file(config):
    if os.path.isfile(config_file):
        print("# Updating file %s" % (config_file))
        f = open(config_file, "r")
        lines = f.readlines()
        f.close()

        f = open(config_file, "w")

        for line in lines:
            line_with_key = False
            for key in config.keys():
                if "#define %s" % (key) in line:
                    f.write("#define %s %s\n" % (key, config[key]))
                    line_with_key = True
                    break
            if not line_with_key:
                if "#define MCA_VERBOSE_LEVEL" in line:
                    if eval_perf:
                        verb_level = 1
                    else:
                        verb_level = 2
                    f.write("#define MCA_VERBOSE_LEVEL %d\n" % verb_level)  
                else:
                    f.write(line)

        f.close()
    else:
        print("# Creating file %s" % (config_file))
        f = open(config_file, "w")
        f.write("\n")
        for key in config.keys():
            f.write("#define %s %s\n" % (key, config[key]))
        f.write("\n")
        f.close()




if not os.path.exists(output_dir):
    my_mkdir(output_dir)

if not os.path.exists(logs_dir):
    my_mkdir(logs_dir)

if not os.path.exists(data_dir):
    my_mkdir(data_dir)

if use_dsk and not os.path.exists(rand_img_dir):
    my_mkdir(rand_img_dir)




stat_array = {}
perf_array = {}

for config in configs:
    img_idx = 0
    fconfig = frozenset(config.iteritems())
    perf_array[fconfig] = {}
    update_config_file(config)
    features = config['FEATURES'] == '1'
    
    # Compile application
    my_chdir(top_path)
    cmd = ['make']
    #if eval_perf:
    #    cmd.extend(['IGNORE_ASSERT=true'])
    print_and_call(cmd)
    my_chdir(scripts_path)

    while not use_rand_images and img_idx != len(images) or use_rand_images and img_idx != 101:
        # Compute image and stat filenames
        if use_rand_images:
            if use_dsk:
                random_img_file = get_random_img_file(img_idx, img_size, img_size, granularity, rand_seed)
                random_img_file = os.path.join(rand_img_dir, random_img_file)
            else:
                random_img_file = "random.pgm"
            if not (use_dsk and os.path.isfile(random_img_file)):
                # We re-generate the random image if not using the dsk or if it does not exist
                print("# Generating random image %s with density = %d" % (random_img_file, img_idx))
                gen_random_image(random_img_file, img_size, img_size, 1, float(img_idx) / 100, rand_seed)

            image = random_img_file
        else:
            image = images[img_idx]
        img_basename = os.path.splitext(os.path.basename(image))[0]
        perf_array[fconfig][img_basename] = {}
        ref_bmpfile  = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + "_ref.bmp")
        ref_statfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + "_ref.txt")

        for nthreads in threads:
            perf_array[fconfig][img_basename][nthreads] = {}
            for run in range(num_app_runs):
                if not os.path.exists(ref_bmpfile):
                    bmpfile = ref_bmpfile
                else:
                    bmpfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + ".bmp")
                    if os.path.exists(bmpfile):
                        os.remove(bmpfile)
    
                if not os.path.exists(ref_statfile):
                    statfile = ref_statfile
                else:
                    statfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + ".txt")
    
                cmd = []
                if use_valgrind:
                    cmd.append('valgrind')
    
                cmd.extend([short_path(binary_file), '-n', str(nthreads), '-i', short_path(image)])

                if num_internal_runs > 1:
                    cmd.extend(['-r', str(num_internal_runs)])
                
                if check_results:
                    cmd.extend(['-o', short_path(bmpfile), '-g'])
                
                if check_results and features:
                    cmd.append('-d')
    
                config_keys = config.keys()
                logfile = get_filename(logs_dir, nthreads, config, features, img_basename)
                output = print_and_popen(cmd, logfile)
                outlines = output.splitlines()

                # if performance evaluation, get timing measurements
                # Only the last application run is considered
                if eval_perf:
                    for line in outlines:
                        tokens = line.split()
                        if len(tokens) == 0:
                            continue
                        tag = tokens[0]
                        pattern = re.compile('\[THREAD_STEP_([0-9]+)\]');
                        match = pattern.match(tag)
                        if match:
                            step = match.group(1)
                            value = tokens[len(tokens) - 1]
                            perf_array[fconfig][img_basename][nthreads][step] = int(value)


                # Checking against reference output image    
                if check_results and bmpfile != ref_bmpfile:
                    print("diff %s %s" % (short_path(bmpfile), short_path(ref_bmpfile)))
                    if not filecmp.cmp(bmpfile, ref_bmpfile):
                        print("*** Error: files %s and %s differ" % (short_path(bmpfile), short_path(ref_bmpfile)))
                        sys.exit(1)
    
                # Checking for valgrind errors
                if use_valgrind:
                    if not "== ERROR SUMMARY: 0 errors from 0 contexts" in output:
                        print("*** Error: Valgrind error")
                        sys.exit(1)
                    if not "== All heap blocks were freed -- no leaks are possible" in output:
                        print("*** Error: Valgrind detected a memory leak")
                        sys.exit(1)
    
                # Extracting features for correctness verification
                if check_results and features:
                    stat_array = {}
                    in_stats = False
                    index = 0
                    for line in outlines:
                        if "[STATS]" in line:
                            in_stats = True
                            continue
                        if "[/STATS]" in line:
                            in_stats = False
                            break
                        if in_stats:
                            tokens = line.split()
                            assert(len(tokens) == 8)
                            stat_array[index] = {}
                            for j in range(len(tokens)):
                                stat_array[index][j] = tokens[j]
                            index += 1
    
                    # Dump stat array in stat file
                    file = open(statfile, 'w')
                    for i in range(len(stat_array)):
                        for j in range(8):
                            file.write("%s " % stat_array[i][j])
                        file.write("\n");
                    file.close()
    
                    # Comparison to reference
                    if statfile != ref_statfile:
                        print("diff %s %s" % (short_path(statfile), short_path(ref_statfile)))
                        if not filecmp.cmp(statfile, ref_statfile):
                            print("*** Error: feature files %s and %s differ" % (short_path(statfile), short_path(ref_statfile)))
                            sys.exit(1)

            # End of the num_runs simus
            if eval_perf:
                if use_rand_images:
                    idx = img_idx
                else:
                    idx = None
                datafile = get_filename(data_dir, nthreads, config, features, img_basename)
                file = open(datafile, 'w')
                for step in sorted(perf_array[fconfig][img_basename][nthreads].keys()):
                    # Average time for each step
                    file.write("[STEP_%s]   %d\n" % (step, perf_array[fconfig][img_basename][nthreads][step]))

        img_idx += 1