#!/usr/bin/python # -*- coding: utf-8 -*- # Note: This code should stay compatible with # python 2 as it will execute on computation servers # FIXME? # When deactivating assert for perf eval, exec times are slower... from __future__ import print_function import os import sys import re import filecmp import shutil import random from common import * # Directories and files top_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")) scripts_path = os.path.join(top_path, "scripts") pyconf_file = os.path.join(scripts_path, "config.py") config_file = os.path.join(top_path, "include/config.h") images_dir = os.path.join(top_path, "../../images") binary_file = os.path.join(top_path, "appli.elf") base_output_dir = "output" # For output images and stat files (ref + exec) base_logs_dir = "logs" # Log of execution, i.e. what is printed on screen base_data_dir = "data" # For timing information extracted #images = ['boulons.pgm', 'cadastre.pgm', 'alea1.pgm', 'alea2.pgm', 'alea3.pgm'] images = ['cadastre.pgm'] images = map(lambda x:os.path.join(images_dir, x), images) # Parameters # - with eval_perf, num_internal_runs should be used, as this allows to mitigate the cost of the extra # run required to have the correct "ne" value (number of labels), and only the times from last application run are taken # - With check_results, num_app_runs should be used, so as to have a number of checks equals to the number of runs, # because only one check per application run is performed num_app_runs = 1 # Number of times the application is launched per configuration num_internal_runs = 10 # Number of times the image is processed inside the application check_results = False eval_perf = True use_valgrind = False use_rand_images = True threads = [1, 4, 16, 64] use_dsk = True # Using dsk will store generated random images, otherwise they are re-generated at each run to save disk space granularities = [1, 4, 16] img_size = 2048 # Configurations configs = [ #{'SLOW':'1', 'FAST':'0', 'FEATURES':'0', 'PARMERGE':'0', 'ARSP':'0'}, {'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'0', 'ARSP':'0'}, #{'SLOW':'1', 'FAST':'0', 'FEATURES':'1', 'PARMERGE':'0', 'ARSP':'0'}, {'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'0', 'ARSP':'0'}, {'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'0'}, {'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'1', 'ARSP':'0'}, {'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'1'}, {'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'1', 'ARSP':'1'}, ] # Other parameters which shouldn't be changed rand_seed = 7 check_pyconf_file(pyconf_file) # Loading config file exec(file(pyconf_file)) if use_dsk: try: dsk_dir if not os.path.exists(dsk_dir): print("mkdir %s", dsk_dir) os.mkdir(dsk_dir) except NameError: print("*** Warning: variable dsk_dir is not defined in file %s; using current directory for storing output files" % (short_path(pyconf_file))) use_dsk = False except OSError: print("*** Warning: Impossible to create directory %s; using current directory for storing output files" % (dsk_dir)) use_dsk = False if use_rand_images: try: rand_img_dir if not os.path.exists(rand_img_dir): print("mkdir %s", rand_img_dir) os.mkdir(rand_img_dir) except NameError: print("*** Error: variable rand_img_dir, containing the path the directory of the random images (either pre-existing or to be generated), is not defined in file %s" % (short_path(pyconf_file))) sys.exit(1) except OSError: print("*** Error: Impossible to create directory %s" % (rand_img_dir)) sys.exit(1) # Updating output directories if use_dsk: output_dir = os.path.join(dsk_dir, base_output_dir) logs_dir = os.path.join(dsk_dir, base_logs_dir) data_dir = os.path.join(dsk_dir, base_data_dir) else: output_dir = os.path.join(scripts_path, base_output_dir) logs_dir = os.path.join(scripts_path, base_logs_dir) data_dir = os.path.join(scripts_path, base_data_dir) if check_results and eval_perf: print("*** Warning: check_results and eval_perf modes are both set\n") if eval_perf and use_valgrind: print("*** Warning: using valgrind while eval_perf mode is set\n") if eval_perf and num_app_runs != 1: print("*** Warning: using eval_perf with num_app_runs != 1\n") if check_results and num_internal_runs != 1: print("*** Warning: using check_results with num_internal_runs != 1\n") def update_config_file(config): if os.path.isfile(config_file): print("# Updating file %s" % (config_file)) f = open(config_file, "r") lines = f.readlines() f.close() f = open(config_file, "w") for line in lines: line_with_key = False for key in config.keys(): if "#define %s" % (key) in line: f.write("#define %s %s\n" % (key, config[key])) line_with_key = True break if not line_with_key: if "#define MCA_VERBOSE_LEVEL" in line: if eval_perf: verb_level = 1 else: verb_level = 2 f.write("#define MCA_VERBOSE_LEVEL %d\n" % verb_level) else: f.write(line) f.close() else: print("# Creating file %s" % (config_file)) f = open(config_file, "w") f.write("\n") for key in config.keys(): f.write("#define %s %s\n" % (key, config[key])) f.write("\n") f.close() if not os.path.exists(output_dir): my_mkdir(output_dir) if not os.path.exists(logs_dir): my_mkdir(logs_dir) if not os.path.exists(data_dir): my_mkdir(data_dir) stat_array = {} perf_array = {} for config in configs: fconfig = frozenset(config.iteritems()) perf_array[fconfig] = {} update_config_file(config) features = config['FEATURES'] == '1' # Compile application my_chdir(top_path) cmd = ['make'] #if eval_perf: # cmd.extend(['IGNORE_ASSERT=true']) print_and_call(cmd) my_chdir(scripts_path) for granularity in granularities: perf_array[fconfig][granularity] = {} img_idx = 0 while not use_rand_images and img_idx != len(images) or use_rand_images and img_idx != 101: # Compute image and stat filenames if use_rand_images: random_img_file = get_random_img_file(img_idx, img_size, img_size, granularity, rand_seed) random_img_file = os.path.join(rand_img_dir, random_img_file) if not os.path.isfile(random_img_file): # We generate the random image if it does not exist print("# Generating random image %s with granularity = %d and density = %d" % (random_img_file, granularity, img_idx)) gen_random_image(random_img_file, img_size, img_size, granularity, float(img_idx) / 100, rand_seed) image = random_img_file else: image = images[img_idx] img_basename = os.path.splitext(os.path.basename(image))[0] perf_array[fconfig][granularity][img_basename] = {} ref_bmpfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + "_ref.bmp") ref_statfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + "_ref.txt") for nthreads in threads: perf_array[fconfig][granularity][img_basename][nthreads] = {} for run in range(num_app_runs): if not os.path.exists(ref_bmpfile): # Generating the reference file if it does not exist bmpfile = ref_bmpfile else: bmpfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + ".bmp") if os.path.exists(bmpfile): os.remove(bmpfile) if not os.path.exists(ref_statfile): statfile = ref_statfile else: statfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + ".txt") if os.path.exists(statfile): os.remove(statfile) cmd = [] if use_valgrind: cmd.append('valgrind') cmd.extend([short_path(binary_file), '-n', str(nthreads), '-i', short_path(image)]) if num_internal_runs > 1: cmd.extend(['-r', str(num_internal_runs)]) if check_results: cmd.extend(['-o', short_path(bmpfile), '-g']) if check_results and features: cmd.append('-d') config_keys = config.keys() logfile = get_filename(logs_dir, nthreads, config, features, img_basename) output = print_and_popen(cmd, logfile) outlines = output.splitlines() # if performance evaluation, get timing measurements # Only the last application run is considered if eval_perf: for line in outlines: tokens = line.split() if len(tokens) == 0: continue tag = tokens[0] pattern = re.compile('\[THREAD_STEP_([0-9]+)\]'); match = pattern.match(tag) if match: step = match.group(1) value = tokens[len(tokens) - 1] perf_array[fconfig][granularity][img_basename][nthreads][step] = int(value) # Checking against reference output image if check_results and bmpfile != ref_bmpfile: print("diff %s %s" % (short_path(bmpfile), short_path(ref_bmpfile))) if not filecmp.cmp(bmpfile, ref_bmpfile): print("*** Error: files %s and %s differ" % (short_path(bmpfile), short_path(ref_bmpfile))) sys.exit(1) # Checking for valgrind errors if use_valgrind: if not "== ERROR SUMMARY: 0 errors from 0 contexts" in output: print("*** Error: Valgrind error") sys.exit(1) if not "== All heap blocks were freed -- no leaks are possible" in output: print("*** Error: Valgrind detected a memory leak") sys.exit(1) # Extracting features for correctness verification if check_results and features: stat_array = {} in_stats = False index = 0 for line in outlines: if "[STATS]" in line: in_stats = True continue if "[/STATS]" in line: in_stats = False break if in_stats: tokens = line.split() assert(len(tokens) == 8) stat_array[index] = {} for j in range(len(tokens)): stat_array[index][j] = tokens[j] index += 1 # Dump stat array in stat file file = open(statfile, 'w') for i in range(len(stat_array)): for j in range(8): # 8 is the number of features per element file.write("%s " % stat_array[i][j]) file.write("\n"); file.close() # Comparison to reference if statfile != ref_statfile: print("diff %s %s" % (short_path(statfile), short_path(ref_statfile))) if not filecmp.cmp(statfile, ref_statfile): print("*** Error: feature files %s and %s differ" % (short_path(statfile), short_path(ref_statfile))) sys.exit(1) # End of the num_runs simus if eval_perf: datafile = get_filename(data_dir, nthreads, config, features, img_basename) file = open(datafile, 'w') for step in sorted(perf_array[fconfig][granularity][img_basename][nthreads].keys()): # Average time for each step file.write("[STEP_%s] %d\n" % (step, perf_array[fconfig][granularity][img_basename][nthreads][step])) img_idx += 1 #end image list (101 rand or explicit list) #end granularity #end config