[822] | 1 | #!/usr/bin/python |
---|
| 2 | # -*- coding: utf-8 -*- |
---|
| 3 | |
---|
| 4 | # Note: This code should stay compatible with |
---|
| 5 | # python 2 as it will execute on computation servers |
---|
| 6 | |
---|
[823] | 7 | # TODO: |
---|
| 8 | # Can we do something about assert in perf eval? |
---|
[822] | 9 | |
---|
| 10 | from __future__ import print_function |
---|
| 11 | import os |
---|
| 12 | import sys |
---|
| 13 | import re |
---|
| 14 | import filecmp |
---|
| 15 | import shutil |
---|
| 16 | import random |
---|
| 17 | |
---|
| 18 | from common import * |
---|
| 19 | |
---|
| 20 | # Directories and files |
---|
| 21 | top_path = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")) |
---|
| 22 | scripts_path = os.path.join(top_path, "scripts") |
---|
| 23 | pyconf_file = os.path.join(scripts_path, "config.py") |
---|
| 24 | config_file = os.path.join(top_path, "include/config.h") |
---|
| 25 | images_dir = os.path.join(top_path, "../../images") |
---|
| 26 | binary_file = os.path.join(top_path, "appli.elf") |
---|
| 27 | |
---|
| 28 | base_output_dir = "output" # For output images and stat files (ref + exec) |
---|
| 29 | base_logs_dir = "logs" # Log of execution, i.e. what is printed on screen |
---|
| 30 | base_data_dir = "data" # For timing information extracted |
---|
| 31 | base_rand_img_dir = "random_images" # For storing random images if using dsk |
---|
| 32 | |
---|
| 33 | |
---|
| 34 | |
---|
| 35 | #images = ['boulons.pgm', 'cadastre.pgm', 'alea1.pgm', 'alea2.pgm', 'alea3.pgm'] |
---|
| 36 | images = ['cadastre.pgm'] |
---|
| 37 | |
---|
| 38 | images = map(lambda x:os.path.join(images_dir, x), images) |
---|
| 39 | |
---|
| 40 | # Parameters |
---|
[823] | 41 | # - with eval_perf, num_internal_runs should be used, as this allows to mitigate the cost of the extra |
---|
| 42 | # run required to have the correct "ne" value (number of labels), and only the times from last application run are taken |
---|
| 43 | # - With check_results, num_app_runs should be used, so as to have a number of checks equals to the number of runs, |
---|
| 44 | # because only one check per application run is performed |
---|
| 45 | num_app_runs = 1 # Number of times the application is launched per configuration |
---|
| 46 | num_internal_runs = 20 # Number of times the image is processed inside the application |
---|
[822] | 47 | check_results = False |
---|
| 48 | eval_perf = True |
---|
| 49 | use_valgrind = False |
---|
| 50 | use_rand_images = True |
---|
[823] | 51 | threads = [1, 2, 4, 8, 16, 32, 64] |
---|
| 52 | #threads = [1, 2] |
---|
[822] | 53 | use_dsk = True |
---|
| 54 | # Using dsk will store generated random images, otherwise they are re-generated at each run to save disk space |
---|
| 55 | |
---|
| 56 | # Configurations |
---|
| 57 | configs = [ |
---|
[823] | 58 | #{'SLOW':'1', 'FAST':'0', 'FEATURES':'0', 'PARMERGE':'0', 'ARSP':'0'}, |
---|
| 59 | #{'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'0', 'ARSP':'0'}, |
---|
| 60 | #{'SLOW':'1', 'FAST':'0', 'FEATURES':'1', 'PARMERGE':'0', 'ARSP':'0'}, |
---|
| 61 | #{'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'0', 'ARSP':'0'}, |
---|
| 62 | {'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'0'}, |
---|
[822] | 63 | #{'SLOW':'0', 'FAST':'1', 'FEATURES':'1', 'PARMERGE':'1', 'ARSP':'0'}, |
---|
| 64 | #{'SLOW':'0', 'FAST':'1', 'FEATURES':'0', 'PARMERGE':'1', 'ARSP':'1'}, |
---|
| 65 | ] |
---|
| 66 | |
---|
| 67 | |
---|
| 68 | # Other parameters which shouldn't be changed |
---|
| 69 | rand_seed = 7 |
---|
| 70 | granularity = 1 # constant for now |
---|
[823] | 71 | img_size = 2048 |
---|
[822] | 72 | |
---|
| 73 | check_pyconf_file(pyconf_file) |
---|
| 74 | |
---|
| 75 | # Loading config file |
---|
| 76 | exec(file(pyconf_file)) |
---|
| 77 | |
---|
| 78 | if use_dsk: |
---|
| 79 | try: |
---|
| 80 | dsk_dir |
---|
| 81 | if not os.path.exists(dsk_dir): |
---|
| 82 | print("mkdir %s", dsk_dir) |
---|
| 83 | os.mkdir(dsk_dir) |
---|
| 84 | except NameError: |
---|
| 85 | print("*** Warning: variable dsk_dir is not defined in %s file; using current directory for storing output files" % (short_path(pyconf_file))) |
---|
| 86 | use_dsk = False |
---|
| 87 | except OSError: |
---|
| 88 | print("*** Warning: Impossible to create directory %s; using current directory for storing output files" % (dsk_dir)) |
---|
| 89 | use_dsk = False |
---|
| 90 | |
---|
| 91 | |
---|
| 92 | # Updating output directories |
---|
| 93 | if use_dsk: |
---|
| 94 | output_dir = os.path.join(dsk_dir, base_output_dir) |
---|
| 95 | logs_dir = os.path.join(dsk_dir, base_logs_dir) |
---|
| 96 | data_dir = os.path.join(dsk_dir, base_data_dir) |
---|
| 97 | rand_img_dir = os.path.join(dsk_dir, base_rand_img_dir) |
---|
| 98 | else: |
---|
| 99 | output_dir = os.path.join(scripts_path, base_output_dir) |
---|
| 100 | logs_dir = os.path.join(scripts_path, base_logs_dir) |
---|
| 101 | data_dir = os.path.join(scripts_path, base_data_dir) |
---|
| 102 | |
---|
| 103 | |
---|
[823] | 104 | if check_results and eval_perf: |
---|
| 105 | print("*** Warning: check_results and eval_perf modes are both set\n") |
---|
| 106 | if eval_perf and use_valgrind: |
---|
| 107 | print("*** Warning: using valgrind while eval_perf mode is set\n") |
---|
| 108 | if eval_perf and num_app_runs != 1: |
---|
| 109 | print("*** Warning: using eval_perf with num_app_runs != 1\n") |
---|
| 110 | if check_results and num_internal_runs != 1: |
---|
| 111 | print("*** Warning: using check_results with num_internal_runs != 1\n") |
---|
[822] | 112 | |
---|
[823] | 113 | |
---|
| 114 | |
---|
[822] | 115 | def gen_random_image(filename, x, y, granularity, density, seed): |
---|
| 116 | random.seed(seed) |
---|
| 117 | img = [[0 for a in range(x)] for b in range(y)] |
---|
| 118 | for i in range(0, x, granularity): |
---|
| 119 | for j in range(0, y, granularity): |
---|
| 120 | r = random.random(); |
---|
| 121 | if r < density: |
---|
| 122 | px = 255 |
---|
| 123 | else: |
---|
| 124 | px = 0 |
---|
| 125 | |
---|
| 126 | for di in range(0, granularity): |
---|
| 127 | for dj in range(0, granularity): |
---|
| 128 | if i + di < x and j + dj < y: |
---|
| 129 | img[i + di][j + dj] = px; |
---|
| 130 | |
---|
| 131 | f = open(filename, 'wb') |
---|
| 132 | f.write("P5\n%d %d\n255\n" % (x, y)) |
---|
| 133 | for j in range(0, y): |
---|
| 134 | bimg = bytearray(img[j]) |
---|
| 135 | f.write(bimg) |
---|
| 136 | f.close() |
---|
| 137 | |
---|
| 138 | |
---|
| 139 | |
---|
| 140 | def update_config_file(config): |
---|
| 141 | if os.path.isfile(config_file): |
---|
| 142 | print("# Updating file %s" % (config_file)) |
---|
| 143 | f = open(config_file, "r") |
---|
| 144 | lines = f.readlines() |
---|
| 145 | f.close() |
---|
| 146 | |
---|
| 147 | f = open(config_file, "w") |
---|
| 148 | |
---|
| 149 | for line in lines: |
---|
| 150 | line_with_key = False |
---|
| 151 | for key in config.keys(): |
---|
| 152 | if "#define %s" % (key) in line: |
---|
| 153 | f.write("#define %s %s\n" % (key, config[key])) |
---|
| 154 | line_with_key = True |
---|
| 155 | break |
---|
| 156 | if not line_with_key: |
---|
[823] | 157 | if "#define MCA_VERBOSE_LEVEL" in line: |
---|
| 158 | if eval_perf: |
---|
| 159 | verb_level = 1 |
---|
| 160 | else: |
---|
| 161 | verb_level = 2 |
---|
| 162 | f.write("#define MCA_VERBOSE_LEVEL %d\n" % verb_level) |
---|
| 163 | else: |
---|
| 164 | f.write(line) |
---|
[822] | 165 | |
---|
| 166 | f.close() |
---|
| 167 | else: |
---|
| 168 | print("# Creating file %s" % (config_file)) |
---|
| 169 | f = open(config_file, "w") |
---|
| 170 | f.write("\n") |
---|
| 171 | for key in config.keys(): |
---|
| 172 | f.write("#define %s %s\n" % (key, config[key])) |
---|
| 173 | f.write("\n") |
---|
| 174 | f.close() |
---|
| 175 | |
---|
| 176 | |
---|
| 177 | |
---|
| 178 | |
---|
| 179 | if not os.path.exists(output_dir): |
---|
| 180 | my_mkdir(output_dir) |
---|
| 181 | |
---|
| 182 | if not os.path.exists(logs_dir): |
---|
| 183 | my_mkdir(logs_dir) |
---|
| 184 | |
---|
| 185 | if not os.path.exists(data_dir): |
---|
| 186 | my_mkdir(data_dir) |
---|
| 187 | |
---|
| 188 | if use_dsk and not os.path.exists(rand_img_dir): |
---|
| 189 | my_mkdir(rand_img_dir) |
---|
| 190 | |
---|
| 191 | |
---|
| 192 | |
---|
| 193 | |
---|
| 194 | stat_array = {} |
---|
| 195 | perf_array = {} |
---|
| 196 | |
---|
| 197 | for config in configs: |
---|
| 198 | img_idx = 0 |
---|
| 199 | fconfig = frozenset(config.iteritems()) |
---|
| 200 | perf_array[fconfig] = {} |
---|
| 201 | update_config_file(config) |
---|
| 202 | features = config['FEATURES'] == '1' |
---|
| 203 | |
---|
| 204 | # Compile application |
---|
| 205 | my_chdir(top_path) |
---|
| 206 | cmd = ['make'] |
---|
[823] | 207 | #if eval_perf: |
---|
| 208 | # cmd.extend(['IGNORE_ASSERT=true']) |
---|
[822] | 209 | print_and_call(cmd) |
---|
| 210 | my_chdir(scripts_path) |
---|
| 211 | |
---|
| 212 | while not use_rand_images and img_idx != len(images) or use_rand_images and img_idx != 101: |
---|
| 213 | # Compute image and stat filenames |
---|
| 214 | if use_rand_images: |
---|
| 215 | if use_dsk: |
---|
| 216 | random_img_file = get_random_img_file(img_idx, img_size, img_size, granularity, rand_seed) |
---|
| 217 | random_img_file = os.path.join(rand_img_dir, random_img_file) |
---|
| 218 | else: |
---|
| 219 | random_img_file = "random.pgm" |
---|
| 220 | if not (use_dsk and os.path.isfile(random_img_file)): |
---|
| 221 | # We re-generate the random image if not using the dsk or if it does not exist |
---|
| 222 | print("# Generating random image %s with density = %d" % (random_img_file, img_idx)) |
---|
| 223 | gen_random_image(random_img_file, img_size, img_size, 1, float(img_idx) / 100, rand_seed) |
---|
| 224 | |
---|
| 225 | image = random_img_file |
---|
| 226 | else: |
---|
| 227 | image = images[img_idx] |
---|
| 228 | img_basename = os.path.splitext(os.path.basename(image))[0] |
---|
| 229 | perf_array[fconfig][img_basename] = {} |
---|
| 230 | ref_bmpfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + "_ref.bmp") |
---|
| 231 | ref_statfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + "_ref.txt") |
---|
| 232 | |
---|
| 233 | for nthreads in threads: |
---|
| 234 | perf_array[fconfig][img_basename][nthreads] = {} |
---|
[823] | 235 | for run in range(num_app_runs): |
---|
[822] | 236 | if not os.path.exists(ref_bmpfile): |
---|
| 237 | bmpfile = ref_bmpfile |
---|
| 238 | else: |
---|
| 239 | bmpfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + ".bmp") |
---|
| 240 | if os.path.exists(bmpfile): |
---|
| 241 | os.remove(bmpfile) |
---|
| 242 | |
---|
| 243 | if not os.path.exists(ref_statfile): |
---|
| 244 | statfile = ref_statfile |
---|
| 245 | else: |
---|
| 246 | statfile = os.path.join(output_dir, os.path.splitext(os.path.basename(image))[0] + ".txt") |
---|
| 247 | |
---|
| 248 | cmd = [] |
---|
| 249 | if use_valgrind: |
---|
| 250 | cmd.append('valgrind') |
---|
| 251 | |
---|
| 252 | cmd.extend([short_path(binary_file), '-n', str(nthreads), '-i', short_path(image)]) |
---|
[823] | 253 | |
---|
| 254 | if num_internal_runs > 1: |
---|
| 255 | cmd.extend(['-r', str(num_internal_runs)]) |
---|
[822] | 256 | |
---|
| 257 | if check_results: |
---|
| 258 | cmd.extend(['-o', short_path(bmpfile), '-g']) |
---|
| 259 | |
---|
| 260 | if check_results and features: |
---|
| 261 | cmd.append('-d') |
---|
| 262 | |
---|
| 263 | config_keys = config.keys() |
---|
| 264 | logfile = get_filename(logs_dir, nthreads, config, features, img_basename) |
---|
| 265 | output = print_and_popen(cmd, logfile) |
---|
| 266 | outlines = output.splitlines() |
---|
| 267 | |
---|
| 268 | # if performance evaluation, get timing measurements |
---|
[823] | 269 | # Only the last application run is considered |
---|
[822] | 270 | if eval_perf: |
---|
| 271 | for line in outlines: |
---|
| 272 | tokens = line.split() |
---|
| 273 | if len(tokens) == 0: |
---|
| 274 | continue |
---|
| 275 | tag = tokens[0] |
---|
| 276 | pattern = re.compile('\[THREAD_STEP_([0-9]+)\]'); |
---|
| 277 | match = pattern.match(tag) |
---|
| 278 | if match: |
---|
| 279 | step = match.group(1) |
---|
| 280 | value = tokens[len(tokens) - 1] |
---|
[823] | 281 | perf_array[fconfig][img_basename][nthreads][step] = int(value) |
---|
[822] | 282 | |
---|
| 283 | |
---|
| 284 | # Checking against reference output image |
---|
| 285 | if check_results and bmpfile != ref_bmpfile: |
---|
| 286 | print("diff %s %s" % (short_path(bmpfile), short_path(ref_bmpfile))) |
---|
| 287 | if not filecmp.cmp(bmpfile, ref_bmpfile): |
---|
| 288 | print("*** Error: files %s and %s differ" % (short_path(bmpfile), short_path(ref_bmpfile))) |
---|
| 289 | sys.exit(1) |
---|
| 290 | |
---|
| 291 | # Checking for valgrind errors |
---|
| 292 | if use_valgrind: |
---|
| 293 | if not "== ERROR SUMMARY: 0 errors from 0 contexts" in output: |
---|
| 294 | print("*** Error: Valgrind error") |
---|
| 295 | sys.exit(1) |
---|
| 296 | if not "== All heap blocks were freed -- no leaks are possible" in output: |
---|
| 297 | print("*** Error: Valgrind detected a memory leak") |
---|
| 298 | sys.exit(1) |
---|
| 299 | |
---|
| 300 | # Extracting features for correctness verification |
---|
| 301 | if check_results and features: |
---|
| 302 | stat_array = {} |
---|
| 303 | in_stats = False |
---|
| 304 | index = 0 |
---|
| 305 | for line in outlines: |
---|
| 306 | if "[STATS]" in line: |
---|
| 307 | in_stats = True |
---|
| 308 | continue |
---|
| 309 | if "[/STATS]" in line: |
---|
| 310 | in_stats = False |
---|
| 311 | break |
---|
| 312 | if in_stats: |
---|
| 313 | tokens = line.split() |
---|
| 314 | assert(len(tokens) == 8) |
---|
| 315 | stat_array[index] = {} |
---|
| 316 | for j in range(len(tokens)): |
---|
| 317 | stat_array[index][j] = tokens[j] |
---|
| 318 | index += 1 |
---|
| 319 | |
---|
| 320 | # Dump stat array in stat file |
---|
| 321 | file = open(statfile, 'w') |
---|
| 322 | for i in range(len(stat_array)): |
---|
| 323 | for j in range(8): |
---|
| 324 | file.write("%s " % stat_array[i][j]) |
---|
| 325 | file.write("\n"); |
---|
| 326 | file.close() |
---|
| 327 | |
---|
| 328 | # Comparison to reference |
---|
| 329 | if statfile != ref_statfile: |
---|
| 330 | print("diff %s %s" % (short_path(statfile), short_path(ref_statfile))) |
---|
| 331 | if not filecmp.cmp(statfile, ref_statfile): |
---|
| 332 | print("*** Error: feature files %s and %s differ" % (short_path(statfile), short_path(ref_statfile))) |
---|
| 333 | sys.exit(1) |
---|
| 334 | |
---|
| 335 | # End of the num_runs simus |
---|
| 336 | if eval_perf: |
---|
| 337 | if use_rand_images: |
---|
| 338 | idx = img_idx |
---|
| 339 | else: |
---|
| 340 | idx = None |
---|
| 341 | datafile = get_filename(data_dir, nthreads, config, features, img_basename) |
---|
| 342 | file = open(datafile, 'w') |
---|
| 343 | for step in sorted(perf_array[fconfig][img_basename][nthreads].keys()): |
---|
| 344 | # Average time for each step |
---|
[823] | 345 | file.write("[STEP_%s] %d\n" % (step, perf_array[fconfig][img_basename][nthreads][step])) |
---|
[822] | 346 | |
---|
| 347 | img_idx += 1 |
---|
| 348 | |
---|
| 349 | |
---|
| 350 | |
---|
| 351 | |
---|
| 352 | |
---|
| 353 | |
---|
| 354 | |
---|
| 355 | |
---|
| 356 | |
---|
| 357 | |
---|
| 358 | |
---|
| 359 | |
---|
| 360 | |
---|
| 361 | |
---|
| 362 | |
---|
| 363 | |
---|
| 364 | |
---|