Index: soft/giet_vm/applications/transpose/main.c
===================================================================
--- soft/giet_vm/applications/transpose/main.c	(revision 574)
+++ soft/giet_vm/applications/transpose/main.c	(revision 589)
@@ -4,16 +4,18 @@
 // author : Alain Greiner
 ///////////////////////////////////////////////////////////////////////////////////////
-// This multi-threaded application makes a transpose for a NN*NN pixels 
-// sequence of images.
+// This multi-threaded application makes a transpose for a NN*NN pixels image.
 // It can run on a multi-processors, multi-clusters architecture, with one thread
 // per processor. 
 //
-// The image sequence is read from a file (one byte per pixel).
+// The image is read from a file (one byte per pixel), transposed and
+// saved in a second file. Then the transposed image is read from the second file,
+// transposed again and saved in a third file.
+//
 // The input and output buffers containing the image are distributed in all clusters.
 //
-// - The image size NN must fit the frame buffer size: 128 bytes
+// - The image size NN must fit the frame buffer size.
 // - The block size in block device must be 512 bytes.
-// - The number of clusters  must be a power of 2 no larger than 32
-// - The number of processors per cluster must be a power of 2 no larger than 4
+// - The number of clusters  must be a power of 2 no larger than 64.
+// - The number of processors per cluster must be a power of 2 no larger than 4.
 //
 // For each image the application makes a self test (checksum for each line).
@@ -25,11 +27,14 @@
 #include "malloc.h"
 
-#define BLOCK_SIZE          512                 // block size on disk
-#define CLUSTERS_MAX        32                  // max number of clusters
-#define PROCS_MAX           4                   // max number of processors per cluster
-#define NN                  256                 // image size : nlines = npixels
-#define NB_IMAGES           1                   // number of images to be handled
-#define FILE_PATHNAME       "misc/lena.raw"     // pathname on virtual disk
-#define INSTRUMENTATION_OK  0                   // display statistics on TTY when non zero
+#define BLOCK_SIZE            512                         // block size on disk
+#define X_MAX                 8                           // max number of clusters in row
+#define Y_MAX                 8                           // max number of clusters in column
+#define PROCS_MAX             4                           // max number of procs per cluster
+#define CLUSTER_MAX           (X_MAX * Y_MAX)             // max number of clusters
+#define NN                    256                         // image size : nlines = npixels
+#define INITIAL_FILE_PATH     "misc/lena.raw"             // pathname on virtual disk
+#define TRANSPOSED_FILE_PATH  "/home/lena_transposed.raw" // pathname on virtual disk
+#define RESTORED_FILE_PATH    "/home/lena_restored.raw"   // pathname on virtual disk
+#define INSTRUMENTATION_OK    1                           // display statistics on TTY 
 
 ///////////////////////////////////////////////////////
@@ -38,15 +43,17 @@
 
 // instrumentation counters for each processor in each cluster 
-unsigned int LOAD_START[CLUSTERS_MAX][PROCS_MAX];
-unsigned int LOAD_END  [CLUSTERS_MAX][PROCS_MAX];
-unsigned int TRSP_START[CLUSTERS_MAX][PROCS_MAX];
-unsigned int TRSP_END  [CLUSTERS_MAX][PROCS_MAX];
-unsigned int DISP_START[CLUSTERS_MAX][PROCS_MAX];
-unsigned int DISP_END  [CLUSTERS_MAX][PROCS_MAX];
+unsigned int LOAD_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int LOAD_END  [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int TRSP_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int TRSP_END  [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int DISP_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int DISP_END  [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int STOR_START[X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
+unsigned int STOR_END  [X_MAX][Y_MAX][PROCS_MAX] = {{{ 0 }}};
 
 // arrays of pointers on distributed buffers
 // one input buffer & one output buffer per cluster
-unsigned char*  buf_in [CLUSTERS_MAX];
-unsigned char*  buf_out[CLUSTERS_MAX];
+unsigned char*  buf_in [CLUSTER_MAX];
+unsigned char*  buf_out[CLUSTER_MAX];
 
 // checksum variables 
@@ -57,5 +64,6 @@
 giet_sqt_barrier_t barrier;
 
-volatile unsigned int init_ok = 0;
+volatile unsigned int global_init_ok = 0;
+volatile unsigned int local_init_ok[X_MAX][Y_MAX] = {{ 0 }};
 
 //////////////////////////////////////////
@@ -63,8 +71,6 @@
 //////////////////////////////////////////
 {
-
     unsigned int l;                  // line index for loops
     unsigned int p;                  // pixel index for loops
-    unsigned int c;                  // cluster index for loops
 
     // processor identifiers
@@ -82,16 +88,21 @@
     giet_procs_number( &x_size , &y_size , &nprocs );
 
-    unsigned int nclusters  = x_size * y_size;               // number of clusters
-    unsigned int ntasks     = x_size * y_size * nprocs;      // number of tasks
-    unsigned int npixels    = NN * NN;                       // pixels per image
-    unsigned int nblocks    = npixels / BLOCK_SIZE;          // blocks per image
-    unsigned int image      = 0;                             // image counter
-    int          file       = 0;                             // file descriptor
-    unsigned int cluster_id = (x * y_size) + y;              // "continuous" index   
-    unsigned int task_id    = (cluster_id * nprocs) + lpid;  // "continuous" task index
-
-    // Processor [0,0,0] makes initialisation
-    // It includes parameters checking, barrier initialization,
-    // distributed buffers allocation, and file open
+    unsigned int nclusters     = x_size * y_size;               // number of clusters
+    unsigned int ntasks        = x_size * y_size * nprocs;      // number of tasks
+    unsigned int npixels       = NN * NN;                       // pixels per image
+    unsigned int iteration     = 0;                             // iiteration iter
+    int          fd_initial    = 0;                             // initial file descriptor
+    int          fd_transposed = 0;                             // transposed file descriptor
+    int          fd_restored   = 0;                             // restored file descriptor
+    unsigned int cluster_id    = (x * y_size) + y;              // "continuous" index   
+    unsigned int task_id       = (cluster_id * nprocs) + lpid;  // "continuous" task index
+
+
+    ///////////////////////////////////////////////////////////////////////
+    // Processor [0,0,0] makes global initialisation
+    // It includes parameters checking, heap and barrier initialization.
+    // Others processors wait initialisation completion
+    ///////////////////////////////////////////////////////////////////////
+
     if ( (x==0) && (y==0) && (lpid==0) )
     {
@@ -101,7 +112,7 @@
         }
         if ((nclusters != 1) && (nclusters != 2) && (nclusters != 4) && 
-            (nclusters != 8) && (nclusters != 16) && (nclusters != 32) )
-        {
-            giet_exit("[TRANSPOSE ERROR] number of clusters must be 1,2,4,8,16,32");
+            (nclusters != 8) && (nclusters != 16) && (nclusters != 32) && (nclusters != 64) )
+        {
+            giet_exit("[TRANSPOSE ERROR] number of clusters must be 1,2,4,8,16,32,64");
         }
         if ( ntasks > NN )
@@ -110,87 +121,169 @@
         }
 
-        // Distributed buffers allocation
-        // The buffers containing one image are distributed in the user 
-        // heap (one buf_in and one buf_out per cluster).
-        // Each buffer contains (NN*NN / nclusters) bytes.
-        for ( c = 0 ; c < nclusters ; c++ )
-        {
-            unsigned int rx = c / y_size;
-            unsigned int ry = c % y_size;
-
-            heap_init( rx, ry );
-            buf_in[c]  = remote_malloc( npixels/nclusters, rx, ry );
-            buf_out[c] = remote_malloc( npixels/nclusters, rx, ry );
-
-            giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] completes buffer allocation"
-                            " for cluster[%d,%d] at cycle %d\n"
-                            " - buf_in  = %x\n"
-                            " - buf_out = %x\n",
-                            rx, ry, giet_proctime(), 
-                            (unsigned int)buf_in[c], 
-                            (unsigned int)buf_out[c] );
-        }
-
-        // Barrier initialisation
+        // distributed heap initialisation
+        unsigned int cx , cy;
+        for ( cx = 0 ; cx < x_size ; cx++ ) 
+        {
+            for ( cy = 0 ; cy < y_size ; cy++ ) 
+            {
+                heap_init( cx , cy );
+            }
+        }
+
+        // barrier initialisation
         sqt_barrier_init( &barrier, x_size , y_size , nprocs );
 
-        giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] completes barrier init at cycle %d\n",
+        giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] completes heap & barrier init at cycle %d\n",
                         giet_proctime() );
 
-        // open file containing images
-        file = giet_fat_open( FILE_PATHNAME , 0 );
-
-        if (file < 0)
-        { 
-            giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d]"
-                            " cannot open file %s",
-                            x , y , lpid , FILE_PATHNAME );
-            giet_exit(" open() failure");
-        }
-        else
-        {
-            giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] open file misc/images.raw\n");
-        }
-        init_ok = 1;
+        // diplay disk content
+        giet_fat_list( "/" );
+        giet_fat_list( "/misc" );
+        giet_fat_list( "/home" );
+        giet_fat_list( "/build" );
+        giet_fat_list( "/build/kernel" );
+        giet_fat_list( "/build/transpose" );
+
+        global_init_ok = 1;
     }
-    else   // others processors wait initialisation completion
+    else  
     {
-        while ( init_ok == 0 );
+        while ( global_init_ok == 0 );
     }
     
-    /////////////////////////
-    // Main loop (on images)
-    while (image < NB_IMAGES)
+    ///////////////////////////////////////////////////////////////////////
+    // In each cluster, only task running on processor[x,y,0] allocates 
+    // the local buffers containing the images in the distributed heap
+    // (one buf_in and one buf_out per cluster).
+    // Other processors in cluster wait completion. 
+    ///////////////////////////////////////////////////////////////////////
+
+    if ( lpid == 0 ) 
     {
-        // pseudo parallel load from disk to buf_in buffer : nblocks/nclusters blocks
-        // only task running on processor with (lpid == 0) does it
-
-        LOAD_START[cluster_id][lpid] = giet_proctime();
+        buf_in[cluster_id]  = remote_malloc( npixels/nclusters, x, y );
+        buf_out[cluster_id] = remote_malloc( npixels/nclusters, x, y );
+
+        if ( (x==0) && (y==0) )
+        giet_shr_printf("\n[TRANSPOSE] Proc [%d,%d,%d] completes buffer allocation"
+                        " for cluster[%d,%d] at cycle %d\n"
+                        " - buf_in  = %x\n"
+                        " - buf_out = %x\n",
+                        x, y, lpid, x, y, giet_proctime(), 
+                        (unsigned int)buf_in[cluster_id], (unsigned int)buf_out[cluster_id] );
+
+        ///////////////////////////////////////////////////////////////////////
+        // In each cluster, only task running on procesor[x,y,0] open the
+        // three private file descriptors for the three files
+        ///////////////////////////////////////////////////////////////////////
+
+        // open initial file
+        fd_initial = giet_fat_open( INITIAL_FILE_PATH , O_RDONLY );  // read_only
+        if ( fd_initial < 0 ) 
+        { 
+            giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot open file %s\n",
+                            x , y , lpid , INITIAL_FILE_PATH );
+            giet_exit(" open() failure");
+        }
+        else if ( (x==0) && (y==0) && (lpid==0) )
+        {
+            giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] open file %s / fd = %d\n",
+                            INITIAL_FILE_PATH , fd_initial );
+        }
+
+        // open transposed file
+        fd_transposed = giet_fat_open( TRANSPOSED_FILE_PATH , O_CREATE );   // create if required
+        if ( fd_transposed < 0 ) 
+        { 
+            giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot open file %s\n",
+                            x , y , lpid , TRANSPOSED_FILE_PATH );
+            giet_exit(" open() failure");
+        }
+        else if ( (x==0) && (y==0) && (lpid==0) )
+        {
+            giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] open file %s / fd = %d\n",
+                            TRANSPOSED_FILE_PATH , fd_transposed );
+        }
+
+        // open restored file
+        fd_restored = giet_fat_open( RESTORED_FILE_PATH , O_CREATE );   // create if required
+        if ( fd_restored < 0 ) 
+        { 
+            giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot open file %s\n",
+                            x , y , lpid , RESTORED_FILE_PATH );
+            giet_exit(" open() failure");
+        }
+        else if ( (x==0) && (y==0) && (lpid==0) )
+        {
+            giet_shr_printf("\n[TRANSPOSE] Proc [0,0,0] open file %s / fd = %d\n",
+                            RESTORED_FILE_PATH , fd_restored );
+        }
+
+        local_init_ok[x][y] = 1;
+    }
+    else
+    {
+        while( local_init_ok[x][y] == 0 );
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+    // Main loop / two iterations:
+    // - first makes  initial    => transposed
+    // - second makes transposed => restored 
+    // All processors execute this main loop.
+    ///////////////////////////////////////////////////////////////////////
+
+    unsigned int fd_in  = fd_initial;
+    unsigned int fd_out = fd_transposed;
+
+    while (iteration < 2)
+    {
+        ///////////////////////////////////////////////////////////////////////
+        // pseudo parallel load from disk to buf_in buffers: npixels/nclusters 
+        // only task running on processor(x,y,0) does it
+        ///////////////////////////////////////////////////////////////////////
+
+        LOAD_START[x][y][lpid] = giet_proctime();
 
         if (lpid == 0)
         {
-            giet_fat_read( file,
-                           buf_in[cluster_id],
-                           (nblocks / nclusters),
-                           ((image*nblocks) + ((nblocks*cluster_id)/nclusters)) );
+            unsigned int offset = ((npixels*cluster_id)/nclusters);
+            if ( giet_fat_lseek( fd_in,
+                                 offset,
+                                 SEEK_SET ) != offset )
+            {
+                giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot seek fd = %d\n",
+                                x , y , lpid , fd_in );
+                giet_exit(" seek() failure");
+            }
+
+            unsigned int pixels = npixels / nclusters;
+            if ( giet_fat_read( fd_in,
+                                buf_in[cluster_id],
+                                pixels ) != pixels )
+            {
+                giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot read fd = %d\n",
+                                x , y , lpid , fd_in );
+                giet_exit(" read() failure");
+            }
 
             if ( (x==0) && (y==0) )
             giet_shr_printf("\n[TRANSPOSE] Proc [%d,%d,%d] completes load"
-                            "  for image %d at cycle %d\n",
-                            x, y, lpid, image, giet_proctime() );
-        }
-
-        LOAD_END[cluster_id][lpid] = giet_proctime();
+                            "  for iteration %d at cycle %d\n",
+                            x, y, lpid, iteration, giet_proctime() );
+        }
+
+        LOAD_END[x][y][lpid] = giet_proctime();
 
         /////////////////////////////
         sqt_barrier_wait( &barrier );
 
+        ///////////////////////////////////////////////////////////////////////
         // parallel transpose from buf_in to buf_out
         // each task makes the transposition for nlt lines (nlt = NN/ntasks)
         // from line [task_id*nlt] to line [(task_id + 1)*nlt - 1]
         // (p,l) are the absolute pixel coordinates in the source image
-
-
-        TRSP_START[cluster_id][lpid] = giet_proctime();
+        ///////////////////////////////////////////////////////////////////////
+
+        TRSP_START[x][y][lpid] = giet_proctime();
 
         unsigned int nlt   = NN / ntasks;      // number of lines per task
@@ -233,85 +326,80 @@
             if ( (x==0) && (y==0) )
             giet_shr_printf("\n[TRANSPOSE] proc [%d,%d,0] completes transpose"
-                            " for image %d at cycle %d\n", 
-                            x, y, image, giet_proctime() );
-
-        }
-        TRSP_END[cluster_id][lpid] = giet_proctime();
+                            " for iteration %d at cycle %d\n", 
+                            x, y, iteration, giet_proctime() );
+
+        }
+        TRSP_END[x][y][lpid] = giet_proctime();
 
         /////////////////////////////
         sqt_barrier_wait( &barrier );
 
-
-        if ( USE_FBF )  // external frame buffer available
-        {
-            // parallel display from local buf_out to frame buffer
-            // all processors contribute to display using memcpy...
-
-            DISP_START[cluster_id][lpid] = giet_proctime();
-
-            unsigned int  npt   = npixels / ntasks;   // number of pixels per task
-
-            giet_fbf_sync_write( npt * task_id, 
-                                 &buf_out[cluster_id][lpid*npt], 
-                                 npt );
-
-            if ( (x==0) && (y==0) && (lpid==0) )
-            giet_shr_printf("\n[TRANSPOSE] Proc [%d,%d,%d] completes display"
-                            " for image %d at cycle %d\n",
-                            x, y, lpid, image, giet_proctime() );
-
-            DISP_END[cluster_id][lpid] = giet_proctime();
-
-            /////////////////////////////
-            sqt_barrier_wait( &barrier );
-        }
-        else         // checksum by processor(x,y,0) in each cluster
-        {
-            if ( lpid == 0 )
-            {
-                unsigned int success = 1;
-                unsigned int start   = cluster_id * nlc;
-                unsigned int stop    = start + nlc;
-
-                for ( l = start ; l < stop ; l++ )
-                {
-                    check_line_after[l] = 0;
-
-                    for ( p = 0 ; p < NN ; p++ )
-                    {
-                        // read one byte in remote buffer
-                        src_cluster = p / nlc;
-                        src_index   = (p % nlc)*NN + l;
-
-                        unsigned char byte = buf_out[src_cluster][src_index];
-
-                        check_line_after[l] = check_line_after[l] + byte;
-                    }
-
-                    if ( check_line_before[l] != check_line_after[l] ) success = 0;
-                }
-
-                if ( success ) 
-                {
-                    giet_shr_printf("\n[TRANSPOSE] proc [%d,%d,0] checksum OK"
-                                    " for image %d at cycle %d\n",
-                                    x, y, image, giet_proctime() );
-                }
-                else
-                {
-                    giet_shr_printf("\n[TRANSPOSE] proc [%d,%d,0] checksum KO"
-                                    " for image %d at cycle %d\n",
-                                    x, y, image, giet_proctime() );
-                }
-            } 
-        }
+        ///////////////////////////////////////////////////////////////////////
+        // parallel display from local buf_out to frame buffer
+        // all tasks contribute to display using memcpy...
+        ///////////////////////////////////////////////////////////////////////
+
+        DISP_START[x][y][lpid] = giet_proctime();
+
+        unsigned int  npt   = npixels / ntasks;   // number of pixels per task
+
+        giet_fbf_sync_write( npt * task_id, 
+                             &buf_out[cluster_id][lpid*npt], 
+                             npt );
+
+        if ( (x==0) && (y==0) && (lpid==0) )
+        giet_shr_printf("\n[TRANSPOSE] Proc [%d,%d,%d] completes display"
+                        " for iteration %d at cycle %d\n",
+                        x, y, lpid, iteration, giet_proctime() );
+
+        DISP_END[x][y][lpid] = giet_proctime();
 
         /////////////////////////////
         sqt_barrier_wait( &barrier );
 
+        ///////////////////////////////////////////////////////////////////////
+        // pseudo parallel store : buf_out buffers to disk : npixels/nclusters 
+        // only task running on processor(x,y,0) does it
+        ///////////////////////////////////////////////////////////////////////
+
+        STOR_START[x][y][lpid] = giet_proctime();
+
+        if ( lpid == 0 )
+        {
+            unsigned int offset = ((npixels*cluster_id)/nclusters);
+            if ( giet_fat_lseek( fd_out,
+                                 offset,
+                                 SEEK_SET ) != offset )
+            {
+                giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot seek fr = %d\n",
+                                x , y , lpid , fd_out );
+                giet_exit(" seek() failure");
+            }
+
+            unsigned int pixels = npixels / nclusters;
+            if ( giet_fat_write( fd_out,
+                                 buf_out[cluster_id],
+                                 pixels ) != pixels )
+            {
+                giet_shr_printf("\n[TRANSPOSE ERROR] Proc [%d,%d,%d] cannot write fd = %d\n",
+                                x , y , lpid , fd_out );
+                giet_exit(" write() failure");
+            }
+
+            if ( (x==0) && (y==0) )
+            giet_shr_printf("\n[TRANSPOSE] Proc [%d,%d,%d] completes store"
+                            "  for iteration %d at cycle %d\n",
+                            x, y, lpid, iteration, giet_proctime() );
+        }
+
+        STOR_END[x][y][lpid] = giet_proctime();
+
+        /////////////////////////////
+        sqt_barrier_wait( &barrier );
+
         // instrumentation done by processor [0,0,0] 
         if ( (x==0) && (y==0) && (lpid==0) && INSTRUMENTATION_OK )
         {
-            int cc, pp;
+            int cx , cy , pp ;
             unsigned int min_load_start = 0xFFFFFFFF;
             unsigned int max_load_start = 0;
@@ -326,23 +414,36 @@
             unsigned int min_disp_ended = 0xFFFFFFFF;
             unsigned int max_disp_ended = 0;
-
-            for (cc = 0; cc < nclusters; cc++)
-            {
-                for (pp = 0; pp < NB_PROCS_MAX; pp++)
-                {
-                    if (LOAD_START[cc][pp] < min_load_start)  min_load_start = LOAD_START[cc][pp];
-                    if (LOAD_START[cc][pp] > max_load_start)  max_load_start = LOAD_START[cc][pp];
-                    if (LOAD_END[cc][pp]   < min_load_ended)  min_load_ended = LOAD_END[cc][pp]; 
-                    if (LOAD_END[cc][pp]   > max_load_ended)  max_load_ended = LOAD_END[cc][pp];
-                    if (TRSP_START[cc][pp] < min_trsp_start)  min_trsp_start = TRSP_START[cc][pp];
-                    if (TRSP_START[cc][pp] > max_trsp_start)  max_trsp_start = TRSP_START[cc][pp];
-                    if (TRSP_END[cc][pp]   < min_trsp_ended)  min_trsp_ended = TRSP_END[cc][pp];
-                    if (TRSP_END[cc][pp]   > max_trsp_ended)  max_trsp_ended = TRSP_END[cc][pp];
-                    if (DISP_START[cc][pp] < min_disp_start)  min_disp_start = DISP_START[cc][pp];
-                    if (DISP_START[cc][pp] > max_disp_start)  max_disp_start = DISP_START[cc][pp];
-                    if (DISP_END[cc][pp]   < min_disp_ended)  min_disp_ended = DISP_END[cc][pp];
-                    if (DISP_END[cc][pp]   > max_disp_ended)  max_disp_ended = DISP_END[cc][pp];
-                }
-            }
+            unsigned int min_stor_start = 0xFFFFFFFF;
+            unsigned int max_stor_start = 0;
+            unsigned int min_stor_ended = 0xFFFFFFFF;
+            unsigned int max_stor_ended = 0;
+
+            for (cx = 0; cx < x_size; cx++)
+            {
+            for (cy = 0; cy < y_size; cy++)
+            {
+            for (pp = 0; pp < NB_PROCS_MAX; pp++)
+            {
+                if (LOAD_START[cx][cy][pp] < min_load_start)  min_load_start = LOAD_START[cx][cy][pp];
+                if (LOAD_START[cx][cy][pp] > max_load_start)  max_load_start = LOAD_START[cx][cy][pp];
+                if (LOAD_END[cx][cy][pp]   < min_load_ended)  min_load_ended = LOAD_END[cx][cy][pp]; 
+                if (LOAD_END[cx][cy][pp]   > max_load_ended)  max_load_ended = LOAD_END[cx][cy][pp];
+                if (TRSP_START[cx][cy][pp] < min_trsp_start)  min_trsp_start = TRSP_START[cx][cy][pp];
+                if (TRSP_START[cx][cy][pp] > max_trsp_start)  max_trsp_start = TRSP_START[cx][cy][pp];
+                if (TRSP_END[cx][cy][pp]   < min_trsp_ended)  min_trsp_ended = TRSP_END[cx][cy][pp];
+                if (TRSP_END[cx][cy][pp]   > max_trsp_ended)  max_trsp_ended = TRSP_END[cx][cy][pp];
+                if (DISP_START[cx][cy][pp] < min_disp_start)  min_disp_start = DISP_START[cx][cy][pp];
+                if (DISP_START[cx][cy][pp] > max_disp_start)  max_disp_start = DISP_START[cx][cy][pp];
+                if (DISP_END[cx][cy][pp]   < min_disp_ended)  min_disp_ended = DISP_END[cx][cy][pp];
+                if (DISP_END[cx][cy][pp]   > max_disp_ended)  max_disp_ended = DISP_END[cx][cy][pp];
+                if (STOR_START[cx][cy][pp] < min_stor_start)  min_stor_start = STOR_START[cx][cy][pp];
+                if (STOR_START[cx][cy][pp] > max_stor_start)  max_stor_start = STOR_START[cx][cy][pp];
+                if (STOR_END[cx][cy][pp]   < min_stor_ended)  min_stor_ended = STOR_END[cx][cy][pp];
+                if (STOR_END[cx][cy][pp]   > max_stor_ended)  max_stor_ended = STOR_END[cx][cy][pp];
+            }
+            }
+            }
+
+            giet_shr_printf("\n   ---------------- Instrumentation Results ---------------------\n");
 
             giet_shr_printf(" - LOAD_START : min = %d / max = %d / med = %d / delta = %d\n",
@@ -369,23 +470,62 @@
                             min_disp_ended, max_disp_ended, (min_disp_ended+max_disp_ended)/2, 
                             max_disp_ended-min_disp_ended); 
-        }
-
-        image++;
+
+            giet_shr_printf(" - STOR_START : min = %d / max = %d / med = %d / delta = %d\n",
+                            min_stor_start, max_stor_start, (min_stor_start+max_stor_start)/2, 
+                            max_stor_start-min_stor_start); 
+
+            giet_shr_printf(" - STOR_END   : min = %d / max = %d / med = %d / delta = %d\n",
+                            min_stor_ended, max_stor_ended, (min_stor_ended+max_stor_ended)/2, 
+                            max_stor_ended-min_stor_ended); 
+        }
 
         /////////////////////////////
         sqt_barrier_wait( &barrier );
 
-    } // end while image      
-
-    // Processor[0,0,0] releases the Distributed buffers
-    if ( (x==0) && (y==0) && (lpid==0) )
+        // update iteration variables
+        fd_in  = fd_transposed;
+        fd_out = fd_restored;
+        iteration++;
+
+    } // end while      
+
+    ///////////////////////////////////////////////////////////////////////
+    // In each cluster, only task running on Processor[x,y,0] releases
+    // the distributed buffers and close the file descriptors.
+    ///////////////////////////////////////////////////////////////////////
+
+    if ( lpid==0 )
     {
-        for ( c = 0 ; c < nclusters ; c++ )
-        {
-            free( buf_in[c] );
-            free( buf_in[c] );
-        }
+        free( buf_in[cluster_id] );
+        free( buf_out[cluster_id] );
+
+        giet_fat_close( fd_initial );
+        giet_fat_close( fd_transposed );
+        giet_fat_close( fd_restored );
     }
 
+    // display disk content
+    if ( (x==0) && (y == 0) && (lpid == 0) )
+    { 
+        giet_fat_list( "/" );
+        giet_fat_list( "/misc" );
+        giet_fat_list( "/home" );
+        giet_fat_list( "/build" );
+        giet_fat_list( "/build/kernel" );
+        giet_fat_list( "/build/transpose" );
+
+        giet_fat_remove( "/home/lena_transposed" , 0 );
+        giet_fat_remove( "/home/lena_restored" , 0 );
+
+        giet_fat_remove( "/home" , 1 );
+
+        giet_fat_list( "/" );
+        giet_fat_list( "/misc" );
+        giet_fat_list( "/home" );
+        giet_fat_list( "/build" );
+        giet_fat_list( "/build/kernel" );
+        giet_fat_list( "/build/transpose" );
+    }
+    
     giet_exit("Completed");
 
