Changeset 347 for soft/giet_vm

Timestamp:

Jun 29, 2014, 12:27:21 PM (10 years ago)

Author:

alain

Message:

Introducing support for distributed page tables, kernel code and user code,
(if it is requested in the mapping).

File:

• soft/giet_vm/giet_boot/boot.c (modified) (58 diffs)

soft/giet_vm/giet_boot/boot.c

@@ -14 +14 @@
 // and where there is one physical memory bank per cluster.
 //
-// This code is executed in the boot phase by proc[0] and performs the following tasks:
-// - load into memory the giet_vm binary files, contained in a FAT32 file system,
+// This code, executed in the boot phase by proc[0,0,0] performs the following tasks:
+// - load into memory the binary files, from a FAT32 file system,
 // - build the various page tables (one page table per vspace) 
 // - initialize the shedulers (one scheduler per processor)
@@ -74 +74 @@
 #include <tty_driver.h>
 #include <xcu_driver.h>
-#include <ioc_driver.h>
+#include <bdv_driver.h>
 #include <dma_driver.h>
 #include <cma_driver.h>
@@ -140 +140 @@
 extern fat32_fs_t fat;
 
-// Page table addresses arrays
+// Page tables base addresses, sizes, and PT2 allocators:
+// For each vspace, it can exist one page table per cluster,
+// but only one virtual base address per vspace
+
 __attribute__((section (".bootdata"))) 
-volatile paddr_t      _ptabs_paddr[GIET_NB_VSPACE_MAX];
+unsigned int _ptabs_vaddr[GIET_NB_VSPACE_MAX];
 
 __attribute__((section (".bootdata"))) 
-volatile unsigned int _ptabs_vaddr[GIET_NB_VSPACE_MAX];
-
-// Next free PT2 index array
+paddr_t _ptabs_paddr[GIET_NB_VSPACE_MAX][X_SIZE][Y_SIZE];
+
 __attribute__((section (".bootdata"))) 
-volatile unsigned int _next_free_pt2[GIET_NB_VSPACE_MAX] =
-{ [0 ... GIET_NB_VSPACE_MAX - 1] = 0 };
-
-// Max PT2 index 
+unsigned int _ptabs_max_pt2[GIET_NB_VSPACE_MAX][X_SIZE][Y_SIZE];
+
 __attribute__((section (".bootdata"))) 
-volatile unsigned int _max_pt2[GIET_NB_VSPACE_MAX] =
-{ [0 ... GIET_NB_VSPACE_MAX - 1] = 0 };
+unsigned int _ptabs_next_pt2[GIET_NB_VSPACE_MAX][X_SIZE][Y_SIZE];
 
 // Scheduler pointers array (virtual addresses)
-// indexed by (x,y,lpid) : ((x << Y_WIDTH) + y)*NB_PROCS_MAX + lpid
+// indexed by (x,y,lpid) : (((x << Y_WIDTH) + y) * NB_PROCS_MAX) + lpid
 __attribute__((section (".bootdata"))) 
-static_scheduler_t* _schedulers[NB_PROCS_MAX<<(X_WIDTH+Y_WIDTH)];
+static_scheduler_t* _schedulers[1<<X_WIDTH][1<<Y_WIDTH][NB_PROCS_MAX];
 
 
@@ -291 +290 @@
 // boot_add_pte() 
 // This function registers a new PTE in the page table defined
-// by the vspace_id argument, and updates both PT1 and PT2.
-// A new PT2 is used when required.
+// by the vspace_id argument, and the (x,y) coordinates.
+// It updates both the PT1 and PT2, and a new PT2 is used if required.
 // As the set of PT2s is implemented as a fixed size array (no dynamic 
 // allocation), this function checks a possible overflow of the PT2 array.
 //////////////////////////////////////////////////////////////////////////////
 void boot_add_pte(unsigned int vspace_id,
+                  unsigned int x,
+                  unsigned int y,
                   unsigned int vpn, 
                   unsigned int flags, 
@@ -304 +305 @@
     unsigned int ix1;
     unsigned int ix2;
-    paddr_t      pt1_pbase;     // PT1 physical base address
-    paddr_t      pt2_pbase = 0; // PT2 physical base address
-    paddr_t      pte_paddr;     // PTE physucal address
+    paddr_t      pt2_pbase;     // PT2 physical base address
+    paddr_t      pte_paddr;     // PTE physical address
     unsigned int pt2_id;        // PT2 index
     unsigned int ptd;           // PTD : entry in PT1
-    unsigned int max_pt2;       // max number of PT2s for a given vspace
 
     ix1 = vpn >> 9;         // 11 bits
     ix2 = vpn & 0x1FF;      //  9 bits
 
-    // check that the _max_pt2[vspace_id] has been set
-    max_pt2 = _max_pt2[vspace_id];
+    // get page table physical base address and size
+    paddr_t      pt1_pbase = _ptabs_paddr[vspace_id][x][y];
+    unsigned int max_pt2   = _ptabs_max_pt2[vspace_id][x][y];
 
     if (max_pt2 == 0) 
@@ -325 +325 @@
     }
 
-
-    // get page table physical base address
-    pt1_pbase = _ptabs_paddr[vspace_id];
-
     // get ptd in PT1
     ptd = _physical_read(pt1_pbase + 4 * ix1);
 
-    if ((ptd & PTE_V) == 0)    // invalid PTD: compute PT2 base address, 
+    if ((ptd & PTE_V) == 0)    // undefined PTD: compute PT2 base address, 
                                // and set a new PTD in PT1 
     {
-        pt2_id = _next_free_pt2[vspace_id];
+        pt2_id = _ptabs_next_pt2[vspace_id][x][y];
         if (pt2_id == max_pt2) 
         {
             _puts("\n[BOOT ERROR] in boot_add_pte() function\n");
-            _puts("the length of the ptab vobj is too small\n");
-
+            _puts("the length of the PTAB vobj is too small\n");
             _puts(" max_pt2 = ");
             _putd( max_pt2 );
@@ -347 +342 @@
             _putd( pt2_id );
             _puts("\n");
-            
             _exit();
         }
-        else 
-        {
-            pt2_pbase = pt1_pbase + PT1_SIZE + PT2_SIZE * pt2_id;
-            ptd = PTE_V | PTE_T | (unsigned int) (pt2_pbase >> 12);
-            _physical_write( pt1_pbase + 4 * ix1, ptd);
-            _next_free_pt2[vspace_id] = pt2_id + 1;
-        }
+
+        pt2_pbase = pt1_pbase + PT1_SIZE + PT2_SIZE * pt2_id;
+        ptd = PTE_V | PTE_T | (unsigned int) (pt2_pbase >> 12);
+        _physical_write( pt1_pbase + 4 * ix1, ptd);
+        _ptabs_next_pt2[vspace_id][x][y] = pt2_id + 1;
     }
     else                       // valid PTD: compute PT2 base address
@@ -392 +384 @@
 
 
-/////////////////////////////////////////////////////////////////////
-// This function build the page table for a given vspace. 
+////////////////////////////////////////////////////////////////////////
+// This function build the page table(s) for a given vspace. 
+// It build as many pages tables as the number of vobjs having
+// the PTAB type in the vspace, because page tables can be replicated.
 // The physical base addresses for all vsegs (global and private)
 // must have been previously computed and stored in the mapping.
-// It initializes the MWMR channels.
-/////////////////////////////////////////////////////////////////////
+//
+// General rule regarding local / shared vsegs:
+// - shared vsegs are mapped in all page tables
+// - local vsegs are mapped only in the "local" page table
+////////////////////////////////////////////////////////////////////////
 void boot_vspace_pt_build(unsigned int vspace_id) 
 {
-    unsigned int vseg_id;
+    unsigned int ptab_id;       // global index for a vseg containing a PTAB
+    unsigned int priv_id;       // global index for a private vseg in a vspace
+    unsigned int glob_id;       // global index for a global vseg
     unsigned int npages;
     unsigned int ppn;
@@ -408 +407 @@
     unsigned int verbose = 0;   // can be used to activate trace in add_pte()
 
-    mapping_header_t * header = (mapping_header_t *)SEG_BOOT_MAPPING_BASE;
-    mapping_vspace_t * vspace = _get_vspace_base(header);
-    mapping_vseg_t   * vseg   = _get_vseg_base(header);
-
-    // private segments
-    for (vseg_id = vspace[vspace_id].vseg_offset;
-         vseg_id < (vspace[vspace_id].vseg_offset + vspace[vspace_id].vsegs);
-         vseg_id++) 
-    {
-        vpn = vseg[vseg_id].vbase >> 12;
-        ppn = (unsigned int) (vseg[vseg_id].pbase >> 12);
-
-        npages = vseg[vseg_id].length >> 12;
-        if ((vseg[vseg_id].length & 0xFFF) != 0) npages++; 
-
-        flags = PTE_V;
-        if (vseg[vseg_id].mode & C_MODE_MASK) flags |= PTE_C;
-        if (vseg[vseg_id].mode & X_MODE_MASK) flags |= PTE_X;
-        if (vseg[vseg_id].mode & W_MODE_MASK) flags |= PTE_W;
-        if (vseg[vseg_id].mode & U_MODE_MASK) flags |= PTE_U;
+    mapping_header_t  * header  = (mapping_header_t *)SEG_BOOT_MAPPING_BASE;
+    mapping_vspace_t  * vspace  = _get_vspace_base(header);
+    mapping_vseg_t    * vseg    = _get_vseg_base(header);
+    mapping_vobj_t    * vobj    = _get_vobj_base(header);
+    mapping_pseg_t    * pseg    = _get_pseg_base(header);
+    mapping_cluster_t * cluster = _get_cluster_base(header);
+
+    // external loop on private vsegs to find all PTAB vobjs in vspace
+    for (ptab_id = vspace[vspace_id].vseg_offset;
+         ptab_id < (vspace[vspace_id].vseg_offset + vspace[vspace_id].vsegs);
+         ptab_id++) 
+    {
+        // get global index of first vobj in vseg
+        unsigned int vobj_id = vseg[ptab_id].vobj_offset;
+
+        if ( vobj[vobj_id].type == VOBJ_TYPE_PTAB )
+        {
+            // get cluster coordinates for the PTAB
+            unsigned int ptab_pseg_id    = vseg[ptab_id].psegid;
+            unsigned int ptab_cluster_id = pseg[ptab_pseg_id].clusterid;
+            unsigned int x_ptab          = cluster[ptab_cluster_id].x;
+            unsigned int y_ptab          = cluster[ptab_cluster_id].y;
+
+            // internal loop on private vsegs to build 
+            // the (vspace_id, x_ptab, y_ptab) page table
+            for (priv_id = vspace[vspace_id].vseg_offset;
+                 priv_id < (vspace[vspace_id].vseg_offset + vspace[vspace_id].vsegs);
+                 priv_id++) 
+            {
+                // get cluster coordinates for private vseg 
+                unsigned int priv_pseg_id    = vseg[priv_id].psegid;
+                unsigned int priv_cluster_id = pseg[priv_pseg_id].clusterid;
+                unsigned int x_priv          = cluster[priv_cluster_id].x;
+                unsigned int y_priv          = cluster[priv_cluster_id].y;
+
+                // only non local or matching private vsegs must be mapped
+                if ( (vseg[priv_id].local == 0 ) ||
+                     ((x_ptab == x_priv) && (y_ptab == y_priv)) )
+                {
+                    vpn = vseg[priv_id].vbase >> 12;
+                    ppn = (unsigned int) (vseg[priv_id].pbase >> 12);
+                    npages = vseg[priv_id].length >> 12;
+                    if ((vseg[priv_id].length & 0xFFF) != 0) npages++; 
+
+                    flags = PTE_V;
+                    if (vseg[priv_id].mode & C_MODE_MASK) flags |= PTE_C;
+                    if (vseg[priv_id].mode & X_MODE_MASK) flags |= PTE_X;
+                    if (vseg[priv_id].mode & W_MODE_MASK) flags |= PTE_W;
+                    if (vseg[priv_id].mode & U_MODE_MASK) flags |= PTE_U;
         
-        // These three flags (Local, Remote and Dirty) are set to 1 to reduce
-        // latency of TLB miss (L/R) and write (D): Avoid hardware update
-        // mechanism for these flags. This optimization can be performed
-        // because GIET_VM does nothing with these flags.
-
-        flags |= PTE_L;
-        flags |= PTE_R;
-        flags |= PTE_D;
+                    // The three flags (Local, Remote and Dirty) are set to 1 to reduce
+                    // latency of TLB miss (L/R) and write (D): Avoid hardware update
+                    // mechanism for these flags because GIET_VM does use these flags.
+
+                    flags |= PTE_L;
+                    flags |= PTE_R;
+                    flags |= PTE_D;
 
 #if BOOT_DEBUG_PT
-        _puts(vseg[vseg_id].name);
-        _puts(" : flags = ");
-        _putx(flags);
-        _puts(" / npages = ");
-        _putd(npages);
-        _puts(" / pbase = ");
-        _putl(vseg[vseg_id].pbase);
-        _puts("\n");
-#endif
-        // loop on 4K pages
-        for (page_id = 0; page_id < npages; page_id++) 
-        {
-            boot_add_pte(vspace_id, vpn, flags, ppn, verbose);
-            vpn++;
-            ppn++;
-        }
-    }
-
-    // global segments
-    for (vseg_id = 0; vseg_id < header->globals; vseg_id++) 
-    {
-        vpn = vseg[vseg_id].vbase >> 12;
-        ppn = (unsigned int)(vseg[vseg_id].pbase >> 12);
-        npages = vseg[vseg_id].length >> 12;
-        if ((vseg[vseg_id].length & 0xFFF) != 0) npages++;
-
-        flags = PTE_V;
-        if (vseg[vseg_id].mode & C_MODE_MASK) flags |= PTE_C;
-        if (vseg[vseg_id].mode & X_MODE_MASK) flags |= PTE_X;
-        if (vseg[vseg_id].mode & W_MODE_MASK) flags |= PTE_W;
-        if (vseg[vseg_id].mode & U_MODE_MASK) flags |= PTE_U;
-
-        // Flags set for optimization (as explained above)
-
-        flags |= PTE_L;
-        flags |= PTE_R;
-        flags |= PTE_D;
+_puts(vseg[priv_id].name);
+_puts(" : flags = ");
+_putx(flags);
+_puts(" / npages = ");
+_putd(npages);
+_puts(" / pbase = ");
+_putl(vseg[priv_id].pbase);
+_puts("\n");
+#endif
+                    // loop on 4K pages
+                    for (page_id = 0; page_id < npages; page_id++) 
+                    {
+                        boot_add_pte(vspace_id, x_ptab, y_ptab, vpn, flags, ppn, verbose);
+                        vpn++;
+                        ppn++;
+                    }
+                } 
+            }  // end internal loop on private vsegs
+
+            // internal loop on global vsegs to build the (x_ptab,y_ptab) page table
+            for (glob_id = 0; glob_id < header->globals; glob_id++) 
+            {
+                // get cluster coordinates for global vseg 
+                unsigned int glob_pseg_id    = vseg[glob_id].psegid;
+                unsigned int glob_cluster_id = pseg[glob_pseg_id].clusterid;
+                unsigned int x_glob          = cluster[glob_cluster_id].x;
+                unsigned int y_glob          = cluster[glob_cluster_id].y;
+
+                // only non local or matching global vsegs must be mapped
+                if ( (vseg[glob_id].local == 0 ) ||
+                     ((x_ptab == x_glob) && (y_ptab == y_glob)) )
+                {
+                    vpn = vseg[glob_id].vbase >> 12;
+                    ppn = (unsigned int)(vseg[glob_id].pbase >> 12);
+                    npages = vseg[glob_id].length >> 12;
+                    if ((vseg[glob_id].length & 0xFFF) != 0) npages++;
+
+                    flags = PTE_V;
+                    if (vseg[glob_id].mode & C_MODE_MASK) flags |= PTE_C;
+                    if (vseg[glob_id].mode & X_MODE_MASK) flags |= PTE_X;
+                    if (vseg[glob_id].mode & W_MODE_MASK) flags |= PTE_W;
+                    if (vseg[glob_id].mode & U_MODE_MASK) flags |= PTE_U;
+
+                    // Flags set for optimization (as explained above)
+
+                    flags |= PTE_L;
+                    flags |= PTE_R;
+                    flags |= PTE_D;
 
 #if BOOT_DEBUG_PT
-        _puts(vseg[vseg_id].name);
-        _puts(" : flags = ");
-        _putx(flags);
-        _puts(" / npages = ");
-        _putd(npages);
-        _puts(" / pbase = ");
-        _putl(vseg[vseg_id].pbase);
-        _puts("\n");
-#endif
-        // loop on 4K pages
-        for (page_id = 0; page_id < npages; page_id++) 
-        {
-            boot_add_pte(vspace_id, vpn, flags, ppn, verbose);
-            vpn++;
-            ppn++;
-        }
-    }
+_puts(vseg[glob_id].name);
+_puts(" : flags = ");
+_putx(flags);
+_puts(" / npages = ");
+_putd(npages);
+_puts(" / pbase = ");
+_putl(vseg[glob_id].pbase);
+_puts("\n");
+#endif
+                    // loop on 4K pages
+                    for (page_id = 0; page_id < npages; page_id++) 
+                    {
+                        boot_add_pte(vspace_id, x_ptab, y_ptab, vpn, flags, ppn, verbose);
+                        vpn++;
+                        ppn++;
+                    }
+                }
+            }   // end internal loop on global vsegs
+
+            _puts("\n[BOOT] Page Table for vspace ");
+            _puts( vspace[vspace_id].name );
+            _puts(" in cluster[");
+            _putd( x_ptab );
+            _puts(",");
+            _putd( y_ptab );
+            _puts("] completed at cycle ");
+            _putd( _get_proctime() );
+            _puts("\n");
+
+#if BOOT_DEBUG_PT
+_puts("vaddr = ");
+_putx( _ptabs_vaddr[vspace_id] );
+_puts(" / paddr = ");
+_putl( _ptabs_paddr[vspace_id][x_ptab][y_ptab] );
+_puts(" / PT2 number = ");
+_putd( _ptabs_next_pt2[vspace_id][x_ptab][y_ptab] );
+_puts("\n");
+#endif
+
+        }  // end if PTAB
+    }  // end first loop on private vsegs
 }   // end boot_vspace_pt_build()
 
@@ -517 +579 @@
 // Set pbase for a vseg when identity mapping is required.
 // The length of the vseg must be known.
-// The ordered linked list of vsegs mapped on pseg must be updated,
-// and overlap with previously mapped vsegs must be checked.
+// The ordered linked list of vsegs mapped on pseg is updated,
+// and overlap with previously mapped vsegs is checked.
 ///////////////////////////////////////////////////////////////////////////
 void boot_vseg_set_paddr_ident(mapping_vseg_t * vseg) 
@@ -713 +775 @@
 // It updates the pbase and the length fields of the vseg.
 // It updates the pbase and vbase fields of all vobjs in the vseg.
-// It updates the _ptabs_paddr[] and _ptabs_vaddr[] arrays. 
+// It updates the _ptabs_paddr[] and _ptabs_vaddr[], _ptabs_max_pt2[],
+// and _ptabs_next_pt2[] arrays. 
 // It is a global vseg if vspace_id = (-1). 
 ///////////////////////////////////////////////////////////////////////////
@@ -727 +790 @@
     mapping_vobj_t   * vobj   = _get_vobj_base(header);
 
-    // loop on the vobjs contained in vseg to compute 
+    // first loop on the vobjs contained in vseg to compute 
     // the vseg length, required for mapping.
     cur_length = 0;
@@ -771 +834 @@
         vobj[vobj_id].paddr = cur_paddr;
         
-        // initialize _ptabs_vaddr[] & boot_ptabs-paddr[] if PTAB
+        // initialize _ptabs_vaddr[] , _ptabs-paddr[] , _ptabs_max_pt2[] if PTAB
         if (vobj[vobj_id].type == VOBJ_TYPE_PTAB) 
         {
@@ -788 +851 @@
                 _exit();
             }
-            // register both physical and virtual page table address
-            _ptabs_vaddr[vspace_id] = vobj[vobj_id].vaddr;
-            _ptabs_paddr[vspace_id] = vobj[vobj_id].paddr;
+            // get cluster coordinates for PTAB
+            unsigned int cluster_xy = (unsigned int)(cur_paddr>>32);
+            unsigned int x          = cluster_xy >> Y_WIDTH;
+            unsigned int y          = cluster_xy & ((1<<Y_WIDTH)-1);
+
+            // register physical and virtual page table addresses, size, and next PT2
+            _ptabs_vaddr[vspace_id]          = vobj[vobj_id].vaddr;
+            _ptabs_paddr[vspace_id][x][y]    = vobj[vobj_id].paddr;
+            _ptabs_max_pt2[vspace_id][x][y]  = (vobj[vobj_id].length - PT1_SIZE) / PT2_SIZE;
+            _ptabs_next_pt2[vspace_id][x][y] = 0;
             
             // reset all valid bits in PT1
@@ -797 +867 @@
                 _physical_write(cur_paddr + offset, 0);
             }
-
-            // computing the number of second level pages
-            _max_pt2[vspace_id] = (vobj[vobj_id].length - PT1_SIZE) / PT2_SIZE;
         }
 
@@ -809 +876 @@
 }    // end boot_vseg_map()
 
-/////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
 // This function builds the page tables for all virtual spaces 
 // defined in the mapping_info data structure, in three steps:
@@ -816 +883 @@
 // - step 2 : It computes the physical base address for all private 
 //            vsegs and all vobjs in each virtual space.
-// - step 3 : It actually fill the page table for each vspace.
+// - step 3 : It actually fill the page table(s) for each vspace.
 //
-// Note: It must exist at least one vspace in the mapping_info...
-/////////////////////////////////////////////////////////////////////
+// It must exist at least one vspace in the mapping.
+// For each vspace, it can exist one page table per cluster.
+///////////////////////////////////////////////////////////////////////////
 void boot_pt_init() 
 {
@@ -841 +909 @@
 #endif
 
+    //////////////////////////////////
     // step 1 : loop on global vsegs 
 
@@ -857 +926 @@
     }
 
+    ////////////////////////////////////////////////////////////
     // step 2 : loop on virtual vspaces to map private vsegs
+
     for (vspace_id = 0; vspace_id < header->vspaces; vspace_id++) 
     {
@@ -867 +938 @@
 #endif
 
-        // vsegs with identity mapping constraint first
         for (vseg_id = vspace[vspace_id].vseg_offset;
              vseg_id < (vspace[vspace_id].vseg_offset + vspace[vspace_id].vsegs);
              vseg_id++) 
         {
-            if (vseg[vseg_id].ident == 1) 
-                boot_vseg_map(&vseg[vseg_id], vspace_id);
-        }
-        // unconstrained vsegs second
-        for (vseg_id = vspace[vspace_id].vseg_offset;
-             vseg_id < (vspace[vspace_id].vseg_offset + vspace[vspace_id].vsegs);
-             vseg_id++) 
-        {
-            if (vseg[vseg_id].ident == 0)
-                boot_vseg_map(&vseg[vseg_id], vspace_id);
+            // private vsegs cannot be identity mapping
+            if (vseg[vseg_id].ident != 0) 
+            {
+                _puts("\n[BOOT ERROR] in boot_pt_init() : vspace ");
+                _puts( vspace[vspace_id].name );
+                _puts(" contains vseg with identity mapping\n");
+                _exit();
+            }
+
+            boot_vseg_map(&vseg[vseg_id], vspace_id);
         }
     }
@@ -917 +987 @@
 #endif
 
+    /////////////////////////////////////////////////////////////
     // step 3 : loop on the vspaces to build the page tables
     for (vspace_id = 0; vspace_id < header->vspaces; vspace_id++) 
@@ -929 +1000 @@
         boot_vspace_pt_build(vspace_id);
 
-        _puts("\n[BOOT] Page Table for vspace \"");
-        _puts( vspace[vspace_id].name );
-        _puts("\" completed at cycle ");
-        _putd( _get_proctime() );
-        _puts("\n");
-
-#if BOOT_DEBUG_PT
-_puts("  vaddr = ");
-_putx( _ptabs_vaddr[vspace_id] );
-_puts(" / paddr = ");
-_putl( _ptabs_paddr[vspace_id] );
-_puts(" / PT2 number = ");
-_putd( _max_pt2[vspace_id] );
-_puts("\n");
-#endif
     }
 } // end boot_pt_init()
@@ -972 +1028 @@
 #endif
 
-        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[vspace_id] >> 13) );
+        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[vspace_id][0][0] >> 13) );
 
         unsigned int ptab_found = 0;
@@ -1224 +1280 @@
 // This function initialises all processors schedulers.
 // This is done by processor 0, and the MMU must be activated.
-// - In Step 1, it initialises the _schedulers[gpid] pointers array, and scan
+// - In Step 1, it initialises the _schedulers[] pointers array, and scan
 //              the processors to initialise the schedulers, including the
 //              idle_task context (ltid == 14) and HWI / SWI / PTI vectors. 
@@ -1277 +1333 @@
     //          and the interrupt vectors.
     // Implementation note:
-    // We need to use both proc_id to scan the mapping info structure,
-    // and lpid to access the schedulers array.
-    // - the _schedulers[] array of pointers can contain "holes", because
-    //   it is indexed by the global pid = cluster_xy*NB_PROCS_MAX + lpid
-    // - the mapping info array of processors is contiguous, it is indexed 
-    //   by proc_id, and use an offset specific in each cluster.
+    // We need to use both (proc_id) to scan the mapping info structure,
+    // and (x,y,lpid) to access the schedulers array.
 
     for (cluster_id = 0 ; cluster_id < X_SIZE*Y_SIZE ; cluster_id++) 
@@ -1288 +1340 @@
         unsigned int x          = cluster[cluster_id].x;
         unsigned int y          = cluster[cluster_id].y;
-        unsigned int cluster_xy = (x<<Y_WIDTH) + y;
 
 #if BOOT_DEBUG_SCHED
@@ -1371 +1422 @@
             {
                 // set the schedulers pointers array
-                _schedulers[cluster_xy * NB_PROCS_MAX + lpid] = 
-                      (static_scheduler_t*)&psched[lpid];
+                _schedulers[x][y][lpid] = (static_scheduler_t*)&psched[lpid];
 
 #if BOOT_DEBUG_SCHED
@@ -1411 +1461 @@
                 psched[lpid].context[IDLE_TASK_INDEX][CTX_CR_ID]    = 0;
                 psched[lpid].context[IDLE_TASK_INDEX][CTX_SR_ID]    = 0xFF03;
-                psched[lpid].context[IDLE_TASK_INDEX][CTX_PTPR_ID]  = _ptabs_paddr[0]>>13;
+                psched[lpid].context[IDLE_TASK_INDEX][CTX_PTPR_ID]  = _ptabs_paddr[0][x][y]>>13;
                 psched[lpid].context[IDLE_TASK_INDEX][CTX_PTAB_ID]  = _ptabs_vaddr[0];
                 psched[lpid].context[IDLE_TASK_INDEX][CTX_TTY_ID]   = 0;
@@ -1495 +1545 @@
             unsigned int y          = cluster[cluster_id].y;
             unsigned int cluster_xy = (x<<Y_WIDTH) + y;
-            psched                  = _schedulers[cluster_xy * NB_PROCS_MAX];
+            psched                  = _schedulers[x][y][0];
 
             // update WTI vector for scheduler[cluster_id][lpid]
@@ -1515 +1565 @@
     unsigned int x          = cluster[cluster_id].x;
     unsigned int y          = cluster[cluster_id].y;
-    unsigned int cluster_xy = (x<<Y_WIDTH) + y;
-    psched                  = _schedulers[cluster_xy * NB_PROCS_MAX];
+    psched                  = _schedulers[x][y][0];
     unsigned int slot;
     unsigned int entry;
@@ -1582 +1631 @@
         // We must set the PTPR depending on the vspace, because the start_vector 
         // and the stack address are defined in virtual space.
-        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[vspace_id] >> 13) );
+        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[vspace_id][0][0] >> 13) );
 
         // loop on the tasks in vspace (task_id is the global index)
@@ -1604 +1653 @@
             // compute gpid (global processor index) and scheduler base address
             unsigned int gpid = cluster_xy * NB_PROCS_MAX + lpid;
-            psched            = _schedulers[gpid];
+            psched            = _schedulers[x][y][lpid];
 
             // ctx_sr : value required before an eret instruction
@@ -1610 +1659 @@
 
             // ctx_ptpr : page table physical base address (shifted by 13 bit)
-            unsigned int ctx_ptpr = (unsigned int)(_ptabs_paddr[vspace_id] >> 13);
+            unsigned int ctx_ptpr = (unsigned int)(_ptabs_paddr[vspace_id][x][y] >> 13);
 
             // ctx_ptab : page_table virtual base address
@@ -1811 +1860 @@
 void boot_mapping_init()
 {
+    // desactivates IOC interrupt
     _ioc_init( 0 );
 
+    // open file "map.bin"
     int fd_id = _fat_open( IOC_BOOT_MODE,
                            "map.bin",
                            0 );         // no creation
-
     if ( fd_id == -1 )
     {
@@ -1829 +1879 @@
 #endif
 
+    // get "map.bin" file size (from fat32) and check it
     unsigned int size    = fat.fd[fd_id].file_size;
+
+    if ( size > SEG_BOOT_MAPPING_SIZE )
+    {
+        _puts("\n[BOOT ERROR] : allocated segment too small for map.bin file\n");
+        _exit();
+    }
+
+    // load "map.bin" file into buffer
     unsigned int nblocks = size >> 9;
     unsigned int offset  = size & 0x1FF;
@@ -1846 +1905 @@
     _fat_close( fd_id );
     
+    // close file "map.bin"
     boot_mapping_check();
+
 } // end boot_mapping_init()
 
 
-//////////////////////////////////////////////////////////////////////////////////
-// This function open the .elf file identified by the "pathname" argument.
-// It loads the complete file in a dedicated buffer, it copies all loadable 
-// segments  at the memory virtual address defined in the .elf file, 
-// and close the file.
-// Notes: 
-// - The processor PTPR should contain the value corresponding to the
-//   vspace containing the .elf file.
-// - As this function requires a temporary memory buffer
-//   to load the complete .elf file before to copy the various segments
-//   to te proper location, it uses the seg_boot_buffer defined in map.xml.
-//////////////////////////////////////////////////////////////////////////////////
-void load_one_elf_file( unsigned int mode,
+/////////////////////////////////////////////////////////////////////////////////////
+// This function load all loadable segments for one .elf file, identified 
+// by the "pathname" argument. Some loadable segments can be copied in several
+// clusters: same virtual address but different physical addresses.  
+// - It open the file.
+// - It loads the complete file in a dedicated buffer (seg_boot_buffer).
+// - It copies each loadable segments  at the virtual address defined in the .elf
+//   file, making several copies if the target vseg is not local.
+// - It closes the file.
+// Note: 
+// - This function is supposed to be executed by processor[0,0,0].
+//   We must use physical addresses to reach the destination buffers that
+//   can be located in remote clusters. We use either a _physical_memcpy(), 
+//   or a _dma_physical_copy() if DMA is available.
+//   The source seg_boot_buffer must be identity mapping.
+//////////////////////////////////////////////////////////////////////////////////////
+void load_one_elf_file( unsigned int is_kernel,     // kernel file if non zero
                         char*        pathname,
-                        unsigned int vspace_id )    // to use the proper page_table
+                        unsigned int vspace_id )    // to scan the proper vspace
 {
+    mapping_header_t  * header  = (mapping_header_t *)SEG_BOOT_MAPPING_BASE;
+    mapping_vspace_t  * vspace  = _get_vspace_base(header);
+    mapping_vseg_t    * vseg    = _get_vseg_base(header);
+    mapping_vobj_t    * vobj    = _get_vobj_base(header);
+
     unsigned int seg_id;
 
@@ -1881 +1951 @@
 
     // open .elf file
-    int fd_id = _fat_open( mode,
+    int fd_id = _fat_open( IOC_BOOT_MODE,
                            pathname,
                            0 );      // no creation
@@ -1907 +1977 @@
 
     // load file in boot_buffer
-    if( _fat_read( mode, 
+    if( _fat_read( IOC_BOOT_MODE, 
                    fd_id, 
                    boot_buffer,
@@ -1947 +2017 @@
     unsigned int nsegments   = elf_header_ptr->e_phnum;
 
-#if BOOT_DEBUG_ELF
-_puts("\n[BOOT DEBUG] File ");
-_puts( pathname );
-_puts(" loaded at cycle ");
-_putd( _get_proctime() );
-_puts(" / bytes = ");
-_putd( nbytes );
-_puts(" / sectors = ");
-_putd( nsectors );
-_puts("\n");
-#endif
-
-    // Loop on loadable segments in the ELF file
+    _puts("\n[BOOT] File ");
+    _puts( pathname );
+    _puts(" loaded at cycle ");
+    _putd( _get_proctime() );
+    _puts("\n");
+
+    // Loop on loadable segments in the .elf file
     for (seg_id = 0 ; seg_id < nsegments ; seg_id++)
     {
@@ -1970 +2034 @@
             unsigned int seg_memsz  = elf_pht_ptr[seg_id].p_memsz;
 
+#if BOOT_DEBUG_ELF
+_puts(" - segment ");
+_putd( seg_id );
+_puts(" / vaddr = ");
+_putx( seg_vaddr );
+_puts(" / file_size = ");
+_putx( seg_filesz );
+_puts("\n");
+#endif
+
             if( seg_memsz < seg_filesz )
             {
@@ -1976 +2050 @@
                 _puts(" in file ");
                 _puts( pathname );
-                _puts(" has a wrong size \n");   
+                _puts(" has a memsz < filesz \n");   
                 _exit();
             }
@@ -1989 +2063 @@
             unsigned int src_vaddr = (unsigned int)boot_buffer + seg_offset;
 
-#if BOOT_DEBUG_ELF
-_puts(" - segment ");
-_putd( seg_id );
-_puts(" / dst_vaddr = ");
-_putx( seg_vaddr );
-_puts(" / src_vaddr = ");
-_putx( src_vaddr );
-_puts(" / size = ");
-_putx( seg_filesz );
-_puts("\n");
-#endif
-
-            // copy the segment from boot buffer to destination buffer
-            if( NB_DMA_CHANNELS > 0 )
-            {
-                _dma_copy( 0,              // DMA cluster index
-                           0,              // DMA channel index 
-                           vspace_id,      
-                           seg_vaddr,
-                           src_vaddr,
-                           seg_filesz );
+            // search all vsegs matching the virtual address
+            unsigned int vseg_first;
+            unsigned int vseg_last;
+            unsigned int vseg_id;
+            unsigned int found = 0;
+            if ( is_kernel )
+            {
+                vseg_first = 0;
+                vseg_last  = header->globals;
             }
             else
             {
-                memcpy( (char*)seg_vaddr,
-                        (char*)src_vaddr,
-                        seg_filesz );
+                vseg_first = vspace[vspace_id].vseg_offset;
+                vseg_last  = vseg_first + vspace[vspace_id].vsegs;
+            }
+
+            for ( vseg_id = vseg_first ; vseg_id < vseg_last ; vseg_id++ )
+            {
+                if ( seg_vaddr == vseg[vseg_id].vbase )  // matching 
+                {
+                    found = 1;
+
+                    // get destination buffer physical address and size
+                    paddr_t      seg_paddr  = vseg[vseg_id].pbase;
+                    unsigned int vobj_id    = vseg[vseg_id].vobj_offset;
+                    unsigned int seg_size   = vobj[vobj_id].length;
+                    
+#if BOOT_DEBUG_ELF
+_puts("   loaded into vseg ");
+_puts( vseg[vseg_id].name );
+_puts(" at paddr = ");
+_putl( seg_paddr );
+_puts(" (buffer size = ");
+_putx( seg_size );
+_puts(")\n");
+#endif
+                    // check vseg size
+                    if ( seg_size < seg_filesz )
+                    {
+                        _puts("\n[BOOT ERROR] in load_one_elf_file()\n");
+                        _puts("vseg ");
+                        _puts( vseg[vseg_id].name );
+                        _puts(" is to small for loadable segment ");
+                        _putx( seg_vaddr );
+                        _puts(" in file ");
+                        _puts( pathname );
+                        _puts(" \n");   
+                        _exit();
+                    }
+
+                    // copy the segment from boot buffer to destination buffer
+                    // using DMA channel[0,0,0] if it is available.
+                    if( NB_DMA_CHANNELS > 0 )
+                    {
+                        _dma_physical_copy( 0,                  // DMA in cluster[0,0]
+                                            0,                  // DMA channel 0
+                                            (paddr_t)seg_paddr, // destination paddr
+                                            (paddr_t)src_vaddr, // source paddr
+                                            seg_filesz );       // size
+                    }
+                    else
+                    {
+                        _physical_memcpy( (paddr_t)seg_paddr,   // destination paddr
+                                          (paddr_t)src_vaddr,   // source paddr
+                                          seg_filesz );         // size
+                    }
+                }
+            }  // end for vsegs in vspace
+
+            // check at least one matching vseg
+            if ( found == 0 )
+            {
+                _puts("\n[BOOT ERROR] in load_one_elf_file()\n");
+                _puts("vseg for loadable segment ");
+                _putx( seg_vaddr );
+                _puts(" in file ");
+                _puts( pathname );
+                _puts(" not found \n");   
+                _exit();
             }
         }
-    } // end for segments
+    }  // end for loadable segments
 
     // close .elf file
@@ -2026 +2152 @@
 
 
-//////////////////////////////////////////////////////////////////////////////////
+/////i////////////////////////////////////////////////////////////////////////////////
 // This function uses the map.bin data structure to load the "kernel.elf" file
-// as well as the various "application.elf" files.
-// The "preloader.elf" file is not loaded, because it has been burned in the ROM.
-// The "boot.elf" file is not loaded, because it has been loaded by the preloader.
+// as well as the various "application.elf" files into memory.
+// - The "preloader.elf" file is not loaded, because it has been burned in the ROM.
+// - The "boot.elf" file is not loaded, because it has been loaded by the preloader.
 // This function scans all vobjs defined in the map.bin data structure to collect
-// all .elf files pathnames, and calls the load_one_elf_file() function to
-// load all loadable segments at the virtual address found in the .elf file.
-//////////////////////////////////////////////////////////////////////////////////
+// all .elf files pathnames, and calls the load_one_elf_file() for each .elf file.
+// As the code can be replicated in several vsegs, the same code can be copied 
+// in one or several clusters by the load_one_elf_file() function.
+//////////////////////////////////////////////////////////////////////////////////////
 void boot_elf_load()
 {
@@ -2063 +2190 @@
     }
 
-    load_one_elf_file( IOC_BOOT_MODE,
-                       vobj[vobj_id].binpath,
+    // Load the kernel 
+    load_one_elf_file( 1,                           // kernel file
+                       vobj[vobj_id].binpath,       // file pathname
                        0 );                         // vspace 0
 
-    _puts("\n[BOOT] File \"");
-    _puts( vobj[vobj_id].binpath );
-    _puts("\" loaded at cycle ");
-    _putd( _get_proctime() );
-    _puts("\n");
-
-    // loop on the vspaces, scanning all vobjs in a vspace,
+    // loop on the vspaces, scanning all vobjs in the vspace,
     // to find the pathname of the .elf file associated to the vspace.
     for( vspace_id = 0 ; vspace_id < header->vspaces ; vspace_id++ )
     {
-        // Set PTPR depending on the vspace, as seg_data is defined in virtual space.
-        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[vspace_id] >> 13) );
-
         // loop on the vobjs in vspace (vobj_id is the global index)
         unsigned int found = 0;
@@ -2102 +2221 @@
         }
 
-        load_one_elf_file( IOC_BOOT_MODE,
-                           vobj[vobj_id].binpath,
-                           vspace_id );
-
-        _puts("\n[BOOT] File \"");
-        _puts( vobj[vobj_id].binpath );
-        _puts("\" loaded at cycle ");
-        _putd( _get_proctime() );
-        _puts("\n");
+        load_one_elf_file( 0,                          // not a kernel file
+                           vobj[vobj_id].binpath,      // file pathname
+                           vspace_id );                // vspace index
 
     }  // end for vspaces
-
-    // restaure vspace 0 PTPR
-    _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[0] >> 13) );
 
 } // end boot_elf_load()
@@ -2161 +2271 @@
         {
             unsigned int type       = periph[periph_id].type;
+            unsigned int subtype    = periph[periph_id].subtype;
             unsigned int channels   = periph[periph_id].channels;
 
@@ -2167 +2278 @@
                 case PERIPH_TYPE_IOC:    // vci_block_device component
                 {
-                    // initialize all channels except channel 0 because it has been
-                    // initialized by the preloader.
-                    for (channel_id = 1; channel_id < channels; channel_id++) 
+                    if ( subtype == PERIPH_SUBTYPE_BDV )
                     {
-                        _ioc_init( channel_id );
+                        _bdv_lock.value = 0;
+#if BOOT_DEBUG_PERI
+_puts("- BDV : channels = ");
+_putd(channels);
+_puts("\n");
+#endif
                     }
-#if BOOT_DEBUG_PERI
-_puts("- IOC : channels = ");
-_putd(channels);
-_puts("\n");
-#endif
+                    else if ( subtype == PERIPH_SUBTYPE_HBA )
+                    {
+                        // TODO
+                    }
+                    else if ( subtype == PERIPH_SUBTYPE_SPI )
+                    {
+                        // TODO
+                    }
                     break;
                 }
@@ -2208 +2325 @@
                 case PERIPH_TYPE_TTY:    // vci_multi_tty component
                 {
-                    // nothing to do
+                    for (channel_id = 0; channel_id < channels; channel_id++) 
+                    {
+                        _tty_lock[channel_id].value = 0;
+                        _tty_rx_full[channel_id]    = 0;
+                    }
 #if BOOT_DEBUG_PERI
 _puts("- TTY : channels = ");
@@ -2218 +2339 @@
                 case PERIPH_TYPE_IOB:    // vci_io_bridge component
                 {
-#if BOOT_DEBUG_PERI
-_puts("- IOB : channels = ");
-_putd(channels);
-_puts("\n");
-#endif
                     if (GIET_USE_IOMMU) 
                     {
@@ -2236 +2352 @@
                 case PERIPH_TYPE_PIC:    // vci_iopic component
                 {
-                          
 #if BOOT_DEBUG_PERI
 _puts("- PIC : channels = ");
@@ -2329 +2444 @@
 // Most of this code is executed by Processor 0 only.
 /////////////////////////////////////////////////////////////////////////
-void boot_init()
+void boot_init() 
 {
     mapping_header_t*  header     = (mapping_header_t *)SEG_BOOT_MAPPING_BASE;
@@ -2341 +2456 @@
         _puts("\n");
 
-        // Loading the map.bin file into memory and checking it
+        // Load the map.bin file into memory and check it
         boot_mapping_init();
 
@@ -2350 +2465 @@
         _puts("\n");
 
-        // Building all page tables
+        // Build page tables
         boot_pt_init();
 
-        // Activating proc 0 MMU 
-        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[0]>>13) );
+        // Activate MMU for proc [0,0,0]
+        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[0][0][0]>>13) );
         _set_mmu_mode( 0xF );
 
@@ -2361 +2476 @@
         _puts("\n");
 
-        // Initialising private vobjs in vspaces
+        // Initialise private vobjs in vspaces
         boot_vobjs_init();
 
@@ -2368 +2483 @@
         _puts("\n");
 
-        // Initializing schedulers
+        // Initialise schedulers
         boot_schedulers_init();
 
@@ -2375 +2490 @@
         _puts("\n");
         
-        // Setting CP0_SCHED register for proc 0
-        _set_sched( (unsigned int)_schedulers[0] );
-
-        // Initializing non replicated peripherals
+        // Set CP0_SCHED register for proc [0,0,0]
+        _set_sched( (unsigned int)_schedulers[0][0][0] );
+
+        // Initialise non replicated peripherals
         boot_peripherals_init();
 
@@ -2394 +2509 @@
         {
             unsigned int nprocs     = cluster[clusterid].procs;
-            unsigned int xdest      = cluster[clusterid].x;
-            unsigned int ydest      = cluster[clusterid].y;
-            unsigned int cluster_xy = (xdest<<Y_WIDTH) + ydest;
+            unsigned int x          = cluster[clusterid].x;
+            unsigned int y          = cluster[clusterid].y;
+            unsigned int cluster_xy = (x<<Y_WIDTH) + y;
 
             for ( p = 0 ; p < nprocs; p++ ) 
@@ -2414 +2529 @@
 
     // all processor initialise SCHED register
-    _set_sched( (unsigned int)_schedulers[gpid] );
-
-    // all processors (but Proc 0) activate MMU
+    unsigned int cluster_xy = gpid / NB_PROCS_MAX;
+    unsigned int lpid       = gpid % NB_PROCS_MAX;
+    unsigned int x          = cluster_xy >> Y_WIDTH;
+    unsigned int y          = cluster_xy & ((1<<Y_WIDTH)-1);
+    _set_sched( (unsigned int)_schedulers[x][y][lpid] );
+
+    // all processors (but Proc[0,0,0]) activate MMU
     if ( gpid != 0 )
     {
-        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[0]>>13) );
+        _set_mmu_ptpr( (unsigned int)(_ptabs_paddr[0][x][y]>>13) );
         _set_mmu_mode( 0xF );
     }