Index: trunk/boot/tsar_mips32/boot.c
===================================================================
--- trunk/boot/tsar_mips32/boot.c	(revision 578)
+++ trunk/boot/tsar_mips32/boot.c	(revision 623)
@@ -3,5 +3,5 @@
  * 
  * Authors :   Vu Son  (2016)
- *             Alain Greiner (2016, 2017,2018)
+ *             Alain Greiner (2016,2017,2018,2019)
  *
  * Copyright (c) UPMC Sorbonne Universites
@@ -26,45 +26,33 @@
  * This file contains the ALMOS-MKH. boot-loader for the TSAR architecture. *
  *                                                                          *
- * It supports clusterised shared memory multi-processor architectures,     *
+ * It supports a clusterised, shared memory, multi-processor architecture,  *
  * where each processor core is identified by a composite index [cxy,lid]   *
  * with one physical memory bank per cluster.                               *
  *                                                                          *
  * The 'boot.elf' file (containing the boot-loader binary code) is stored   *
- * on disk and is loaded into memory by core[0,0] (cxy = 0 / lid = 0),      *
- * and is copied in each other cluter by the local CP0 (lid = 0].           *
+ * on disk (not in the FAT file system), and must be loaded into memory by  *
+ * the preloader running on the core[0][0] (cxy = 0 / lid = 0).             *
  *                                                                          *
- * 1) The boot-loader first phase is executed by core[0,0], while           *
- *    all other cores are waiting in the preloader.                         *
- *    It does the following tasks:                                          *
- *      - load into the memory bank of cluster 0 the 'arch_info.bin'        *
- *        file (containing the hardware architecture description) and the   *
- *        'kernel.elf' file, at temporary locations,                        *    
- *      - initializes the 'boot_info_t' structure in cluster(0,0)           *
- *        (there is 1 'boot_info_t' per cluster), which contains both       *
- *        global and cluster specific information that will be used for     *
- *        kernel initialisation.                                            *
- *      - activate CP0s in all other clusters, using IPIs.                  *
- *      - wait completion reports from CP0s on a global barrier.            * 
+ * The main task of the boot-loader is to load in the first physical page   *
+ * of each cluster a copy of the kernel code (segments "kcode" and "kdata") *
+ * and to build - in each cluster - a cluster specific description of the   *
+ * hardware archtecture, stored in the "kdata" segment as the boot_info_t   *
+ * structure. The "kernel.elf" and "arch_info.bin" files are supposed to be *
+ * stored on disk in a FAT32 file system.                                   *
  *                                                                          *
- * 2) The boot-loader second phase is then executed in parallel by all      *
- *    CP0s (other than core[0,0]). Each CP0 performs the following tasks:   *
- *      - copies into the memory bank of the local cluster the 'boot.elf',  *
- *        the 'arch_info.bin' (at the same addresses as the 'boot.elf' and  *
- *        the 'arch_info.bin' in the memory bank of the cluster(0,0), and   *
- *        the kernel image (at address 0x0),                                *
- *      - initializes the 'boot_info_t' structure of the local cluster,     *
- *      - activate all other cores in the same cluster (CPi).               *
- *      - wait local CPi completion reports on a local barrier.             *
- *      - report completion  on the global barrier.                         *
+ * All cores contribute to the boot procedure, but all cores are not        *
+ * simultaneously active:                                                   *
+ * - in a first phase, only core[0][0] is running (core 0 in cluster 0).    * 
+ * - in a second phase, only core[cxy][0] is running in each cluster.       *
+ * - in last phase, all core[cxy][lid] are running.                         *
  *                                                                          *
- * 3) The boot-loader third phase is executed in parallel by all cores.     *
- *    In each cluster (i) the CP0                                           *
- *      - activates the other cores of cluster(i),                          *
- *      - blocks on the local barrier waiting for all local CPi to report   *
- *        completion on the local barrier,                                  *
- *      - moves the local kernel image from the temporary location to the   *
- *        address 0x0, (erasing the preloader code).                        *
+ * Finally, all cores jump to the kernel_init() function that makes the     *                 
+ * actual kernel initialisation.                                            *
  *                                                                          *
- * 4) All cores jump to kern_init() (maybe not at the same time).           *
+ * Implementation note:                                                     *                      *                                                                          *
+ * To allows each core to use the local copy of both the boot code and the  *
+ * kernel code, the boot-loader builds a minimal and temporary BPT (Boot    *
+ * Page Table) containing only two big pages: page[0] maps the kernel code, *
+ * and page 1 maps the boot code.                                           *
  ****************************************************************************/
 
@@ -96,10 +84,14 @@
  ****************************************************************************/
 
+// the Boot Page Table contains two PTE1, and should be aligned on 8 Kbytes 
+
+uint32_t                        boot_pt[2] __attribute__((aligned(2048)));
+
 // synchronization variables. 
 
-volatile boot_remote_spinlock_t tty0_lock;       // protect TTY0 access
-volatile boot_remote_barrier_t  global_barrier;  // synchronize CP0 cores
-volatile boot_remote_barrier_t  local_barrier;   // synchronize cores in one cluster
-uint32_t                        active_cp0s_nr;  // number of expected CP0s
+volatile boot_remote_spinlock_t tty0_lock;        // protect TTY0 access
+volatile boot_remote_barrier_t  global_barrier;   // synchronize CP0 cores
+volatile boot_remote_barrier_t  local_barrier;    // synchronize cores in one cluster
+uint32_t                        active_cores_nr;  // number of expected CP0s
  
 // kernel segments layout variables
@@ -114,5 +106,5 @@
 uint32_t                        kernel_entry;    // kernel entry point
 
-// Functions called by boot_entry.S 
+// Functions 
 
 extern void boot_entry( void );    // boot_loader entry point
@@ -738,5 +730,5 @@
 
 /*********************************************************************************
- * This function is called by all CP0 to activate the other CPi cores. 
+ * This function is called by all CP0s to activate the other CPi cores. 
  * @ boot_info  : pointer to local 'boot_info_t' structure.
  *********************************************************************************/
@@ -761,4 +753,37 @@
 } // boot_wake_local_cores()
 
+/*********************************************************************************
+ * This function is called by all core[cxy][0] to initialize the Boot Page Table:
+ * map two local big pages for the boot code and kernel code.
+ * @ cxy    : local cluster identifier.
+ *********************************************************************************/
+void boot_page_table_init( cxy_t  cxy )
+{
+    // set PTE1 in slot[0] for kernel code
+    uint32_t kernel_attr  = 0x8A800000;                   // flagss : V,C,X,G
+    uint32_t kernel_ppn1  = (cxy << 20) >> 9;             // big physical page index == 0
+    boot_pt[0]            = kernel_attr | kernel_ppn1;
+
+    // set PTE1 in slot[1] for boot code (no global flag)
+    uint32_t boot_attr    = 0x8A000000;                   // flags : V,C,X
+    uint32_t boot_ppn1    = ((cxy << 20) + 512) >> 9;     // big physical page index == 1
+    boot_pt[1]            = boot_attr | boot_ppn1;
+}
+
+/*********************************************************************************
+ * This function is called by all cores to activate the instruction MMU,
+ * and use the local copy of boot code.
+ *********************************************************************************/
+void boot_activate_ins_mmu( cxy_t cxy )
+{
+    // set mmu_ptpr register
+    uint32_t ptpr = ((uint32_t)boot_pt >> 13) | (cxy << 19);
+    asm volatile ( "mtc2   %0,   $0         \n" : : "r" (ptpr) );
+
+    // set ITLB bit in mmu_mode 
+    asm volatile ( "mfc2   $26,  $1         \n"
+                   "ori    $26,  $26,  0x8  \n" 
+                   "mtc2   $26,  $1         \n" );
+}
 
 /*********************************************************************************
@@ -776,8 +801,8 @@
     if (lid == 0) 
     {
-        /****************************************************
-         * PHASE A : only CP0 in boot cluster executes it 
-         ***************************************************/
-        if (cxy == BOOT_CORE_CXY)
+        /************************************i**********************
+         * PHASE Sequencial : only core[0][0] executes it 
+         **********************************************************/
+        if (cxy == 0)
         {
             boot_printf("\n[BOOT] core[%x,%d] enters at cycle %d\n",
@@ -833,21 +858,21 @@
             boot_check_core(boot_info, lid);
 
-            // Activate other CP0s / get number of active CP0s
-            active_cp0s_nr = boot_wake_all_cp0s() + 1;
+// TO BE DONE 
+// core[0][0] identity maps two big pages for the boot and kernel code,
+// boot_page_table_init( 0 );
+
+// TO BE DONE
+// core[0][0] activates the instruction MMU to use the local copy of boot code
+// boot_activate_ins_mmu( 0 );
+
+            // Activate other core[cxy][0] / get number of activated cores
+            active_cores_nr = boot_wake_all_cp0s() + 1;
 
             // Wait until all clusters (i.e all CP0s) ready to enter kernel.
             boot_remote_barrier( XPTR( BOOT_CORE_CXY , &global_barrier ) ,
-                                 active_cp0s_nr );
+                                 active_cores_nr );
 
             // activate other local cores
             boot_wake_local_cores( boot_info );
-
-// display address extensions
-// uint32_t cp2_data_ext;
-// uint32_t cp2_ins_ext;
-// asm volatile( "mfc2   %0,  $24" : "=&r" (cp2_data_ext) );
-// asm volatile( "mfc2   %0,  $25" : "=&r" (cp2_ins_ext) );
-// boot_printf("\n[BOOT] core[%x,%d] CP2_DATA_EXT = %x / CP2_INS_EXT = %x\n",
-// cxy , lid , cp2_data_ext , cp2_ins_ext );
 
             // Wait until all local cores in cluster ready
@@ -855,26 +880,31 @@
                                  boot_info->cores_nr );
         }
-        /******************************************************************
-         * PHASE B : all CP0s other than CP0 in boot cluster execute it
-         *****************************************************************/
+        /**************************************************************************
+         * PHASE partially parallel : all core[cxy][0] with (cxy != 0) execute it
+         **************************************************************************/
         else
         {
-            // at this point, all INSTRUCTION address extension registers
-            // point on cluster(0,0), but the DATA extension registers point
-            // already on the local cluster to use the local stack. 
-            // To access the bootloader global variables we must first copy
-            // the boot code (data and instructions) in the local cluster.
+            // at this point, the DATA extension registers point
+            // already on the local cluster cxy to use the local stack, 
+            // but all cores must access the code stored in cluster 0 
+
+            // Each CP0 copies the boot code (data and instructions)
+            // from the cluster 0 to the local cluster.
             boot_remote_memcpy( XPTR( cxy           , BOOT_BASE ),
                                 XPTR( BOOT_CORE_CXY , BOOT_BASE ),
                                 BOOT_MAX_SIZE );
 
-            // from now, it is safe to refer to the boot code global variables 
+            // from now, it is safe to refer to the boot global variables 
             boot_printf("\n[BOOT] core[%x,%d] replicated boot code at cycle %d\n",
             cxy , lid , boot_get_proctime() );
 
-			// switch to the INSTRUCTION local memory space, to avoid contention.
-            // asm volatile("mtc2  %0, $25" :: "r"(cxy));
-
-            // Copy the arch_info.bin file into the local memory.
+// TO BE DONE 
+// Each core identity maps two big pages for the boot and kernel code,
+// boot_page_table_init( cxy );
+
+// Each core activates the instruction MMU to use the local copy of boot code
+// boot_activate_ins_mmu( cxy );
+
+            // Each CP0 copies the arch_info.bin into the local memory.
             boot_remote_memcpy(XPTR(cxy,           ARCHINFO_BASE),
                                XPTR(BOOT_CORE_CXY, ARCHINFO_BASE),
@@ -884,15 +914,16 @@
             cxy , lid , boot_get_proctime() );
 
-            // Copy the kcode segment into local memory 
+            // Each CP0 copies the kcode segment into local memory 
             boot_remote_memcpy( XPTR( cxy           , seg_kcode_base ),
                                 XPTR( BOOT_CORE_CXY , seg_kcode_base ),
                                 seg_kcode_size );
 
-            // Copy the kdata segment into local memory
+            // Each CP0 copies the kdata segment into local memory
             boot_remote_memcpy( XPTR( cxy           , seg_kdata_base ),
                                 XPTR( BOOT_CORE_CXY , seg_kdata_base ),
                                 seg_kdata_size );
 
-            // Copy the kentry segment into local memory
+            // [TO BE REMOVED<D-°>
+            // Each CP0 copies the kentry segment into local memory
             boot_remote_memcpy( XPTR( cxy           , seg_kentry_base ),
                                 XPTR( BOOT_CORE_CXY , seg_kentry_base ),
@@ -902,8 +933,8 @@
             cxy , lid , boot_get_proctime() );
 
-            // Get local boot_info_t structure base address.
+            // Each CP0 get local boot_info_t structure base address.
             boot_info = (boot_info_t*)seg_kdata_base;
 
-            // Initialize local boot_info_t structure.
+            // Each CP0 initializes local boot_info_t structure.
             boot_info_init( boot_info , cxy );
 
@@ -911,23 +942,15 @@
             cxy , lid , boot_get_proctime() );
 
-            // Check core information.
+            // Each CP0 checks core information.
             boot_check_core( boot_info , lid );
 
-            // get number of active clusters from BOOT_CORE cluster
-            uint32_t count = boot_remote_lw( XPTR( BOOT_CORE_CXY , &active_cp0s_nr ) );
-
-            // Wait until all clusters (i.e all CP0s) ready to enter kernel
+            // Each CP0 get number of active clusters from BOOT_CORE cluster
+            uint32_t count = boot_remote_lw( XPTR( 0 , &active_cores_nr ) );
+
+            // Wait until all clusters (i.e all CP0s) ready 
             boot_remote_barrier( XPTR( BOOT_CORE_CXY , &global_barrier ) , count );
 
             // activate other local cores
             boot_wake_local_cores( boot_info );
-
-// display address extensions
-// uint32_t cp2_data_ext;
-// uint32_t cp2_ins_ext;
-// asm volatile( "mfc2   %0,  $24" : "=&r" (cp2_data_ext) );
-// asm volatile( "mfc2   %0,  $25" : "=&r" (cp2_ins_ext) );
-// boot_printf("\n[BOOT] core[%x,%d] CP2_DATA_EXT = %x / CP2_INS_EXT = %x\n",
-// cxy , lid , cp2_data_ext , cp2_ins_ext );
 
             // Wait until all local cores in cluster ready
@@ -938,11 +961,11 @@
     else
     {
-        /***************************************************************
-         * PHASE C: all non CP0 cores in all clusters execute it
-         **************************************************************/
-
-        // Switch to the INSTRUCTIONS local memory space 
-        // to avoid contention at the boot cluster.
-        asm volatile("mtc2  %0, $25" :: "r"(cxy));
+        /***********************************************************************
+         * PHASE fully parallel : all cores[cxy][lid] with (lid! = 0) execute it
+         **********************************************************************/
+
+// TO BE DONE
+// each core activate the instruction MMU to use the local copy of the boot code
+// boot_activate_ins_mmu( cxy );
 
         // Get local boot_info_t structure base address.
@@ -952,36 +975,33 @@
         boot_check_core(boot_info, lid);
 
-// display address extensions
-// uint32_t cp2_data_ext;
-// uint32_t cp2_ins_ext;
-// asm volatile( "mfc2   %0,  $24" : "=&r" (cp2_data_ext) );
-// asm volatile( "mfc2   %0,  $25" : "=&r" (cp2_ins_ext) );
-// boot_printf("\n[BOOT] core[%x,%d] CP2_DATA_EXT = %x / CP2_INS_EXT = %x\n",
-// cxy , lid , cp2_data_ext , cp2_ins_ext );
-
         // Wait until all local cores in cluster ready
         boot_remote_barrier( XPTR( cxy , &local_barrier ) , boot_info->cores_nr );
     }
 
+    // the "kernel_entry" global variable, set by boot_kernel_load() define
+    // the adress of the kernel_init() function.
     // Each core initialise the following registers before jumping to kernel: 
-    // - sp_29    : stack pointer on idle thread,
-    // - c0_sr    : reset BEV bit
-    // - a0_04    : pointer on boot_info structure
-    // - c0_ebase : kentry_base(and jump to kernel_entry. 
-
+    // - gr_29    : stack pointer / kernel stack allocated in idle thread descriptor,
+    // - c0_sr    : status register / reset BEV bit
+    // - gr_04    : kernel_init() argument / pointer on boot_info structure
+    // - c0_ebase : kentry_base 
+
+    // compute "sp" from base address of idle thread descriptors array and lid.
     // The array of idle-thread descriptors is allocated in the kdata segment,
-    // just after the boot_info structure
-    uint32_t sp;
+    // just after the boot_info structure.
     uint32_t base;
     uint32_t offset = sizeof( boot_info_t );
     uint32_t pmask  = CONFIG_PPM_PAGE_MASK;
     uint32_t psize  = CONFIG_PPM_PAGE_SIZE;
-
-    // compute base address of idle thread descriptors array
     if( offset & pmask ) base = seg_kdata_base + (offset & ~pmask) + psize;
     else                 base = seg_kdata_base + offset;
-
-    // compute stack pointer
-    sp = base + ((lid + 1) * CONFIG_THREAD_DESC_SIZE) - 16;
+    uint32_t sp = base + ((lid + 1) * CONFIG_THREAD_DESC_SIZE) - 16;
+
+    // get "ebase" from kerneL_info
+    uint32_t ebase = boot_info->kentry_base;
+
+// TO BE DONE
+// The cp0_ebase will not be set by the assenbly code below
+// when the kentry segment will be removed => done in kernel init
 
     asm volatile( "mfc0  $27,  $12           \n"
@@ -997,5 +1017,5 @@
                   : "r"(boot_info) ,
                     "r"(sp) ,
-                    "r"(boot_info->kentry_base) ,
+                    "r"(ebase) ,
                     "r"(kernel_entry) 
                   : "$26" , "$27" , "$29" , "$4" );
