Index: trunk/kernel/kern/cluster.h
===================================================================
--- trunk/kernel/kern/cluster.h	(revision 690)
+++ trunk/kernel/kern/cluster.h	(revision 1)
@@ -1,8 +1,8 @@
 /*
  * cluster.h - Cluster-Manager definition
- *
- * authors  Ghassan Almaless       (2008,2009,2010,2011,2012)
+ * 
+ * authors  Ghassan Almaless (2008,2009,2010,2011,2012)
  *          Mohamed Lamine Karaoui (2015)
- *          Alain Greiner          (2016,2017,2018,2019,2020)
+ *          Alain Greiner (2016)
  *
  * Copyright (c) UPMC Sorbonne Universites
@@ -27,9 +27,10 @@
 #define _CLUSTER_H_
 
-#include <kernel_config.h>
-#include <hal_kernel_types.h>
+#include <almos_config.h>
+#include <hal_types.h>
 #include <bits.h>
-#include <queuelock.h>
-#include <remote_queuelock.h>
+#include <spinlock.h>
+#include <readlock.h>
+#include <remote_barrier.h>
 #include <list.h>
 #include <xlist.h>
@@ -39,4 +40,5 @@
 #include <ppm.h>
 #include <kcm.h>
+#include <khm.h>
 #include <rpc.h>
 #include <core.h>
@@ -47,4 +49,5 @@
 struct core_s;
 struct process_s;
+struct device_s;
 
 
@@ -52,38 +55,34 @@
  * This structure defines the process manager, that is part of the cluster manager.
  * For any process P, the process descriptor is replicated in all clusters containing
- * at least one thread of process P, but only the "reference" cluster descriptor contains
- * the reference (complete) GPT, VSL, and FDT structures. 
- * The "owner" cluster K is in charge to allocate a lpid (local process index), 
- * to the owned processes, and to register the "reference" cluster for these processes.
- *
- * Warning : the "owner" cluster, and the "reference" cluster can be different clusters.
- *
- * The process manager of a cluster K maintains three sets of process descriptors:
- *
- * 1) pref_tbl[] is an array indexed by lpid. There is one entry per owned process.
+ * at least one thread of process P, but only the "reference" cluster descriptor contains 
+ * the reference (complete) structures such as the GPT, the VSL, or the FDT.
+ * The "owner" cluster is in charge to allocate a lpid (local process index), 
+ * for all process owned by a cluster K, and to register the "reference" cluster for
+ * all process owned by K. 
+ *
+ * Warning : the "owner" cluster, and the "reference" cluster can be different clusters. 
+ *
+ * The process manager of a cluster K maintains three structures:
+ * 1) The pref_tbl[] is an array indexed by lpid, for all processes owned by cluster K.
  *    Each entry contains an extended pointer on the reference process descriptor.
- *
- * 2) The local_root is the root of the local list of all process descriptors in cluster K.
+ * 2) The local_root is the root of the local list of process descriptors in cluster K.
  *    A process descriptor P is present in K, as soon as P has a thread in cluster K.
- *    We use an xlist, because this list can be traversed by remote threads.
- *
- * 3) The copies_root[] array is indexed by lpid. There is one entry per owned process,
- *    and each each entry contains the root of the xlist of copies for this process.
- $    We use an xlist, because process copies are distributed in all clusters. 
+ * 3) The copies_root[] array is indexed by lpid. Each entry contains the root of
+ *    the xlist of copies for a given process owned by cluster K.   
  ******************************************************************************************/
 
 typedef struct process_manager_s
 {
-	xptr_t             pref_tbl[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! owned  processes    */
-	queuelock_t        pref_lock;              /*! lock protecting pref_tbl               */
-    uint32_t           pref_nr;                /*! number of processes owned by cluster   */
-
-    xlist_entry_t      local_root;            /*! root of list of process in cluster      */
-    remote_queuelock_t local_lock;            /*! lock protecting local list              */
-    uint32_t           local_nr;              /*! number of process in cluster            */
-
-    xlist_entry_t      copies_root[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! roots of lists   */
-    remote_queuelock_t copies_lock[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! one  per list    */
-    uint32_t           copies_nr[CONFIG_MAX_PROCESS_PER_CLUSTER];    /*! number of copie  */
+	xptr_t            pref_tbl[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! reference  process   */
+	spinlock_t        pref_lock;              /*! lock protecting lpid allocation/release */
+    uint32_t          pref_nr;                /*! number of processes owned by cluster    */
+
+    list_entry_t      local_root;             /*! root of list of process in cluster      */
+    spinlock_t        local_lock;             /*! lock protecting access to local list    */
+    uint32_t          local_nr;               /*! number of process in cluster            */
+
+    xlist_entry_t     copies_root[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! roots of lists    */ 
+    remote_spinlock_t copies_lock[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! one lock per list */
+    uint32_t          copies_nr[CONFIG_MAX_PROCESS_PER_CLUSTER];    /*! number of copies  */
 }
 pmgr_t;
@@ -91,61 +90,60 @@
 /*******************************************************************************************
  * This structure defines a cluster manager.
- * It contains both global platform information, and cluster specific resources
- * controled by the local kernel instance.
+ * It contains both global platform informations, and cluster specific ressources
+ * managed by the local kernel instance.
  ******************************************************************************************/
 
 typedef struct cluster_s
 {
+	spinlock_t        kcm_lock;           /*! local, protect creation of KCM allocators   */
+    remote_barrier_t  barrier;            /*! used to synchronize kernel parallel init    */
+    remote_spinlock_t txt0_lock;          /*! global, only cluster_io lock is used        */
 
     // global parameters
-    uint32_t        paddr_width;       /*! numer of bits in physical address              */
-    uint32_t        x_width;           /*! number of bits to code x_size  (can be 0)      */
-    uint32_t        y_width;           /*! number of bits to code y_size  (can be 0)      */
-    uint32_t        x_size;            /*! number of clusters in a row    (can be 1)      */
-    uint32_t        y_size;            /*! number of clusters in a column (can be 1)      */
-    uint32_t        io_cxy;            /*! io cluster identifier                          */
-    uint32_t        sys_clk;           /*! system_clock frequency (in Hertz)              */
-    uint32_t        dqdt_root_level;   /*! index of root node in dqdt_tbl[]               */
-    uint32_t        nb_txt_channels;   /*! number of TXT channels                         */
-    uint32_t        nb_nic_channels;   /*! number of NIC channels                         */
-    uint32_t        nb_ioc_channels;   /*! number of IOC channels                         */
-    uint32_t        nb_fbf_channels;   /*! number of FBF channels                         */
-
-    // number of cores for each cluster in the mesh
-    uint8_t         cluster_info[CONFIG_MAX_CLUSTERS_X][CONFIG_MAX_CLUSTERS_Y];
+
+	uint32_t          paddr_width;        /*! numer of bits in physical address           */
+    uint32_t          x_width;            /*! number of bits to code x_size  (can be 0)   */
+    uint32_t          y_width;            /*! number of bits to code y_size  (can be 0)   */
+	uint32_t          x_size;             /*! number of clusters in a row    (can be 1)   */
+	uint32_t          y_size;             /*! number of clusters in a column (can be 1)   */
+	cxy_t             io_cxy;             /*! io cluster identifier                       */
+    uint32_t          dqdt_root_level;    /*! index of root node in dqdt_tbl[]            */
 
     // local parameters
-    uint32_t        cores_nr;          /*! actual number of cores in cluster              */
-    uint32_t        ram_size;          /*! physical memory size                           */
-    uint32_t        ram_base;          /*! physical memory base (local address)           */
-
-	core_t          core_tbl[CONFIG_MAX_LOCAL_CORES];    /*! embedded cores               */
-
-    list_entry_t    dev_root;          /*! root of list of devices in cluster             */
-
-    // physical memory allocators: one PPM and severa KCM
-    ppm_t           ppm;
-    kcm_t           kcm[CONFIG_PPM_PAGE_ORDER - CONFIG_CACHE_LINE_ORDER];
-
-    // RPC
-    remote_fifo_t   rpc_fifo[CONFIG_MAX_LOCAL_CORES];    /*! one RPC FIFO per core        */
-    uint32_t        rpc_threads[CONFIG_MAX_LOCAL_CORES]; /*! RPC threads per core         */
-
-    // DQDT
-    dqdt_node_t     dqdt_tbl[CONFIG_DQDT_LEVELS_NR];     /*! embedded DQDT nodes          */
-
-    xptr_t          dqdt_root_xp;      /*! extended pointer on DQDT root node             */
-
-    // Local process manager
-    pmgr_t          pmgr;              /*! embedded process manager                       */
-
-    void          * pic_extend;        /*! PIC implementation specific extension          */
-}
+
+	uint32_t          cores_nr;           /*! number of cores in cluster                  */
+    uint32_t          cores_in_kernel;    /*! number of cores currently in kernel mode    */
+
+	core_t            core_tbl[CONFIG_MAX_CORES_PER_CLUSTER];         /*! embedded cores  */
+
+	ppm_t             ppm;                /*! embedded kernel page manager                */
+	khm_t             khm;                /*! embedded kernel heap manager                */
+	kcm_t             kcm;                /*! embedded kernel cache manager (for KCMs)    */
+
+    kcm_t           * kcm_tbl[KMEM_TYPES_NR];         /*! pointers on allocated KCMs      */
+
+    uint32_t          ram_size;           /*! physical memory size                        */
+    uint32_t          ram_base;           /*! physical memory base (local address)        */
+
+	rpc_fifo_t        rpc_fifo;           /*! cluster RPC fifo (shared)                   */
+	list_entry_t      devlist;            /*! root of list of devices in cluster          */
+    struct device_s * icu;                /*! pointer on local XCU device                 */
+
+    int32_t           pages_var;          /*! pages number increment from last update     */
+    int32_t           threads_var;        /*! threads number increment from last update   */
+
+	dqdt_node_t       dqdt_tbl[CONFIG_MAX_DQDT_DEPTH];     /*! embedded DQDT nodes        */
+
+    pmgr_t            pmgr;               /*! embedded process manager                    */
+
+	char              name[CONFIG_SYSFS_NAME_LEN];  
+	sysfs_entry_t     node;
+} 
 cluster_t;
 
 /******************************************************************************************
  * This global variable is allocated in the kernel_init.c file.
- * There is one cluster_manager per cluster, with the same local address,
- * but different content, in all clusters containing a kernel instance.
+ * There is one cluster_manager per cluster, with the same local address, 
+ * but different value, in all clusters containing a kernel instance. 
  *****************************************************************************************/
 
@@ -159,98 +157,62 @@
 
 /******************************************************************************************
- * These two functions initialise the local cluster manager from information found
- * in the local boot-info structure <info> build by the boot-loader.
- * 1) the cluster_info_init() function is called first, to initialize the structural
- *    constants, including the cluster_info[x][y] array.
- *    It cannot use the TXT0 kernel terminal.
- * 2) the cluster_manager_init() function initializes various complex structures:
- *    - the local DQDT nodes,
- *    - the PPM, KHM, and KCM allocators,
- *    - the local core descriptors,
- *    - the local RPC FIFO,
- *    - the process manager.
- *    It can use the TXT0 kernel terminal.
+ * This generic function initialises the local cluster manager from informations found
+ * in the local boot-info structure. It initializes the following local resources:
+ * - the global platform parameters,
+ * - the specific cluster parameters,
+ * - the lock protecting KCM creation,
+ * - the local DQDT nodes,
+ * - the PPM, KHM, and KCM allocators,
+ * - the local core descriptors,
+ * - the local RPC FIFO,
+ * - the process manager.
+ * It does NOT initialise the local device descriptors.
  ******************************************************************************************
  * @ info : pointer on the local boot_info_t structure build by the bootloader.
  *****************************************************************************************/
-void    cluster_info_init( boot_info_t * info );
-error_t cluster_manager_init( boot_info_t * info );
-
-/******************************************************************************************
- * This debug function displays the current values stored in the cluster_info[][] array
- * of a remote cluster identified by the <cxy> argument.
- * It can be called by a thread running in any cluster.
- ******************************************************************************************
- * @ cxy   : remote cluster identifier.
- *****************************************************************************************/
-void cluster_info_display( cxy_t  cxy );
-
-/******************************************************************************************
- * This function access the local cluster_info[][] array and returns true when the 
- * cluster identified by the <cxy> argument is active (contains a kernel instance).
- ******************************************************************************************
- * @ cxy   : checked cluster identifier.
- * @ return true if cluster contains a kernel instance.
- *****************************************************************************************/
-bool_t cluster_is_active( cxy_t  cxy );
+error_t cluster_init( boot_info_t * info );         
+
+/******************************************************************************************
+ * This function cheks the validity of a cluster identifier. TODO useful ??? [AG]
+ ******************************************************************************************
+ * @ cxy    : cluster identifier to be checked.
+ * @ returns true if the identified cluster does not exist.
+ *****************************************************************************************/
+bool_t cluster_is_undefined( cxy_t cxy );
+
+/******************************************************************************************
+ * This function register sysfs information in cluster TODO ???  [AG]
+ *****************************************************************************************/
+void cluster_sysfs_register();
+
+
+/*****************************************************************************************/
+/***************   Process Management Operations   ***************************************/
+/*****************************************************************************************/
+
+/******************************************************************************************
+ * This function returns an extended pointer on the reference process descriptor
+ * from the process PID.  It can be called by any thread running in any cluster,
+ * as it uses a RPC to the owner cluster if the current cluster is not the owner.
+ ******************************************************************************************
+ * @ pid  : process identifier.
+ * @ return extended pointer on the reference process descriptor.
+ *****************************************************************************************/
+xptr_t cluster_get_reference_process_from_pid( pid_t pid );
 
 /****************************************************************************************** 
- * This function selects (pseudo) randomly a valid cluster.
- * It is called by the vfs_cluster_lookup() function to place a new (missing) inode.
- * It is called by the vmm_page_allocate() function to place a distributed vseg page.
- * It is called by the dev_nic_accept() function to place a new server socket.
- ******************************************************************************************
- * @ returns the selected cluster identifier.
- *****************************************************************************************/
-cxy_t cluster_random_select( void );
-
-/******************************************************************************************
- * This function returns an extended pointer on the process descriptor in owner cluster
- * from the process <pid>. This PID can be be different from the calling process PID.
- * It can be called by any thread running in any cluster,
- ******************************************************************************************
- * @ pid  : process identifier.
- * @ return extended pointer on owner process if found / XPTR_NULL if not found.
- *****************************************************************************************/
-xptr_t cluster_get_owner_process_from_pid( pid_t pid );
-
-/******************************************************************************************
- * This function returns an extended pointer on the reference process descriptor
- * from the process <pid>. This PID can be be different from the calling process PID.
- * It can be called by any thread running in any cluster,
- ******************************************************************************************
- * @ pid  : process identifier.
- * @ return extended pointer on reference process if found / XPTR_NULL if not found.
- *****************************************************************************************/
-xptr_t cluster_get_reference_process_from_pid( pid_t pid );
-
-/******************************************************************************************
- * This function returns an extended pointer on the process descriptor copy for the
- * process identified by <pid> in cluster defined by <cxy> argument.
- * This PID can be be different from the calling process PID.
- * It can be called by any thread running in any cluster,
- ******************************************************************************************
- * @ cxy  : target cluster identifier.
- * @ pid  : process identifier.
- * @ return extended pointer on reference process if found / XPTR_NULL if not found.
- *****************************************************************************************/
-xptr_t cluster_get_process_from_pid_in_cxy( cxy_t cxy,
-                                            pid_t pid );
-
-/******************************************************************************************
  * This function allocates a new PID in local cluster, that becomes the process owner.
- * It registers the process descriptor extended pointer in the local processs manager
- * pref_tbl[] array. The process descriptor itself is not modified.
- * This function is called by the process_make_fork() function, 
- * by the process_make_exec() function, and by the process_init_create() function.
- ******************************************************************************************
- * @ process    : pointer on process descriptor.
+ * It register the process descriptor extended pointer in the local processs manager 
+ * pref_tbl[] array. This function is called by the rpc_process_alloc_pid() function for
+ * remote registration, or by the process_init_create() function for local registration.
+ ******************************************************************************************
+ * @ process    : [in]  extended pointer on the process descriptor.
  * @ pid        : [out] allocated PID.
- * @ return 0 if success / return -1 if no PID slot available.
- *****************************************************************************************/
-error_t cluster_pid_alloc( struct process_s * process,
-                           pid_t            * pid );
-
-/******************************************************************************************
+ * @ return 0 if success / return EAGAIN if no PID slot available 
+ *****************************************************************************************/
+error_t cluster_pid_alloc( xptr_t    process_xp,
+                           pid_t   * pid );
+
+/****************************************************************************************** 
  * This function removes a PID from the local process manager pref_tbl[] array.
  * It checks that removed process is owned by the local cluster and the lpid is legal.
@@ -263,4 +225,6 @@
 /******************************************************************************************
  * This function returns a pointer on the local process descriptor from the PID.
+ * It uses the RPC
+ * to create a local process descriptor copy if it does not exist yet.
  ******************************************************************************************
  * @ pid     : searched process identifier.
@@ -274,5 +238,5 @@
  * @ process     : pointer on local process descriptor.
  *****************************************************************************************/
-void cluster_process_local_link( struct process_s * process );
+void cluster_process_local_link( struct process_s * process ); 
 
 /******************************************************************************************
@@ -281,5 +245,5 @@
  * @ process     : pointer on local process descriptor.
  *****************************************************************************************/
-void cluster_process_local_unlink( struct process_s * process );
+void cluster_process_local_unlink( struct process_s * process ); 
 
 /******************************************************************************************
@@ -289,5 +253,5 @@
  * @ process     : pointer on local process descriptor.
  *****************************************************************************************/
-void cluster_process_copies_link( struct process_s * process );
+void cluster_process_copies_link( struct process_s * process ); 
 
 /******************************************************************************************
@@ -297,28 +261,26 @@
  * @ process     : pointer on local process descriptor.
  *****************************************************************************************/
-void cluster_process_copies_unlink( struct process_s * process );
-
-/******************************************************************************************
- * This function displays on the kernel terminal TXT0 all user processes registered
- * in the cluster defined by the <cxy> argument.
- * It can be called by a thread running in any cluster.
- ******************************************************************************************
- * @ cxy   : cluster identifier.
- * @ owned : only owned process if non zero.
- *****************************************************************************************/
-void cluster_processes_display( cxy_t   cxy,
-                                bool_t  owned );
-
-/******************************************************************************************
- * This function selects the core that has the lowest usage in a - possibly remote -
- * cluster identified by the <cxy> argument.
- * It can be called by a thread running in any cluster.
- ******************************************************************************************
- * @ cxy    : target cluster identifier.
- * @ return the selected core local index.
- *****************************************************************************************/
-lid_t cluster_select_local_core( cxy_t  cxy );
-
-             
+void cluster_process_copies_unlink( struct process_s * process ); 
+
+
+
+/*****************************************************************************************/
+/***************   Cores Management Operations   *****************************************/
+/*****************************************************************************************/
+
+/******************************************************************************************
+ * This function increments the "cores_in_kernel" variable in cluster descriptor.
+ *****************************************************************************************/
+void cluster_core_kernel_enter();
+
+/******************************************************************************************
+ * This function decrements the "cores_in_kernel" variable in cluster descriptor.
+ *****************************************************************************************/
+void cluster_core_kernel_exit();
+
+/******************************************************************************************
+ * This function returns the core local index that has the lowest usage in local cluster.
+ *****************************************************************************************/
+lid_t cluster_select_local_core();
+
 #endif	/* _CLUSTER_H_ */
-
