| [1] | 1 | /* | 
|---|
|  | 2 | * cluster.h - Cluster-Manager definition | 
|---|
| [19] | 3 | * | 
|---|
| [1] | 4 | * authors  Ghassan Almaless (2008,2009,2010,2011,2012) | 
|---|
|  | 5 | *          Mohamed Lamine Karaoui (2015) | 
|---|
| [437] | 6 | *          Alain Greiner (2016,2017,2018) | 
|---|
| [1] | 7 | * | 
|---|
|  | 8 | * Copyright (c) UPMC Sorbonne Universites | 
|---|
|  | 9 | * | 
|---|
|  | 10 | * This file is part of ALMOS-MKH. | 
|---|
|  | 11 | * | 
|---|
|  | 12 | * ALMOS-MKH is free software; you can redistribute it and/or modify it | 
|---|
|  | 13 | * under the terms of the GNU General Public License as published by | 
|---|
|  | 14 | * the Free Software Foundation; version 2.0 of the License. | 
|---|
|  | 15 | * | 
|---|
|  | 16 | * ALMOS-MKH is distributed in the hope that it will be useful, but | 
|---|
|  | 17 | * WITHOUT ANY WARRANTY; without even the implied warranty of | 
|---|
|  | 18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU | 
|---|
|  | 19 | * General Public License for more details. | 
|---|
|  | 20 | * | 
|---|
|  | 21 | * You should have received a copy of the GNU General Public License | 
|---|
|  | 22 | * along with ALMOS-MKH; if not, write to the Free Software Foundation, | 
|---|
|  | 23 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA | 
|---|
|  | 24 | */ | 
|---|
|  | 25 |  | 
|---|
|  | 26 | #ifndef _CLUSTER_H_ | 
|---|
|  | 27 | #define _CLUSTER_H_ | 
|---|
|  | 28 |  | 
|---|
| [14] | 29 | #include <kernel_config.h> | 
|---|
| [457] | 30 | #include <hal_kernel_types.h> | 
|---|
| [1] | 31 | #include <bits.h> | 
|---|
| [564] | 32 | #include <queuelock.h> | 
|---|
|  | 33 | #include <remote_queuelock.h> | 
|---|
| [1] | 34 | #include <list.h> | 
|---|
|  | 35 | #include <xlist.h> | 
|---|
|  | 36 | #include <dqdt.h> | 
|---|
|  | 37 | #include <kmem.h> | 
|---|
|  | 38 | #include <hal_atomic.h> | 
|---|
|  | 39 | #include <ppm.h> | 
|---|
|  | 40 | #include <kcm.h> | 
|---|
|  | 41 | #include <khm.h> | 
|---|
|  | 42 | #include <rpc.h> | 
|---|
|  | 43 | #include <core.h> | 
|---|
|  | 44 | #include <process.h> | 
|---|
|  | 45 |  | 
|---|
|  | 46 | /**** Forward declarations  ****/ | 
|---|
|  | 47 |  | 
|---|
|  | 48 | struct core_s; | 
|---|
|  | 49 | struct process_s; | 
|---|
|  | 50 |  | 
|---|
|  | 51 |  | 
|---|
|  | 52 | /******************************************************************************************* | 
|---|
|  | 53 | * This structure defines the process manager, that is part of the cluster manager. | 
|---|
|  | 54 | * For any process P, the process descriptor is replicated in all clusters containing | 
|---|
| [19] | 55 | * at least one thread of process P, but only the "reference" cluster descriptor contains | 
|---|
| [23] | 56 | * the reference (complete) GPT, VSL, and FDT structures. | 
|---|
|  | 57 | * The "owner" cluster K is in charge to allocate a lpid (local process index), | 
|---|
|  | 58 | * to the owned processes, and to register the "reference" cluster for these processes. | 
|---|
| [1] | 59 | * | 
|---|
| [19] | 60 | * Warning : the "owner" cluster, and the "reference" cluster can be different clusters. | 
|---|
| [1] | 61 | * | 
|---|
| [443] | 62 | * The process manager of a cluster K maintains three sets of process descriptors: | 
|---|
|  | 63 | * | 
|---|
|  | 64 | * 1) pref_tbl[] is an array indexed by lpid. There is one entry per owned process. | 
|---|
| [1] | 65 | *    Each entry contains an extended pointer on the reference process descriptor. | 
|---|
| [23] | 66 | * | 
|---|
| [443] | 67 | * 2) The local_root is the root of the local list of all process descriptors in cluster K. | 
|---|
| [1] | 68 | *    A process descriptor P is present in K, as soon as P has a thread in cluster K. | 
|---|
| [564] | 69 | *    We use an xlist, because this list can be traversed by remote threads. | 
|---|
| [23] | 70 | * | 
|---|
|  | 71 | * 3) The copies_root[] array is indexed by lpid. There is one entry per owned process, | 
|---|
|  | 72 | *    and each each entry contains the root of the xlist of copies for this process. | 
|---|
| [564] | 73 | $    We use an xlist, because process copies are distributed in all clusters. | 
|---|
| [1] | 74 | ******************************************************************************************/ | 
|---|
|  | 75 |  | 
|---|
|  | 76 | typedef struct process_manager_s | 
|---|
|  | 77 | { | 
|---|
| [564] | 78 | xptr_t             pref_tbl[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! owned  processes    */ | 
|---|
|  | 79 | queuelock_t        pref_lock;              /*! lock protecting pref_tbl               */ | 
|---|
|  | 80 | uint32_t           pref_nr;                /*! number of processes owned by cluster   */ | 
|---|
| [1] | 81 |  | 
|---|
| [564] | 82 | xlist_entry_t      local_root;            /*! root of list of process in cluster      */ | 
|---|
|  | 83 | remote_queuelock_t local_lock;            /*! lock protecting local list              */ | 
|---|
|  | 84 | uint32_t           local_nr;              /*! number of process in cluster            */ | 
|---|
| [1] | 85 |  | 
|---|
| [564] | 86 | xlist_entry_t      copies_root[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! roots of lists   */ | 
|---|
|  | 87 | remote_queuelock_t copies_lock[CONFIG_MAX_PROCESS_PER_CLUSTER];  /*! one  per list    */ | 
|---|
|  | 88 | uint32_t           copies_nr[CONFIG_MAX_PROCESS_PER_CLUSTER];    /*! number of copie  */ | 
|---|
| [1] | 89 | } | 
|---|
|  | 90 | pmgr_t; | 
|---|
|  | 91 |  | 
|---|
|  | 92 | /******************************************************************************************* | 
|---|
|  | 93 | * This structure defines a cluster manager. | 
|---|
| [19] | 94 | * It contains both global platform information, and cluster specific resources | 
|---|
| [279] | 95 | * controled by the local kernel instance. | 
|---|
| [1] | 96 | ******************************************************************************************/ | 
|---|
|  | 97 |  | 
|---|
|  | 98 | typedef struct cluster_s | 
|---|
|  | 99 | { | 
|---|
|  | 100 |  | 
|---|
|  | 101 | // global parameters | 
|---|
| [564] | 102 | uint32_t        paddr_width;       /*! numer of bits in physical address              */ | 
|---|
| [440] | 103 | uint32_t        x_width;           /*! number of bits to code x_size  (can be 0)      */ | 
|---|
|  | 104 | uint32_t        y_width;           /*! number of bits to code y_size  (can be 0)      */ | 
|---|
| [564] | 105 | uint32_t        x_size;            /*! number of clusters in a row    (can be 1)      */ | 
|---|
|  | 106 | uint32_t        y_size;            /*! number of clusters in a column (can be 1)      */ | 
|---|
|  | 107 | cxy_t           io_cxy;            /*! io cluster identifier                          */ | 
|---|
| [440] | 108 | uint32_t        dqdt_root_level;   /*! index of root node in dqdt_tbl[]               */ | 
|---|
|  | 109 | uint32_t        nb_txt_channels;   /*! number of TXT channels                         */ | 
|---|
|  | 110 | uint32_t        nb_nic_channels;   /*! number of NIC channels                         */ | 
|---|
|  | 111 | uint32_t        nb_ioc_channels;   /*! number of IOC channels                         */ | 
|---|
|  | 112 | uint32_t        nb_fbf_channels;   /*! number of FBF channels                         */ | 
|---|
| [1] | 113 |  | 
|---|
| [564] | 114 | char            cluster_info[CONFIG_MAX_CLUSTERS_X][CONFIG_MAX_CLUSTERS_Y]; | 
|---|
|  | 115 |  | 
|---|
| [1] | 116 | // local parameters | 
|---|
| [564] | 117 | uint32_t        cores_nr;          /*! actual number of cores in cluster              */ | 
|---|
| [440] | 118 | uint32_t        ram_size;          /*! physical memory size                           */ | 
|---|
|  | 119 | uint32_t        ram_base;          /*! physical memory base (local address)           */ | 
|---|
| [279] | 120 |  | 
|---|
| [440] | 121 | core_t          core_tbl[CONFIG_MAX_LOCAL_CORES];    /*! embedded cores               */ | 
|---|
| [1] | 122 |  | 
|---|
| [564] | 123 | list_entry_t    dev_root;          /*! root of list of devices in cluster             */ | 
|---|
| [279] | 124 |  | 
|---|
|  | 125 | // memory allocators | 
|---|
| [564] | 126 | ppm_t           ppm;               /*! embedded kernel page manager                   */ | 
|---|
|  | 127 | khm_t           khm;               /*! embedded kernel heap manager                   */ | 
|---|
|  | 128 | kcm_t           kcm;               /*! embedded kernel KCMs manager                   */ | 
|---|
| [428] | 129 |  | 
|---|
| [440] | 130 | kcm_t         * kcm_tbl[KMEM_TYPES_NR];              /*! pointers on allocated KCMs   */ | 
|---|
| [564] | 131 | busylock_t      kcm_lock;                            /*! protect kcm_tbl[] updates    */ | 
|---|
| [1] | 132 |  | 
|---|
| [279] | 133 | // RPC | 
|---|
| [564] | 134 | remote_fifo_t   rpc_fifo[CONFIG_MAX_LOCAL_CORES];    /*! one RPC FIFO per core        */ | 
|---|
| [440] | 135 | uint32_t        rpc_threads[CONFIG_MAX_LOCAL_CORES]; /*! RPC threads per core         */ | 
|---|
| [1] | 136 |  | 
|---|
| [279] | 137 | // DQDT | 
|---|
| [564] | 138 | dqdt_node_t     dqdt_tbl[CONFIG_DQDT_LEVELS_NR];     /*! embedded DQDT nodes          */ | 
|---|
| [1] | 139 |  | 
|---|
| [582] | 140 | xptr_t          dqdt_root_xp;      /*! extended pointer on DQDT root node             */ | 
|---|
|  | 141 |  | 
|---|
| [279] | 142 | // Local process manager | 
|---|
| [440] | 143 | pmgr_t          pmgr;              /*! embedded process manager                       */ | 
|---|
| [188] | 144 |  | 
|---|
| [440] | 145 | void          * pic_extend;        /*! PIC implementation specific extension          */ | 
|---|
| [19] | 146 | } | 
|---|
| [1] | 147 | cluster_t; | 
|---|
|  | 148 |  | 
|---|
|  | 149 | /****************************************************************************************** | 
|---|
|  | 150 | * This global variable is allocated in the kernel_init.c file. | 
|---|
| [19] | 151 | * There is one cluster_manager per cluster, with the same local address, | 
|---|
|  | 152 | * but different content, in all clusters containing a kernel instance. | 
|---|
| [1] | 153 | *****************************************************************************************/ | 
|---|
|  | 154 |  | 
|---|
|  | 155 | extern cluster_t cluster_manager; | 
|---|
|  | 156 |  | 
|---|
|  | 157 | /****************************************************************************************** | 
|---|
|  | 158 | * This macro returns a local pointer on the local cluster manager. | 
|---|
|  | 159 | *****************************************************************************************/ | 
|---|
|  | 160 |  | 
|---|
|  | 161 | #define LOCAL_CLUSTER    (&cluster_manager) | 
|---|
|  | 162 |  | 
|---|
|  | 163 | /****************************************************************************************** | 
|---|
| [564] | 164 | * These two functions initialise the local cluster manager from information found | 
|---|
|  | 165 | * in the local boot-info structure <info> build by the boot-loader. | 
|---|
|  | 166 | * 1) the cluster_info_init() function is called first, to initialize the structural | 
|---|
|  | 167 | *    constants, and cannot use the TXT0 kernel terminal. | 
|---|
|  | 168 | * 2) the cluster_manager_init() function initialize various complex structures: | 
|---|
|  | 169 | *    - the local DQDT nodes, | 
|---|
|  | 170 | *    - the PPM, KHM, and KCM allocators, | 
|---|
|  | 171 | *    - the local core descriptors, | 
|---|
|  | 172 | *    - the local RPC FIFO, | 
|---|
|  | 173 | *    - the process manager. | 
|---|
|  | 174 | *    It does NOT initialise the local device descriptors. | 
|---|
|  | 175 | *    It can use the TXT0 kernel terminal. | 
|---|
| [1] | 176 | ****************************************************************************************** | 
|---|
|  | 177 | * @ info : pointer on the local boot_info_t structure build by the bootloader. | 
|---|
|  | 178 | *****************************************************************************************/ | 
|---|
| [564] | 179 | void    cluster_info_init( boot_info_t * info ); | 
|---|
|  | 180 | error_t cluster_manager_init( boot_info_t * info ); | 
|---|
| [1] | 181 |  | 
|---|
|  | 182 | /****************************************************************************************** | 
|---|
| [407] | 183 | * This function checks the validity of a cluster identifier. | 
|---|
| [1] | 184 | ****************************************************************************************** | 
|---|
|  | 185 | * @ cxy    : cluster identifier to be checked. | 
|---|
|  | 186 | * @ returns true if the identified cluster does not exist. | 
|---|
|  | 187 | *****************************************************************************************/ | 
|---|
|  | 188 | bool_t cluster_is_undefined( cxy_t cxy ); | 
|---|
|  | 189 |  | 
|---|
| [564] | 190 | /****************************************************************************************** | 
|---|
|  | 191 | * This function uses the local cluster_info[][] array in cluster descriptor, | 
|---|
|  | 192 | * and returns true when the cluster identified by the <cxy> argument is active. | 
|---|
|  | 193 | ****************************************************************************************** | 
|---|
|  | 194 | * @ cxy   : cluster identifier. | 
|---|
|  | 195 | * @ return true if cluster contains a kernel instance. | 
|---|
|  | 196 | *****************************************************************************************/ | 
|---|
|  | 197 | bool_t cluster_is_active( cxy_t  cxy ); | 
|---|
| [1] | 198 |  | 
|---|
| [564] | 199 | /****************************************************************************************** | 
|---|
|  | 200 | * This function (pseudo) randomly selects a valid cluster. | 
|---|
|  | 201 | * It is called by the vfs_cluster_lookup() function to place a new (missing) inode. | 
|---|
|  | 202 | * It is called by the vmm_page_allocate() function to place a distributed vseg page. | 
|---|
|  | 203 | ****************************************************************************************** | 
|---|
|  | 204 | * @ returns the selected cluster identifier. | 
|---|
|  | 205 | *****************************************************************************************/ | 
|---|
|  | 206 | cxy_t cluster_random_select( void ); | 
|---|
| [1] | 207 |  | 
|---|
|  | 208 | /****************************************************************************************** | 
|---|
| [433] | 209 | * This function returns an extended pointer on the process descriptor in owner cluster | 
|---|
| [443] | 210 | * from the process <pid>. This PID can be be different from the calling process PID. | 
|---|
| [433] | 211 | * It can be called by any thread running in any cluster, | 
|---|
|  | 212 | ****************************************************************************************** | 
|---|
|  | 213 | * @ pid  : process identifier. | 
|---|
|  | 214 | * @ return extended pointer on owner process if found / XPTR_NULL if not found. | 
|---|
|  | 215 | *****************************************************************************************/ | 
|---|
|  | 216 | xptr_t cluster_get_owner_process_from_pid( pid_t pid ); | 
|---|
|  | 217 |  | 
|---|
|  | 218 | /****************************************************************************************** | 
|---|
| [1] | 219 | * This function returns an extended pointer on the reference process descriptor | 
|---|
| [443] | 220 | * from the process <pid>. This PID can be be different from the calling process PID. | 
|---|
| [23] | 221 | * It can be called by any thread running in any cluster, | 
|---|
| [1] | 222 | ****************************************************************************************** | 
|---|
|  | 223 | * @ pid  : process identifier. | 
|---|
| [433] | 224 | * @ return extended pointer on reference process if found / XPTR_NULL if not found. | 
|---|
| [1] | 225 | *****************************************************************************************/ | 
|---|
|  | 226 | xptr_t cluster_get_reference_process_from_pid( pid_t pid ); | 
|---|
|  | 227 |  | 
|---|
| [19] | 228 | /****************************************************************************************** | 
|---|
| [443] | 229 | * This function returns an extended pointer on the process descriptor copy for the | 
|---|
|  | 230 | * process identified by <pid> in cluster defined by <cxy> argument. | 
|---|
|  | 231 | * This PID can be be different from the calling process PID. | 
|---|
|  | 232 | * It can be called by any thread running in any cluster, | 
|---|
|  | 233 | ****************************************************************************************** | 
|---|
|  | 234 | * @ cxy  : target cluster identifier. | 
|---|
|  | 235 | * @ pid  : process identifier. | 
|---|
|  | 236 | * @ return extended pointer on reference process if found / XPTR_NULL if not found. | 
|---|
|  | 237 | *****************************************************************************************/ | 
|---|
|  | 238 | xptr_t cluster_get_process_from_pid_in_cxy( cxy_t cxy, | 
|---|
|  | 239 | pid_t pid ); | 
|---|
|  | 240 |  | 
|---|
|  | 241 | /****************************************************************************************** | 
|---|
| [1] | 242 | * This function allocates a new PID in local cluster, that becomes the process owner. | 
|---|
| [19] | 243 | * It registers the process descriptor extended pointer in the local processs manager | 
|---|
| [428] | 244 | * pref_tbl[] array. The process descriptor itself is not modified. | 
|---|
|  | 245 | * This function is called by the process_make_fork() function, | 
|---|
| [416] | 246 | * by the process_make_exec() function, and by the process_init_create() function. | 
|---|
| [1] | 247 | ****************************************************************************************** | 
|---|
| [416] | 248 | * @ process    : pointer on process descriptor. | 
|---|
| [1] | 249 | * @ pid        : [out] allocated PID. | 
|---|
| [416] | 250 | * @ return 0 if success / return -1 if no PID slot available. | 
|---|
| [1] | 251 | *****************************************************************************************/ | 
|---|
| [416] | 252 | error_t cluster_pid_alloc( struct process_s * process, | 
|---|
|  | 253 | pid_t            * pid ); | 
|---|
| [1] | 254 |  | 
|---|
| [19] | 255 | /****************************************************************************************** | 
|---|
| [1] | 256 | * This function removes a PID from the local process manager pref_tbl[] array. | 
|---|
|  | 257 | * It checks that removed process is owned by the local cluster and the lpid is legal. | 
|---|
|  | 258 | * No memory is released by this function. | 
|---|
|  | 259 | ****************************************************************************************** | 
|---|
|  | 260 | * @ pid        : allocated PID. | 
|---|
|  | 261 | *****************************************************************************************/ | 
|---|
|  | 262 | void cluster_pid_release( pid_t  pid ); | 
|---|
|  | 263 |  | 
|---|
|  | 264 | /****************************************************************************************** | 
|---|
|  | 265 | * This function returns a pointer on the local process descriptor from the PID. | 
|---|
|  | 266 | ****************************************************************************************** | 
|---|
|  | 267 | * @ pid     : searched process identifier. | 
|---|
|  | 268 | * @ returns process descriptor pointer if found / returns NULL if not found. | 
|---|
|  | 269 | *****************************************************************************************/ | 
|---|
|  | 270 | struct process_s * cluster_get_local_process_from_pid( pid_t pid ); | 
|---|
|  | 271 |  | 
|---|
|  | 272 | /****************************************************************************************** | 
|---|
|  | 273 | * This function registers a local process descriptor in the process manager local_list. | 
|---|
|  | 274 | ****************************************************************************************** | 
|---|
|  | 275 | * @ process     : pointer on local process descriptor. | 
|---|
|  | 276 | *****************************************************************************************/ | 
|---|
| [19] | 277 | void cluster_process_local_link( struct process_s * process ); | 
|---|
| [1] | 278 |  | 
|---|
|  | 279 | /****************************************************************************************** | 
|---|
|  | 280 | * This function removes a local process descriptor from the process manager local_list. | 
|---|
|  | 281 | ****************************************************************************************** | 
|---|
|  | 282 | * @ process     : pointer on local process descriptor. | 
|---|
|  | 283 | *****************************************************************************************/ | 
|---|
| [19] | 284 | void cluster_process_local_unlink( struct process_s * process ); | 
|---|
| [1] | 285 |  | 
|---|
|  | 286 | /****************************************************************************************** | 
|---|
|  | 287 | * This function registers a local process descriptor in the owner process manager | 
|---|
|  | 288 | * copies_list, that can be in a remote cluster. | 
|---|
|  | 289 | ****************************************************************************************** | 
|---|
|  | 290 | * @ process     : pointer on local process descriptor. | 
|---|
|  | 291 | *****************************************************************************************/ | 
|---|
| [19] | 292 | void cluster_process_copies_link( struct process_s * process ); | 
|---|
| [1] | 293 |  | 
|---|
|  | 294 | /****************************************************************************************** | 
|---|
|  | 295 | * This function removes a local process descriptor from the owner process manager | 
|---|
|  | 296 | * copies_list, that can be in a remote cluster. | 
|---|
|  | 297 | ****************************************************************************************** | 
|---|
|  | 298 | * @ process     : pointer on local process descriptor. | 
|---|
|  | 299 | *****************************************************************************************/ | 
|---|
| [19] | 300 | void cluster_process_copies_unlink( struct process_s * process ); | 
|---|
| [1] | 301 |  | 
|---|
| [564] | 302 | /****************************************************************************************** | 
|---|
| [428] | 303 | * This function displays on the kernel terminal TXT0 all user processes registered | 
|---|
|  | 304 | * in the cluster defined by the <cxy> argument. | 
|---|
|  | 305 | * It can be called by a thread running in any cluster, because is use remote accesses | 
|---|
|  | 306 | * to scan the xlist of registered processes. | 
|---|
| [564] | 307 | ****************************************************************************************** | 
|---|
| [428] | 308 | * @ cxy   : cluster identifier. | 
|---|
| [583] | 309 | * @ owned : only owned process if non zero. | 
|---|
| [564] | 310 | *****************************************************************************************/ | 
|---|
| [583] | 311 | void cluster_processes_display( cxy_t   cxy, | 
|---|
|  | 312 | bool_t  owned ); | 
|---|
| [1] | 313 |  | 
|---|
|  | 314 | /****************************************************************************************** | 
|---|
| [564] | 315 | * This function uses the local boot_inforeturns the core local index that has the lowest usage in local cluster. | 
|---|
| [1] | 316 | *****************************************************************************************/ | 
|---|
| [485] | 317 | lid_t cluster_select_local_core( void ); | 
|---|
| [1] | 318 |  | 
|---|
| [564] | 319 |  | 
|---|
| [1] | 320 | #endif  /* _CLUSTER_H_ */ | 
|---|
| [564] | 321 |  | 
|---|