[445] | 1 | /* |
---|
| 2 | * almosmkh.h - User level ALMOS-MKH specific library definition. |
---|
| 3 | * |
---|
[685] | 4 | * Author Alain Greiner (2016,2017,2018,2019,2020) |
---|
[445] | 5 | * |
---|
| 6 | * Copyright (c) UPMC Sorbonne Universites |
---|
| 7 | * |
---|
| 8 | * This file is part of ALMOS-MKH. |
---|
| 9 | * |
---|
| 10 | * ALMOS-MKH is free software; you can redistribute it and/or modify it |
---|
| 11 | * under the terms of the GNU General Public License as published by |
---|
| 12 | * the Free Software Foundation; version 2.0 of the License. |
---|
| 13 | * |
---|
| 14 | * ALMOS-MKH is distributed in the hope that it will be useful, but |
---|
| 15 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
---|
| 17 | * General Public License for more details. |
---|
| 18 | * |
---|
| 19 | * You should have received a copy of the GNU General Public License |
---|
| 20 | * along with ALMOS-MKH; if not, write to the Free Software Foundation, |
---|
| 21 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA |
---|
| 22 | */ |
---|
| 23 | |
---|
[444] | 24 | #ifndef _LIBALMOSMKH_H_ |
---|
| 25 | #define _LIBALMOSMKH_H_ |
---|
| 26 | |
---|
[445] | 27 | /*************************************************************************************** |
---|
[685] | 28 | * This file defines an user level, ALMOS-MKH specific library, containing various |
---|
| 29 | * non standard functions and system calls: |
---|
| 30 | * |
---|
| 31 | * - non standard get_uint32() and get_string() functions. |
---|
[445] | 32 | * - non standard system calls. |
---|
[685] | 33 | * - non standard debug functions. |
---|
| 34 | * - non standard remote_malloc() functions. |
---|
| 35 | * - non standard parallel_pthread_create() function. |
---|
| 36 | * - non standard FBF access syscalls. |
---|
[445] | 37 | **************************************************************************************/ |
---|
| 38 | |
---|
[444] | 39 | #include <pthread.h> |
---|
[445] | 40 | #include <shared_almos.h> |
---|
[444] | 41 | |
---|
[657] | 42 | |
---|
[685] | 43 | /****************** Non standard (ALMOS_MKH specific) functions **********************/ |
---|
| 44 | |
---|
| 45 | |
---|
| 46 | /*************************************************************************************** |
---|
| 47 | * This syscall returns a 32 bits integer captures from the STDIN terminal. |
---|
| 48 | * The value is copied in buffer pointed by the <buf> argument |
---|
| 49 | * Both decimal numbers and hexadecimal numbers (prefixed by 0x) are supported. |
---|
| 50 | *************************************************************************************** |
---|
| 51 | * @ buffer : pointer on buffer. |
---|
| 52 | * @ returns 0 if success / returns -1 if failure. |
---|
| 53 | **************************************************************************************/ |
---|
| 54 | void get_uint32( unsigned int * buffer ); |
---|
| 55 | |
---|
| 56 | |
---|
| 57 | /*************************************************************************************** |
---|
| 58 | * These syscalls return a NUL terminated string captured from the STDIN terminal. |
---|
| 59 | * The characters are copied in buffer pointed by the <string> argument. |
---|
| 60 | * If the number of captured characters exceeds <maxlen>, all characters are discarded, |
---|
| 61 | * and removed from the <string> buffer, and a new string capture start. |
---|
| 62 | * - All printable characters 'c' such as (0x20 <= c) and (c < 0x7f) are echoed |
---|
| 63 | * and copied in the target buffer. |
---|
| 64 | * - The LF (0x0A) or CR (0x13) characters are interpreted as "end of string". |
---|
| 65 | * It is echoed, and replaced by the NUL (0x00) character in the target buffer. |
---|
| 66 | * - The DEL (0x7F), BS (0x08) control characters are taken into account, and can be |
---|
| 67 | * used to correct both the buffer content and the echoed string. |
---|
| 68 | * - All other non printable characters are discarded |
---|
| 69 | *************************************************************************************** |
---|
| 70 | * @ string : pointer on the user space string buffer. |
---|
| 71 | * @ maxlen : max number of bytes in string buffer, including the terminating NUL. |
---|
| 72 | * @ returns string length (including NUL) if success / returns 0 if failure. |
---|
| 73 | **************************************************************************************/ |
---|
| 74 | int get_string( char * string, |
---|
| 75 | int maxlen ); |
---|
| 76 | |
---|
| 77 | |
---|
| 78 | |
---|
[445] | 79 | /****************** Non standard (ALMOS_MKH specific) system calls ********************/ |
---|
[444] | 80 | |
---|
| 81 | |
---|
[445] | 82 | /*************************************************************************************** |
---|
[589] | 83 | * This syscall gives the process identified by the <pid> argument the exclusive |
---|
| 84 | * ownership of its TXT terminal. |
---|
[445] | 85 | *************************************************************************************** |
---|
[444] | 86 | * @ pid : process identifier. |
---|
| 87 | * @ returns O if success / returns -1 if process not found. |
---|
[445] | 88 | **************************************************************************************/ |
---|
[444] | 89 | int fg( unsigned int pid ); |
---|
| 90 | |
---|
| 91 | /*************************************************************************************** |
---|
[589] | 92 | * This syscall stores in the buffer identified by the <owner> argument a non zero |
---|
[457] | 93 | * value when the process identified by the <pid> argument is currently the exclusive |
---|
| 94 | * owner of its TXT terminal. |
---|
| 95 | *************************************************************************************** |
---|
| 96 | * @ pid : [in] process identifier. |
---|
| 97 | * @ owner : [out] pointer on buffer to store the |
---|
| 98 | * @ returns O if success / returns -1 if process not found. |
---|
| 99 | **************************************************************************************/ |
---|
| 100 | int is_fg( unsigned int pid, |
---|
| 101 | unsigned int * owner ); |
---|
| 102 | |
---|
| 103 | /*************************************************************************************** |
---|
[661] | 104 | * This syscall returns the hardware platform parameters in the hard_config_t pointed |
---|
| 105 | * by the <config> argument. This structure is defined in the shared_almos.h file. |
---|
| 106 | * The available configuration parameters are : |
---|
| 107 | * - cores : x_size / y_size / ncores |
---|
| 108 | * - peripherals : nb_txt_channels / nb_nic_channels / nb_ioc_channels / nb_fbf_channels |
---|
[444] | 109 | *************************************************************************************** |
---|
[661] | 110 | * @ config : [out] pointer on hard_config_t structure in user space. |
---|
| 111 | * @ return 0 if success / return -1 if illegal argument. |
---|
[444] | 112 | **************************************************************************************/ |
---|
[661] | 113 | int get_config( struct hard_config_s * config ); |
---|
[444] | 114 | |
---|
| 115 | /*************************************************************************************** |
---|
[637] | 116 | * This syscall returns the cluster identifier and the local index |
---|
| 117 | * for the calling core. |
---|
[444] | 118 | *************************************************************************************** |
---|
| 119 | * @ cxy : [out] cluster identifier. |
---|
| 120 | * @ lid : [out] core local index in cluster. |
---|
| 121 | * @ return always 0. |
---|
| 122 | **************************************************************************************/ |
---|
[637] | 123 | int get_core_id( unsigned int * cxy, |
---|
| 124 | unsigned int * lid ); |
---|
[444] | 125 | |
---|
[637] | 126 | /*************************************************************************************** |
---|
| 127 | * This syscall returns the number of cores in a given cluster. |
---|
| 128 | *************************************************************************************** |
---|
| 129 | * @ cxy : [in] target cluster identifier. |
---|
| 130 | * @ ncores : [out] number of cores in target cluster. |
---|
| 131 | * @ return always 0. |
---|
| 132 | **************************************************************************************/ |
---|
| 133 | int get_nb_cores( unsigned int cxy, |
---|
| 134 | unsigned int * ncores ); |
---|
| 135 | |
---|
| 136 | /*************************************************************************************** |
---|
| 137 | * This syscall uses the DQDT to search, in a macro-cluster specified by the |
---|
| 138 | * <cxy_base> and <level> arguments arguments, the core with the lowest load. |
---|
| 139 | * it writes in the <cxy> and <lid> buffers the selected core cluster identifier |
---|
| 140 | * and the local core index. |
---|
| 141 | *************************************************************************************** |
---|
| 142 | * @ cxy_base : [in] any cluster identifier in macro-cluster.in clusters array. |
---|
| 143 | * @ level : [in] macro-cluster level in [1,2,3,4,5]. |
---|
| 144 | * @ cxy : [out] selected core cluster identifier. |
---|
[650] | 145 | * @ lid : [out] selected core local index. |
---|
[637] | 146 | * @ return 0 if success / 1 if no core in macro-cluster / -1 if illegal arguments. |
---|
| 147 | **************************************************************************************/ |
---|
| 148 | int get_best_core( unsigned int cxy_base, |
---|
| 149 | unsigned int level, |
---|
| 150 | unsigned int * cxy, |
---|
| 151 | unsigned int * lid ); |
---|
| 152 | |
---|
[685] | 153 | /*************************************************************************************** |
---|
| 154 | * This syscall returns in the user buffer defined by the <u_buf> and <size> arguments |
---|
| 155 | * a string containing the description of the processes registered in cluster specified |
---|
| 156 | * by the <cxy> argument (one line per process in the NUL terminated string). |
---|
| 157 | * Only the owned processes are returned when the <owned> argument is non zero. |
---|
| 158 | * It can be called by any thread running in any cluster. |
---|
| 159 | *************************************************************************************** |
---|
| 160 | * @ cxy : [in] target cluster identifier. |
---|
| 161 | * @ owned : [in] only owned processes if non zero. |
---|
| 162 | * @ return 0 if success / return -1 if illegal argument. |
---|
| 163 | **************************************************************************************/ |
---|
| 164 | int get_processes( unsigned int cxy, |
---|
| 165 | unsigned int owned, |
---|
| 166 | char * u_buf, |
---|
| 167 | unsigned int size ); |
---|
| 168 | |
---|
[444] | 169 | /*************************************************************************************** |
---|
[637] | 170 | * This function returns the value contained in the calling core cycles counter, |
---|
[444] | 171 | * taking into account a possible overflow on 32 bits architectures. |
---|
| 172 | *************************************************************************************** |
---|
| 173 | * @ cycle : [out] current cycle value. |
---|
| 174 | * @ return always 0. |
---|
| 175 | **************************************************************************************/ |
---|
| 176 | int get_cycle( unsigned long long * cycle ); |
---|
| 177 | |
---|
[685] | 178 | /*************************************************************************************** |
---|
| 179 | * This syscall copies in the user structure defined by the <info> argument the values |
---|
| 180 | * registered in the calling thread "thread_info_t" kernel structure. |
---|
| 181 | ****************************************************************************************** |
---|
| 182 | * @ info : [out] pointer on thread_info_t structure in user space. |
---|
| 183 | * @ return 0 if success / return -1 if illegal argument. |
---|
| 184 | *****************************************************************************************/ |
---|
| 185 | int get_thread_info( thread_info_t * info ); |
---|
| 186 | |
---|
[589] | 187 | /*************************************************************************************** |
---|
| 188 | * This syscall allows the calling thread to specify the target cluster for |
---|
| 189 | * a subsequent fork(). It must be called for each fork(). |
---|
| 190 | *************************************************************************************** |
---|
| 191 | * @ cxy : [in] target cluster identifier. |
---|
| 192 | * @ return 0 if success / returns -1 if illegal cxy argument. |
---|
| 193 | **************************************************************************************/ |
---|
| 194 | int place_fork( unsigned int cxy ); |
---|
| 195 | |
---|
[444] | 196 | /*************************************************************************************** |
---|
[589] | 197 | * This syscall implements the operations related to User Thread Local Storage. |
---|
[445] | 198 | *************************************************************************************** |
---|
| 199 | * @ operation : UTLS operation type as defined in "shared_sycalls.h" file. |
---|
| 200 | * @ value : argument value for the UTLS_SET operation. |
---|
| 201 | * @ return value for the UTLS_GET and UTLS_GET_ERRNO / return -1 if failure. |
---|
| 202 | **************************************************************************************/ |
---|
| 203 | int utls( unsigned int operation, |
---|
| 204 | unsigned int value ); |
---|
| 205 | |
---|
| 206 | |
---|
[678] | 207 | |
---|
[445] | 208 | /***************** Non standard (ALMOS-MKH specific) debug functions ******************/ |
---|
| 209 | |
---|
[641] | 210 | /*************************************************************************************** |
---|
[589] | 211 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[685] | 212 | * an user defined message specified by the <string> argument, plus the current cycle. |
---|
[444] | 213 | *************************************************************************************** |
---|
| 214 | * @ string : [in] user defined message. |
---|
| 215 | **************************************************************************************/ |
---|
| 216 | void display_string( char * string ); |
---|
| 217 | |
---|
| 218 | /*************************************************************************************** |
---|
| 219 | * This debug function displays on the kernel terminal TXT0 |
---|
[610] | 220 | * the state of the VMM for the process <pid> in cluster <cxy>. |
---|
[444] | 221 | * It can be called by any thread running in any cluster. |
---|
| 222 | *************************************************************************************** |
---|
[625] | 223 | * @ cxy : [in] target cluster identifier. |
---|
[444] | 224 | * @ pid : [in] process identifier. |
---|
[640] | 225 | * @ mapping : [in] detailed mapping if non zero. |
---|
[444] | 226 | * @ return 0 if success / return -1 if illegal argument. |
---|
| 227 | **************************************************************************************/ |
---|
[640] | 228 | int display_vmm( unsigned int cxy, |
---|
| 229 | unsigned int pid, |
---|
| 230 | unsigned int mapping ); |
---|
[444] | 231 | |
---|
| 232 | /*************************************************************************************** |
---|
[589] | 233 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[444] | 234 | * the state of the core scheduler identified by the <cxy> and <lid> arguments. |
---|
| 235 | * It can be called by any thread running in any cluster. |
---|
| 236 | *************************************************************************************** |
---|
| 237 | * @ cxy : [in] target cluster identifier. |
---|
| 238 | * @ lid : [in] target core local index. |
---|
| 239 | * @ return 0 if success / return -1 if illegal arguments. |
---|
| 240 | **************************************************************************************/ |
---|
| 241 | int display_sched( unsigned int cxy, |
---|
| 242 | unsigned int lid ); |
---|
| 243 | |
---|
| 244 | /*************************************************************************************** |
---|
[589] | 245 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[444] | 246 | * the list of process registered in a given cluster identified by the <cxy> argument. |
---|
[589] | 247 | * Only the owned processes are displayed when the <owned> argument is non zero. |
---|
[444] | 248 | * It can be called by any thread running in any cluster. |
---|
| 249 | *************************************************************************************** |
---|
| 250 | * @ cxy : [in] target cluster identifier. |
---|
[589] | 251 | * @ owned : [in] only owned processes if non zero. |
---|
[444] | 252 | * @ return 0 if success / return -1 if illegal argument. |
---|
| 253 | **************************************************************************************/ |
---|
[589] | 254 | int display_cluster_processes( unsigned int cxy, |
---|
| 255 | unsigned int owned ); |
---|
[444] | 256 | |
---|
| 257 | /*************************************************************************************** |
---|
[589] | 258 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[445] | 259 | * the list of processes attached to a given TXT channel. |
---|
| 260 | * It can be called by any thread running in any cluster. |
---|
| 261 | *************************************************************************************** |
---|
| 262 | * @ txt_id : [in] TXT terminal indes. |
---|
| 263 | * @ return 0 if success / return -1 if illegal argument. |
---|
| 264 | **************************************************************************************/ |
---|
| 265 | int display_txt_processes( unsigned int txt_id ); |
---|
| 266 | |
---|
| 267 | /*************************************************************************************** |
---|
[589] | 268 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[597] | 269 | * the set of busylocks hold by a thread identified by the <pid> and <trdid> arguments. |
---|
| 270 | * It can be called by any thread running in any cluster. |
---|
| 271 | *************************************************************************************** |
---|
| 272 | * @ pid : [in] process identifier. |
---|
| 273 | * @ trdid : [in] thread identifier. |
---|
| 274 | * @ return 0 if success / return -1 if illegal arguments. |
---|
| 275 | **************************************************************************************/ |
---|
| 276 | int display_busylocks( unsigned int pid, |
---|
| 277 | unsigned int trdid ); |
---|
| 278 | |
---|
| 279 | /*************************************************************************************** |
---|
| 280 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[444] | 281 | * the list of channel devices available in the architecture. |
---|
| 282 | * It can be called by any thread running in any cluster. |
---|
| 283 | *************************************************************************************** |
---|
| 284 | * @ return always 0. |
---|
| 285 | **************************************************************************************/ |
---|
[478] | 286 | int display_chdev( void ); |
---|
[444] | 287 | |
---|
| 288 | /*************************************************************************************** |
---|
[589] | 289 | * This debug syscall displays on the kernel terminal TXT0 |
---|
[444] | 290 | * the list of channel device or pseudo-files registered in the VFS cache. |
---|
| 291 | * It can be called by any thread running in any cluster. |
---|
| 292 | *************************************************************************************** |
---|
| 293 | * @ return always 0. |
---|
| 294 | **************************************************************************************/ |
---|
[478] | 295 | int display_vfs( void ); |
---|
[444] | 296 | |
---|
| 297 | /*************************************************************************************** |
---|
[589] | 298 | * This debug syscall displays on the kernel terminal TXT0 the current DQDT state. |
---|
[444] | 299 | * It can be called by any thread running in any cluster. |
---|
| 300 | *************************************************************************************** |
---|
| 301 | * @ return always 0. |
---|
| 302 | **************************************************************************************/ |
---|
[478] | 303 | int display_dqdt( void ); |
---|
[444] | 304 | |
---|
[611] | 305 | /*************************************************************************************** |
---|
| 306 | * This debug syscall displays on the kernel terminal TXT0 the content of a given |
---|
| 307 | * page of a given VFS mapper. |
---|
| 308 | * It can be called by any thread running in any cluster. |
---|
| 309 | *************************************************************************************** |
---|
| 310 | * @ path : pathname identifying the file/directory in VFS. |
---|
| 311 | * @ page_id : page index in file. |
---|
| 312 | * @ nbytes : number of bytes to display. |
---|
| 313 | * @ return 0 if success / return -1 if file or page not found. |
---|
| 314 | **************************************************************************************/ |
---|
| 315 | int display_mapper( char * path, |
---|
| 316 | unsigned int page_id, |
---|
| 317 | unsigned int nbytes); |
---|
| 318 | |
---|
[623] | 319 | /*************************************************************************************** |
---|
| 320 | * This debug syscall displays on the kernel terminal TXT0 |
---|
| 321 | * the state of the barrier used by the process identified by the <pid> argument. |
---|
| 322 | * It can be called by any thread running in any cluster. |
---|
| 323 | *************************************************************************************** |
---|
| 324 | * @ pid : [in] process identifier. |
---|
| 325 | * @ return 0 if success / return -1 if illegal arguments. |
---|
| 326 | **************************************************************************************/ |
---|
| 327 | int display_barrier( unsigned int pid ); |
---|
| 328 | |
---|
[626] | 329 | /*************************************************************************************** |
---|
[657] | 330 | * This debug syscall displays on the kernel terminal TXT0 the content of <nb_slots> |
---|
| 331 | * of the FAT mapper, starting from <min_slot>. |
---|
| 332 | * If <nb_slots> is zero, it displays the state of the FATFS context in cluster |
---|
| 333 | * defined by the <min_slot> argument interpreted as a cluster identifier. |
---|
[626] | 334 | * It can be called by any thread running in any cluster. |
---|
| 335 | *************************************************************************************** |
---|
[657] | 336 | * @ min_slot : first slot in page |
---|
| 337 | * @ nb_slots : number of slots to display. |
---|
[626] | 338 | * @ return 0 if success / return -1 if page not found. |
---|
| 339 | **************************************************************************************/ |
---|
[657] | 340 | int display_fat( unsigned int min_slot, |
---|
| 341 | unsigned int nb_slots ); |
---|
[626] | 342 | |
---|
[661] | 343 | /*************************************************************************************** |
---|
| 344 | * This debug syscall displays on the kernel terminal the current state of |
---|
| 345 | * a socket identified by the <pid> and <fdid> arguments. |
---|
| 346 | * It can be called by any thread running in any cluster. |
---|
| 347 | *************************************************************************************** |
---|
| 348 | * @ pid : target process identifier. |
---|
| 349 | * @ fdid : file descriptor identifying the target socket. |
---|
| 350 | * @ return 0 if success / return -1 if socket not found. |
---|
| 351 | **************************************************************************************/ |
---|
| 352 | int display_socket( unsigned int pid, |
---|
| 353 | unsigned int fdid ); |
---|
| 354 | |
---|
[678] | 355 | /*************************************************************************************** |
---|
| 356 | * This debug syscall displays on the kernel terminal the set of open file descriptors |
---|
| 357 | * registered in the fd_array of a process identified by the <pid> argument. |
---|
| 358 | * It can be called by any thread running in any cluster. |
---|
| 359 | *************************************************************************************** |
---|
| 360 | * @ pid : target process identifier. |
---|
| 361 | setoreturn 0 if success / return -1 if undefined pid. |
---|
| 362 | **************************************************************************************/ |
---|
| 363 | int display_fd_array( unsigned int pid ); |
---|
| 364 | |
---|
| 365 | /*************************************************************************************** |
---|
| 366 | * This debug syscall displays on the kernel terminal the set of registered windows |
---|
| 367 | * in the FBF device. It can be called by any thread running in any cluster. |
---|
| 368 | *************************************************************************************** |
---|
| 369 | * @ pid : target process identifier / all processes when pid == 0 |
---|
| 370 | * return 0 if success / return -1 if undefined pid. |
---|
| 371 | **************************************************************************************/ |
---|
| 372 | int display_fbf_windows( unsigned int pid ); |
---|
| 373 | |
---|
| 374 | /*************************************************************************************** |
---|
[589] | 375 | * This debug syscall is used to activate / desactivate the context switches trace |
---|
[444] | 376 | * for a core identified by the <cxy> and <lid> arguments. |
---|
| 377 | * It can be called by any thread running in any cluster. |
---|
[678] | 378 | **************************************************************************************** |
---|
[444] | 379 | * @ active : activate trace if non zero / desactivate if zero. |
---|
| 380 | * @ cxy : cluster identifier. |
---|
| 381 | * @ lid : core local index. |
---|
| 382 | * @ returns O if success / returns -1 if illegal arguments. |
---|
[678] | 383 | ***************************************************************************************/ |
---|
[444] | 384 | int trace( unsigned int active, |
---|
[450] | 385 | unsigned int cxy, |
---|
| 386 | unsigned int lid ); |
---|
[444] | 387 | |
---|
[678] | 388 | /*************************************************************************************** |
---|
[589] | 389 | * This syscall implements an user-level interactive debugger that can be |
---|
[445] | 390 | * introduced in any user application to display various kernel distributed structures. |
---|
[678] | 391 | **************************************************************************************/ |
---|
[478] | 392 | void idbg( void ); |
---|
[444] | 393 | |
---|
| 394 | |
---|
[678] | 395 | /****************** Non standard (ALMOS-MKH specific) malloc operations **************/ |
---|
[444] | 396 | |
---|
[678] | 397 | //////////////////////////////////////////////////////////////////////////////////////// |
---|
[444] | 398 | // General principles: |
---|
| 399 | // - In user space the HEAP zone spread between the ELF zone and the STACK zone, |
---|
| 400 | // as defined in the kernel_config.h file. |
---|
| 401 | // - The malloc library uses the mmap() syscall to create - on demand - |
---|
| 402 | // one vseg in a given cluster. The size of this vseg is defined below |
---|
| 403 | // by the MALLOC_LOCAL_STORE_SIZE parameter. |
---|
| 404 | // - For a standard malloc(), the target cluster is the cluster containing |
---|
| 405 | // the core running the client thread. |
---|
| 406 | // - For a remote_malloc(), the target cluster is explicitely defined |
---|
| 407 | // by the argument. |
---|
| 408 | // - In each cluster, the available storage in virtual space is handled by a |
---|
| 409 | // local allocator using the buddy algorithm. |
---|
| 410 | // |
---|
| 411 | // TODO : In this first implementation one single - fixed size - vseg |
---|
| 412 | // is allocated on demand in each cluster. |
---|
| 413 | // We should introduce the possibility to dynamically allocate |
---|
[678] | 414 | // several vsegs in each cluster (using several mmap) when required. |
---|
| 415 | //////////////////////////////////////////////////////////////////////////////////////// |
---|
[444] | 416 | // Free blocks organisation in each cluster : |
---|
| 417 | // - All free blocks have a size that is a power of 2, larger or equal |
---|
| 418 | // to MALLOC_MIN_BLOCK_SIZE (typically 64 bytes). |
---|
| 419 | // - All free blocks are aligned. |
---|
| 420 | // - They are pre-classed in an array of linked lists, where all blocks in a |
---|
| 421 | // given list have the same size. |
---|
| 422 | // - The NEXT pointer implementing those linked lists is written |
---|
| 423 | // in the first bytes of the block itself, using the unsigned int type. |
---|
| 424 | // - The pointers on the first free block for each size are stored in an |
---|
| 425 | // array of pointers free[32] in the storage(x,y) descriptor. |
---|
[678] | 426 | //////////////////////////////////////////////////////////////////////////////////////// |
---|
[444] | 427 | // Allocation policy: |
---|
| 428 | // - The block size required by the user can be any value, but the allocated |
---|
| 429 | // block size can be larger than the requested size: |
---|
| 430 | // - The allocator computes actual_size, that is the smallest power of 2 |
---|
| 431 | // value larger or equal to the requested size AND larger or equal to |
---|
| 432 | // MALLOC_MIN_BLOCK_SIZE. |
---|
| 433 | // - It pop the linked list of free blocks corresponding to actual_size, |
---|
| 434 | // and returns the block B if the list[actual_size] is not empty. |
---|
| 435 | // - If the list[actual_size] is empty, it pop the list[actual_size * 2]. |
---|
| 436 | // If a block B' is found, it breaks this block in 2 B/2 blocks, returns |
---|
| 437 | // the first B/2 block and push the other B/2 block into list[actual_size]. |
---|
| 438 | // - If the list[actual_size * 2] is empty, it pop the list[actual_size * 4]. |
---|
| 439 | // If a block B is found, it break this block in 3 blocks B/4, B/4 and B/2, |
---|
| 440 | // returns the first B/4 block, push the other blocks B/4 and B/2 into |
---|
| 441 | // the proper lists. etc... |
---|
| 442 | // - If no block satisfying the request is available it returns a failure |
---|
| 443 | // (NULL pointer). |
---|
| 444 | // - This allocation policy has the nice following property: |
---|
| 445 | // If the vseg is aligned (the vseg base is a multiple of the |
---|
| 446 | // vseg size), all allocated blocks are aligned on the actual_size. |
---|
[678] | 447 | //////////////////////////////////////////////////////////////////////////////////////// |
---|
[444] | 448 | // Free policy: |
---|
| 449 | // - Each allocated block is registered in an alloc[] array of unsigned char. |
---|
| 450 | // - This registration is required by the free() operation, because the size |
---|
| 451 | // of the allocated block must be obtained from the base address of the block. |
---|
| 452 | // - The number of entries in this array is equal to the max number |
---|
| 453 | // of allocated block : MALLOC_LOCAL_STORE_SIZE / MALLOC_MIN_BLOCK_SIZE. |
---|
| 454 | // - For each allocated block, the value registered in the alloc[] array |
---|
| 455 | // is log2( size_of_allocated_block ). |
---|
| 456 | // - The index in this array is computed from the allocated block base address: |
---|
| 457 | // index = (block_base - vseg_base) / MALLOC_MIN_BLOCK_SIZE |
---|
| 458 | // - The alloc[] array is stored at the end of heap segment. This consume |
---|
| 459 | // (1 / MALLOC_MIN_BLOCK_SIZE) of the total storage capacity. |
---|
[445] | 460 | ///////////////////////////////////////////////////////////////////////////////////////// |
---|
[444] | 461 | |
---|
| 462 | |
---|
| 463 | #define MALLOC_INITIALIZED 0xBABEF00D // magic number when initialised |
---|
| 464 | #define MALLOC_MIN_BLOCK_SIZE 0x40 // 64 bytes |
---|
[643] | 465 | #define MALLOC_LOCAL_STORE_SIZE 0x2000000 // 32 Mbytes |
---|
[444] | 466 | #define MALLOC_MAX_CLUSTERS 0x100 // 256 clusters |
---|
| 467 | |
---|
[445] | 468 | ///////////////////////////////////////////////////////////////////////////////////////// |
---|
| 469 | // store(x,y) descriptor (one per cluster) |
---|
| 470 | ///////////////////////////////////////////////////////////////////////////////////////// |
---|
[444] | 471 | |
---|
| 472 | typedef struct malloc_store_s |
---|
| 473 | { |
---|
[445] | 474 | pthread_mutex_t mutex; // lock protecting exclusive access to local heap |
---|
[444] | 475 | unsigned int initialized; // initialised when value == MALLOC_INITIALIZED |
---|
| 476 | unsigned int cxy; // cluster identifier |
---|
| 477 | unsigned int store_base; // store base address |
---|
| 478 | unsigned int store_size; // store size (bytes) |
---|
| 479 | unsigned int alloc_base; // alloc[] array base address |
---|
| 480 | unsigned int alloc_size; // alloc[] array size (bytes) |
---|
| 481 | unsigned int free[32]; // array of addresses of first free block |
---|
| 482 | } |
---|
| 483 | malloc_store_t; |
---|
| 484 | |
---|
| 485 | /***************************************************************************************** |
---|
| 486 | * This function allocates <size> bytes of memory in user space, and returns a pointer |
---|
| 487 | * to the allocated buffer. The pysical memory is allocated from store located in |
---|
| 488 | * cluster identified by the <cxy> argument. |
---|
| 489 | ***************************************************************************************** |
---|
| 490 | * @ size : number of requested bytes. |
---|
| 491 | * @ cxy : target cluster identifier. |
---|
| 492 | * @ returns a pointer on the allocated buffer if success / returns NULL if failure |
---|
| 493 | ****************************************************************************************/ |
---|
| 494 | void * remote_malloc( unsigned int size, |
---|
| 495 | unsigned int cxy ); |
---|
| 496 | |
---|
| 497 | /***************************************************************************************** |
---|
| 498 | * This function releases the memory buffer identified by the <ptr> argument, |
---|
| 499 | * to the store identified by the <cxy> argument. |
---|
[650] | 500 | * It does nothing, but displays an error message, if the ptr is illegal. |
---|
[444] | 501 | ***************************************************************************************** |
---|
| 502 | * @ ptr : pointer on the released buffer. |
---|
| 503 | * @ cxy : target cluster identifier. |
---|
| 504 | ****************************************************************************************/ |
---|
| 505 | void remote_free( void * ptr, |
---|
| 506 | unsigned int cxy ); |
---|
| 507 | |
---|
| 508 | /***************************************************************************************** |
---|
| 509 | * This function releases the memory buffer identified by the <ptr> argument, |
---|
| 510 | * to the store located in cluster identified by the <cxy> argument, and allocates |
---|
| 511 | * a new buffer containing <size> bytes from this store. |
---|
| 512 | * The content of the old buffer is copied to the new buffer, up to <size> bytes. |
---|
| 513 | * It displays an error message, but does nothing if the ptr is illegal. |
---|
| 514 | ***************************************************************************************** |
---|
| 515 | * @ ptr : pointer on the released buffer. |
---|
| 516 | * @ size : new buffer requested size (bytes). |
---|
| 517 | * @ cxy : target cluster identifier. |
---|
| 518 | * @ return a pointer on allocated buffer if success / return NULL if failure |
---|
| 519 | ****************************************************************************************/ |
---|
| 520 | void * remote_realloc( void * ptr, |
---|
| 521 | unsigned int size, |
---|
| 522 | unsigned int cxy ); |
---|
[445] | 523 | |
---|
[444] | 524 | /***************************************************************************************** |
---|
| 525 | * This function allocates enough space for <count> objects that are <size> bytes |
---|
| 526 | * of memory each from the store located in cluster identied by the <cxy> argument. |
---|
| 527 | * The allocated memory is filled with bytes of value zero. |
---|
| 528 | ***************************************************************************************** |
---|
| 529 | * @ count : number of requested objects. |
---|
| 530 | * @ size : number of bytes per object. |
---|
| 531 | * @ cxy : target cluster identifier. |
---|
| 532 | * @ returns a pointer on allocated buffer if success / returns NULL if failure |
---|
| 533 | ****************************************************************************************/ |
---|
| 534 | void * remote_calloc( unsigned int count, |
---|
| 535 | unsigned int size, |
---|
| 536 | unsigned int cxy ); |
---|
| 537 | |
---|
[637] | 538 | /********* Non standard (ALMOS-MKH specific) pthread_parallel_create() syscall *********/ |
---|
| 539 | |
---|
| 540 | ////////////////////////////////////////////////////////////////////////////////////////// |
---|
[650] | 541 | // This syscall can be used to parallelize the creation, and the termination |
---|
| 542 | // of a parallel multi-threaded application. |
---|
| 543 | // It removes in the main thread the sequencial loop that creates the N working threads |
---|
| 544 | // (N pthread_create() ), and removes also the sequencial loop that waits completion |
---|
| 545 | // of these N working threads (N pthread_join() ). |
---|
| 546 | // It creates one <work> thread (in detached mode) per core in the target architecture. |
---|
| 547 | // Each <work> thread is identified by a continuous [tid] index. |
---|
| 548 | // For a regular architecture, defined by the [x_size , y_size , ncores] parameters, |
---|
| 549 | // the number of working threads can be the simply computed as (x_size * y_size * ncores), |
---|
| 550 | // and the coordinates[x,y,lid] of the core running the thread[tid] cand be directly |
---|
| 551 | // derived from the [tid] value with the following relations: |
---|
| 552 | // . cid = (x * y_size) + y |
---|
| 553 | // . tid = (cid * ncores ) + lid |
---|
| 554 | // . lid = tid % ncores |
---|
| 555 | // . cid = tid / ncores |
---|
| 556 | // . y = cid % y_size |
---|
| 557 | // . x = cid / y_size |
---|
| 558 | // The pthread_parallel_create() function returns only when all <work> threads completed |
---|
[637] | 559 | // (successfully or not). |
---|
| 560 | // |
---|
[650] | 561 | // WARNING : The function executed by the working thread is application specific, |
---|
| 562 | // but the structure defining the arguments passed to this function is imposed. |
---|
| 563 | // The "pthread_parallel_work_args_t" structure is defined below, and contains |
---|
| 564 | // two fields: the tid value, and a pointer on a pthread_barrier_t. |
---|
| 565 | // This barrier must be used by each working thread to signal completion before exit. |
---|
| 566 | // The global variables implementing these stuctures for each working thread |
---|
| 567 | // are allocated and initialised by the pthread_parallel_create() function. |
---|
[637] | 568 | // |
---|
[650] | 569 | // Implementation note: the pthread_parallel_create()a function creates a distributed |
---|
| 570 | // quad-tree (DQT) of <build> threads covering all cores required to execute the parallel |
---|
| 571 | // application. This quad tree is entirely defined by the root_level parameter. |
---|
[637] | 572 | // Depending on the hardware topology, this DQT can be truncated, (i.e. some |
---|
| 573 | // parent nodes can have less than 4 chidren), if (x_size != y_size), or if one size |
---|
[650] | 574 | // is not a power of 2. Each <build> thread is identified by two indexes [cid][level]. |
---|
| 575 | // Each <build> thread makes the following tasks: |
---|
[637] | 576 | // 1) It calls the pthread_create() function to create up to 4 children threads, that |
---|
[650] | 577 | // are <work> threads when (level == 0), or <build> threads, when (level > 0). |
---|
| 578 | // 2) It allocates and initializes the barrier, used to block the parent thread until |
---|
| 579 | // children completion. |
---|
[637] | 580 | // 3) It calls the pthread_barrier_wait( self ) to wait until all children threads |
---|
| 581 | // completed (successfully or not). |
---|
| 582 | // 4) It calls the pthread_barrier_wait( parent ) to unblock the parent thread. |
---|
| 583 | ////////////////////////////////////////////////////////////////////////////////////////// |
---|
| 584 | |
---|
| 585 | /***************************************************************************************** |
---|
[650] | 586 | * structure defining the arguments for the <build> thread function |
---|
| 587 | ****************************************************************************************/ |
---|
| 588 | typedef struct pthread_parallel_build_args_s |
---|
| 589 | { |
---|
| 590 | unsigned char cid; // this <build> thread cluster index |
---|
| 591 | unsigned char level; // this <build> thread level in quad-tree |
---|
| 592 | unsigned char parent_cid; // parent <build> thread cluster index |
---|
| 593 | pthread_barrier_t * parent_barrier; // pointer on parent <build> thread barrier |
---|
| 594 | unsigned char root_level; // quad-tree root level |
---|
| 595 | void * work_func; // pointer on working thread function |
---|
| 596 | unsigned int x_size; // platform global parameter |
---|
| 597 | unsigned int y_size; // platform global parameter |
---|
| 598 | unsigned int ncores; // platform global parameter |
---|
| 599 | unsigned int error; // return value : 0 if success |
---|
| 600 | } |
---|
| 601 | pthread_parallel_build_args_t; |
---|
| 602 | |
---|
| 603 | /***************************************************************************************** |
---|
| 604 | * structure defining the arguments for the <work> thread function |
---|
| 605 | ****************************************************************************************/ |
---|
| 606 | typedef struct pthread_parallel_work_args_s |
---|
| 607 | { |
---|
| 608 | unsigned int tid; // thread identifier |
---|
| 609 | pthread_barrier_t * barrier; // to signal completion |
---|
| 610 | } |
---|
| 611 | pthread_parallel_work_args_t; |
---|
| 612 | |
---|
| 613 | /***************************************************************************************** |
---|
| 614 | * This blocking function creates N working threads identified by the [tid] continuous |
---|
| 615 | * index, that execute the code defined by the <work_func> argument, and returns only |
---|
| 616 | * when all working threads completed. |
---|
| 617 | * The number N of created threads is entirely defined by the <root_level> argument, |
---|
| 618 | * that defines an abstract quad-tree, with a square base : root_level in [0,1,2,3,4], |
---|
[637] | 619 | * side in [1,2,4,8,16], nclusters in [1,4,16,64,256]. This base is called macro_cluster. |
---|
[650] | 620 | * A working thread is created on all cores contained in this abstract macro-cluster. |
---|
[637] | 621 | * The actual number of physical clusters containing cores can be smaller than the number |
---|
[650] | 622 | * of clusters covered by the abstract quad tree. |
---|
| 623 | * All threads execute the same <work_func> function, on different arguments, that are |
---|
| 624 | * specified as an array of structures pthread_parallel_work_args_t, allocated and |
---|
| 625 | * initialised by this function. |
---|
[637] | 626 | ***************************************************************************************** |
---|
| 627 | * @ root_level : [in] DQT root level in [0,1,2,3,4]. |
---|
| 628 | * @ work_func : [in] pointer on start function. |
---|
| 629 | * @ return 0 if success / return -1 if failure. |
---|
| 630 | ****************************************************************************************/ |
---|
[650] | 631 | int pthread_parallel_create( unsigned int root_level, |
---|
| 632 | void * work_func ); |
---|
[637] | 633 | |
---|
[650] | 634 | |
---|
| 635 | |
---|
[643] | 636 | /********* Non standard (ALMOS-MKH specific) Frame Buffer access syscalls *************/ |
---|
| 637 | |
---|
| 638 | ////////////////////////////////////////////////////////////////////////////////////////// |
---|
| 639 | // The following system calls can be used to access the SoCLib Frame Buffer, that |
---|
| 640 | // is a very simple graphic controller, that is seen by the software as a single |
---|
| 641 | // buffer of <height> lines of <width> pixels. |
---|
| 642 | ////////////////////////////////////////////////////////////////////////////////////////// |
---|
| 643 | |
---|
| 644 | /***************************************************************************************** |
---|
[678] | 645 | * This syscall returns in the <width>, <height> and <type> arguments |
---|
[657] | 646 | * the implementation specific frame buffer features. |
---|
[643] | 647 | ***************************************************************************************** |
---|
| 648 | * @ width : [out] number of pixels per line. |
---|
| 649 | * @ height : [out] number of lines. |
---|
| 650 | * @ type : [out] pixel encoding type. |
---|
[657] | 651 | * @ returns 0 if success / returns -1 if not found. |
---|
[643] | 652 | ****************************************************************************************/ |
---|
[678] | 653 | int fbf_get_config( int * width, |
---|
| 654 | int * height, |
---|
| 655 | int * type ); |
---|
[643] | 656 | |
---|
| 657 | /***************************************************************************************** |
---|
[678] | 658 | * This syscall creates a new window defined by the <l_zero>, <p_zero>, <nlines>, |
---|
| 659 | * and <npixels> arguments for the calling process, and register the client process PID, |
---|
| 660 | * the allocated <wid>, and the window size and coordinates in the list of windows rooted |
---|
| 661 | * in the FBF device descriptor. It returns in the output argument <buffer> the pointer |
---|
| 662 | * on the buffer associated to the window, mapped in user space. |
---|
[657] | 663 | * Warning : Both pixel [p_zero,l_zero] and pixel [p_zero+npixels-1,l_zero+nlines-1] |
---|
| 664 | * must be contained in the frame buffer. |
---|
[643] | 665 | ***************************************************************************************** |
---|
[657] | 666 | * @ l_zero : [in] first line index in FBF coordinates system. |
---|
| 667 | * @ p_zero : [in] first pixel index in FBF coordinates system. |
---|
| 668 | * @ nlines : [in] number of lines in the window. |
---|
| 669 | * @ npixels : [in] number of pixels per line in the window. |
---|
[678] | 670 | * @ buffer : [out] pointer on buffer associated to window, mapped in user space. |
---|
| 671 | * @ returns <kern_wid> if success / returns -1 if illegal size or coordinates. |
---|
[643] | 672 | ****************************************************************************************/ |
---|
[678] | 673 | int fbf_create_window( int l_zero, |
---|
| 674 | int p_zero, |
---|
| 675 | int nlines, |
---|
| 676 | int npixels, |
---|
| 677 | void ** buffer ); |
---|
[643] | 678 | |
---|
| 679 | /***************************************************************************************** |
---|
[678] | 680 | * This syscall activates/desactivates the window identified by the <wid> argument, |
---|
| 681 | * according to the <active> argument. The calling process must be the window owner. |
---|
| 682 | ***************************************************************************************** |
---|
| 683 | * @ wid : window identifier. |
---|
| 684 | * @ active : activate window if non zero / desactivate window if zero. |
---|
| 685 | * @ returns 0 if success / returns -1 if illegal wid argument. |
---|
| 686 | ****************************************************************************************/ |
---|
| 687 | int fbf_active_window( int wid, |
---|
| 688 | int active ); |
---|
| 689 | |
---|
| 690 | /***************************************************************************************** |
---|
| 691 | * This syscall deletes an existing window, identified by the <wid> argument. |
---|
[657] | 692 | * The calling process must be the window owner. |
---|
[643] | 693 | ***************************************************************************************** |
---|
[657] | 694 | * @ wid : window identifier. |
---|
[678] | 695 | * @ returns 0 if success / returns -1 if illegal wid argument. |
---|
[657] | 696 | ****************************************************************************************/ |
---|
[678] | 697 | int fbf_delete_window( int wid ); |
---|
[657] | 698 | |
---|
| 699 | /***************************************************************************************** |
---|
[678] | 700 | * This syscall refreshes in FBF all lines of a window identified by the <wid> argument, |
---|
[657] | 701 | * when the line index is in the interval [line_first,line_last[. |
---|
| 702 | * It scans all registered windows to take into account the windows overlap. |
---|
[678] | 703 | * The calling process must be the window owner, and the line_fist / line_last indexes |
---|
| 704 | * must be compatible with the window height. |
---|
[657] | 705 | ***************************************************************************************** |
---|
| 706 | * @ wid : window identifier. |
---|
[678] | 707 | * @ line_first : first line index in window. |
---|
| 708 | * @ line_last : last line index in window (excluded). |
---|
| 709 | * @ returns 0 if success / returns -1 if illegal arguments. |
---|
[657] | 710 | ****************************************************************************************/ |
---|
[678] | 711 | int fbf_refresh_window( int wid, |
---|
| 712 | int line_first, |
---|
| 713 | int line_last ); |
---|
[657] | 714 | |
---|
| 715 | /***************************************************************************************** |
---|
[678] | 716 | * This syscall gives the highest priority to the window identified by the <wid> |
---|
| 717 | * argument, and refresh the FBF to take into account the new windows overlap priorities. |
---|
| 718 | * The calling process must be the window owner. |
---|
| 719 | ***************************************************************************************** |
---|
| 720 | * @ wid : window identifier. |
---|
| 721 | * @ returns 0 if success / returns -1 if wid illegal argument. |
---|
| 722 | ****************************************************************************************/ |
---|
| 723 | int fbf_front_window( int wid ); |
---|
| 724 | |
---|
| 725 | /***************************************************************************************** |
---|
| 726 | * This syscall changes the size of a window identified by the <wid> argument as defined |
---|
[657] | 727 | * by the <width> and height> arguments. The calling process must be the window owner. |
---|
[678] | 728 | * Both pixel [p_zero,l_zero] and pixel [p_zero+width-1,l_zero+height-1] must be |
---|
| 729 | * contained in the frame buffer. |
---|
[657] | 730 | ***************************************************************************************** |
---|
| 731 | * @ wid : window identifier. |
---|
| 732 | * @ width : number of pixels in line (one byte per pixel). |
---|
| 733 | * @ height : number of lines in window. |
---|
| 734 | * @ returns 0 if success / returns -1 if illegal arguments. |
---|
| 735 | ****************************************************************************************/ |
---|
[678] | 736 | int fbf_resize_window( int wid, |
---|
| 737 | int width, |
---|
| 738 | int height ); |
---|
[657] | 739 | |
---|
| 740 | /***************************************************************************************** |
---|
[678] | 741 | * This syscall changes the <l_zero> & <p_zero> coordinates of a window identified |
---|
[657] | 742 | * by the <wid> argument. The calling process must be the window owner. |
---|
[678] | 743 | * Both pixel [p_zero,l_zero] and pixel [p_zero+width-1,l_zero+height-1] |
---|
[657] | 744 | * must be contained in the frame buffer. |
---|
| 745 | ***************************************************************************************** |
---|
| 746 | * @ wid : window identifier. |
---|
| 747 | * @ l_zero : new first line index in FBF. |
---|
| 748 | * @ p_zero : new first pixel index in FBF. |
---|
| 749 | * @ returns 0 if success / returns -1 if illegal arguments. |
---|
| 750 | ****************************************************************************************/ |
---|
[678] | 751 | int fbf_move_window( int wid, |
---|
| 752 | int l_zero, |
---|
| 753 | int p_zero ); |
---|
[657] | 754 | |
---|
[678] | 755 | |
---|
| 756 | |
---|
| 757 | |
---|
[657] | 758 | /***************************************************************************************** |
---|
| 759 | * This syscall is deprecated (january 2020 AG) |
---|
| 760 | * This blocking function moves <length> pixels from the user buffer defined by <buffer> |
---|
| 761 | * argument to the frame buffer, starting at <offset> in FBF. |
---|
| 762 | ***************************************************************************************** |
---|
[643] | 763 | * @ buffer : pointer on buffer in user space. |
---|
| 764 | * @ length : number of pixels (one byte per pixel). |
---|
[657] | 765 | * @ offset : first pixel index in window. |
---|
| 766 | * @ returns 0 if success / returns -1 if illegal arguments. |
---|
[643] | 767 | ****************************************************************************************/ |
---|
[678] | 768 | int fbf_write( void * buffer, |
---|
| 769 | int length, |
---|
| 770 | int offset ); |
---|
[643] | 771 | |
---|
[657] | 772 | /***************************************************************************************** |
---|
| 773 | * This syscall is deprecated (january 2020 AG) |
---|
| 774 | * This blocking function moves <length> pixels from the frame buffer, starting at |
---|
| 775 | * <offset> in FBF, to the user buffer defined by <buffer> argument. |
---|
| 776 | ***************************************************************************************** |
---|
| 777 | * @ buffer : pointer on buffer in user space. |
---|
| 778 | * @ length : number of pixels (one byte per pixel). |
---|
| 779 | * @ offset : first pixel index in window. |
---|
| 780 | * @ returns 0 if success / returns -1 if illegal arguments. |
---|
| 781 | ****************************************************************************************/ |
---|
[678] | 782 | int fbf_read( void * buffer, |
---|
| 783 | int length, |
---|
| 784 | int offset ); |
---|
[657] | 785 | |
---|
| 786 | |
---|
[444] | 787 | #endif /* _LIBALMOSMKH_H_ */ |
---|
| 788 | |
---|