Changeset 635 for trunk/kernel/mm/kcm.c
- Timestamp:
- Jun 26, 2019, 11:42:37 AM (5 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/kernel/mm/kcm.c
r619 r635 1 1 /* 2 * kcm.c - Per clusterKernel Cache Manager implementation.2 * kcm.c - Kernel Cache Manager implementation. 3 3 * 4 * Author Ghassan Almaless (2008,2009,2010,2011,2012) 5 * Alain Greiner (2016,2017,2018,2019) 4 * Author Alain Greiner (2016,2017,2018,2019) 6 5 * 7 6 * Copyright (c) UPMC Sorbonne Universites … … 38 37 39 38 39 ///////////////////////////////////////////////////////////////////////////////////// 40 // Local access functions 41 ///////////////////////////////////////////////////////////////////////////////////// 42 40 43 ////////////////////////////////////////////////////////////////////////////////////// 41 // This static function returns pointer on an allocated block from an active page. 42 // It returns NULL if no block available in selected page. 43 // It changes the page status if required. 44 // This static function must be called by a local thread. 45 // It returns a pointer on a block allocated from a non-full kcm_page. 46 // It makes a panic if no block is available in selected page. 47 // It changes the page status as required. 44 48 ////////////////////////////////////////////////////////////////////////////////////// 45 // @ kcm : pointer on kcm allocator. 46 // @ kcm_page : pointer on active kcm page to use. 47 ///////////////////////////////////////////////////////////////////////////////////// 48 static void * kcm_get_block( kcm_t * kcm, 49 kcm_page_t * kcm_page ) 50 { 51 52 #if DEBUG_KCM 53 thread_t * this = CURRENT_THREAD; 54 uint32_t cycle = (uint32_t)hal_get_cycles(); 49 // @ kcm : pointer on KCM allocator. 50 // @ kcm_page : pointer on a non-full kcm_page. 51 // @ return pointer on allocated block. 52 ///////////////////////////////////////////////////////////////////////////////////// 53 static void * __attribute__((noinline)) kcm_get_block( kcm_t * kcm, 54 kcm_page_t * kcm_page ) 55 { 56 // initialise variables 57 uint32_t size = 1 << kcm->order; 58 uint32_t max = kcm->max_blocks; 59 uint32_t count = kcm_page->count; 60 uint64_t status = kcm_page->status; 61 62 assert( (count < max) , "kcm_page should not be full" ); 63 64 uint32_t index = 1; 65 uint64_t mask = (uint64_t)0x2; 66 uint32_t found = 0; 67 68 // allocate first free block in kcm_page, update status, 69 // and count , compute index of allocated block in kcm_page 70 while( index <= max ) 71 { 72 if( (status & mask) == 0 ) // block non allocated 73 { 74 kcm_page->status = status | mask; 75 kcm_page->count = count + 1; 76 found = 1; 77 78 break; 79 } 80 81 index++; 82 mask <<= 1; 83 } 84 85 // change the page list if almost full 86 if( count == max-1 ) 87 { 88 list_unlink( &kcm_page->list); 89 kcm->active_pages_nr--; 90 91 list_add_first( &kcm->full_root , &kcm_page->list ); 92 kcm->full_pages_nr ++; 93 } 94 95 // compute return pointer 96 void * ptr = (void *)((intptr_t)kcm_page + (index * size) ); 97 98 #if (DEBUG_KCM & 1) 99 thread_t * this = CURRENT_THREAD; 100 uint32_t cycle = (uint32_t)hal_get_cycles(); 55 101 if( DEBUG_KCM < cycle ) 56 printk("\n[%s] thread[%x,%x] enters for %s / page %x / count %d / active %d\n", 57 __FUNCTION__, this->process->pid, this->trdid, kmem_type_str(kcm->type), 58 (intptr_t)kcm_page , kcm_page->count , kcm_page->active ); 59 #endif 60 61 assert( kcm_page->active , "kcm_page should be active" ); 62 63 // get first block available 64 int32_t index = bitmap_ffs( kcm_page->bitmap , kcm->blocks_nr ); 65 66 assert( (index != -1) , "kcm_page should not be full" ); 67 68 // allocate block 69 bitmap_clear( kcm_page->bitmap , index ); 70 71 // increase kcm_page count 72 kcm_page->count ++; 73 74 // change the kcm_page to busy if no more free block in page 75 if( kcm_page->count >= kcm->blocks_nr ) 76 { 77 kcm_page->active = 0; 78 list_unlink( &kcm_page->list); 79 kcm->active_pages_nr --; 80 81 list_add_first( &kcm->busy_root , &kcm_page->list); 82 kcm->busy_pages_nr ++; 83 kcm_page->busy = 1; 84 } 85 86 // compute return pointer 87 void * ptr = (void *)((intptr_t)kcm_page + CONFIG_KCM_SLOT_SIZE 88 + (index * kcm->block_size) ); 89 90 #if DEBUG_KCM 91 cycle = (uint32_t)hal_get_cycles(); 92 if( DEBUG_KCM < cycle ) 93 printk("\n[%s] thread[%x,%x] exit for %s / ptr %x / page %x / count %d\n", 94 __FUNCTION__, this->process->pid, this->trdid, kmem_type_str(kcm->type), 95 (intptr_t)ptr, (intptr_t)kcm_page, kcm_page->count ); 102 printk("\n[%s] thread[%x,%x] allocated block %x in page %x / size %d / count %d / cycle %d\n", 103 __FUNCTION__, this->process->pid, this->trdid, ptr, kcm_page, size, count + 1, cycle ); 96 104 #endif 97 105 98 106 return ptr; 99 } 100 101 ///////////////////////////////////////////////////////////////////////////////////// 102 // This static function releases a previously allocated block. 103 // It changes the kcm_page status if required. 104 ///////////////////////////////////////////////////////////////////////////////////// 105 // @ kcm : pointer on kcm allocator. 106 // @ kcm_page : pointer on kcm_page. 107 // @ ptr : pointer on block to be released. 108 ///////////////////////////////////////////////////////////////////////////////////// 109 static void kcm_put_block ( kcm_t * kcm, 110 kcm_page_t * kcm_page, 111 void * ptr ) 112 { 113 uint32_t index; 114 107 108 } // end kcm_get_block() 109 110 ///////////////////////////////////////////////////////////////////////////////////// 111 // This private static function must be called by a local thread. 112 // It releases a previously allocated block to the relevant kcm_page. 113 // It makes a panic if the released block is not allocated in this page. 114 // It changes the kcm_page status as required. 115 ///////////////////////////////////////////////////////////////////////////////////// 116 // @ kcm : pointer on kcm allocator. 117 // @ kcm_page : pointer on kcm_page. 118 // @ block_ptr : pointer on block to be released. 119 ///////////////////////////////////////////////////////////////////////////////////// 120 static void __attribute__((noinline)) kcm_put_block ( kcm_t * kcm, 121 kcm_page_t * kcm_page, 122 void * block_ptr ) 123 { 124 // initialise variables 125 uint32_t max = kcm->max_blocks; 126 uint32_t size = 1 << kcm->order; 127 uint32_t count = kcm_page->count; 128 uint64_t status = kcm_page->status; 129 115 130 // compute block index from block pointer 116 index = ((uint8_t *)ptr - (uint8_t *)kcm_page - CONFIG_KCM_SLOT_SIZE) / kcm->block_size; 117 118 assert( !bitmap_state( kcm_page->bitmap , index ) , "page already freed" ); 119 120 assert( (kcm_page->count > 0) , "count already zero" ); 121 122 bitmap_set( kcm_page->bitmap , index ); 123 kcm_page->count --; 124 125 // change the page to active if it was busy 126 if( kcm_page->busy ) 127 { 128 kcm_page->busy = 0; 131 uint32_t index = ((intptr_t)block_ptr - (intptr_t)kcm_page) / size; 132 133 // compute mask in bit vector 134 uint64_t mask = ((uint64_t)0x1) << index; 135 136 assert( (status & mask) , "released block not allocated : status (%x,%x) / mask(%x,%x)", 137 GET_CXY(status), GET_PTR(status), GET_CXY(mask ), GET_PTR(mask ) ); 138 139 // update status & count in kcm_page 140 kcm_page->status = status & ~mask; 141 kcm_page->count = count - 1; 142 143 // change the page mode if page was full 144 if( count == max ) 145 { 129 146 list_unlink( &kcm_page->list ); 130 kcm-> busy_pages_nr --;147 kcm->full_pages_nr --; 131 148 132 149 list_add_last( &kcm->active_root, &kcm_page->list ); 133 150 kcm->active_pages_nr ++; 134 kcm_page->active = 1; 135 } 136 137 // change the kcm_page to free if last block in active page 138 if( (kcm_page->active) && (kcm_page->count == 0) ) 139 { 140 kcm_page->active = 0; 141 list_unlink( &kcm_page->list); 142 kcm->active_pages_nr --; 143 144 list_add_first( &kcm->free_root , &kcm_page->list); 145 kcm->free_pages_nr ++; 146 } 147 } 148 149 ///////////////////////////////////////////////////////////////////////////////////// 150 // This static function allocates one page from PPM. It initializes 151 // the kcm_page descriptor, and introduces the new kcm_page into freelist. 152 ///////////////////////////////////////////////////////////////////////////////////// 153 static error_t freelist_populate( kcm_t * kcm ) 154 { 155 page_t * page; 156 kcm_page_t * kcm_page; 157 kmem_req_t req; 158 159 // get one page from local PPM 160 req.type = KMEM_PAGE; 161 req.size = 0; 162 req.flags = AF_KERNEL; 163 page = kmem_alloc( &req ); 164 165 if( page == NULL ) 166 { 167 printk("\n[ERROR] in %s : failed to allocate page in cluster %x\n", 168 __FUNCTION__ , local_cxy ); 169 return ENOMEM; 170 } 171 172 // get page base address 173 xptr_t base_xp = ppm_page2base( XPTR( local_cxy , page ) ); 174 kcm_page = (kcm_page_t *)GET_PTR( base_xp ); 175 176 // initialize KCM-page descriptor 177 bitmap_set_range( kcm_page->bitmap , 0 , kcm->blocks_nr ); 178 179 kcm_page->busy = 0; 180 kcm_page->active = 0; 181 kcm_page->count = 0; 182 kcm_page->kcm = kcm; 183 kcm_page->page = page; 184 185 // introduce new page in free-list 186 list_add_first( &kcm->free_root , &kcm_page->list ); 187 kcm->free_pages_nr ++; 188 189 return 0; 190 } 191 192 ///////////////////////////////////////////////////////////////////////////////////// 193 // This private function gets one KCM page from the KCM freelist. 194 // It populates the freelist if required. 195 ///////////////////////////////////////////////////////////////////////////////////// 196 static kcm_page_t * freelist_get( kcm_t * kcm ) 197 { 198 error_t error; 199 kcm_page_t * kcm_page; 200 201 // get a new page from PPM if freelist empty 202 if( kcm->free_pages_nr == 0 ) 203 { 204 error = freelist_populate( kcm ); 205 if( error ) return NULL; 206 } 207 208 // get first KCM page from freelist and unlink it 209 kcm_page = LIST_FIRST( &kcm->free_root, kcm_page_t , list ); 210 list_unlink( &kcm_page->list ); 211 kcm->free_pages_nr --; 151 } 152 153 #if (DEBUG_KCM & 1) 154 thread_t * this = CURRENT_THREAD; 155 uint32_t cycle = (uint32_t)hal_get_cycles(); 156 if( DEBUG_KCM < cycle ) 157 printk("\n[%s] thread[%x,%x] released block %x in page %x / size %d / count %d / cycle %d\n", 158 __FUNCTION__, this->process->pid, this->trdid, block_ptr, kcm_page, size, count - 1, cycle ); 159 #endif 160 161 } // kcm_put_block() 162 163 ///////////////////////////////////////////////////////////////////////////////////// 164 // This private static function must be called by a local thread. 165 // It returns one non-full kcm_page with te following policy : 166 // - if the "active_list" is non empty, it returns the first "active" page, 167 // without modifying the KCM state. 168 // - if the "active_list" is empty, it allocates a new page fromm PPM, inserts 169 // this page in the active_list, and returns it. 170 ///////////////////////////////////////////////////////////////////////////////////// 171 // @ kcm : local pointer on local KCM allocator. 172 // @ return pointer on a non-full kcm page if success / returns NULL if no memory. 173 ///////////////////////////////////////////////////////////////////////////////////// 174 static kcm_page_t * __attribute__((noinline)) kcm_get_page( kcm_t * kcm ) 175 { 176 kcm_page_t * kcm_page; 177 178 uint32_t active_pages_nr = kcm->active_pages_nr; 179 180 if( active_pages_nr > 0 ) // return first active page 181 { 182 kcm_page = LIST_FIRST( &kcm->active_root , kcm_page_t , list ); 183 } 184 else // allocate a new page from PPM 185 { 186 // get one 4 Kbytes page from local PPM 187 page_t * page = ppm_alloc_pages( 0 ); 188 189 if( page == NULL ) 190 { 191 printk("\n[ERROR] in %s : failed to allocate page in cluster %x\n", 192 __FUNCTION__ , local_cxy ); 193 194 return NULL; 195 } 196 197 // get page base address 198 xptr_t base_xp = ppm_page2base( XPTR( local_cxy , page ) ); 199 200 // get local pointer on kcm_page 201 kcm_page = GET_PTR( base_xp ); 202 203 // initialize kcm_page descriptor 204 kcm_page->status = 0; 205 kcm_page->count = 0; 206 kcm_page->kcm = kcm; 207 kcm_page->page = page; 208 209 // introduce new page in KCM active_list 210 list_add_first( &kcm->active_root , &kcm_page->list ); 211 kcm->active_pages_nr ++; 212 } 212 213 213 214 return kcm_page; 214 } 215 216 } // end kcm_get_page() 215 217 216 218 ////////////////////////////// 217 219 void kcm_init( kcm_t * kcm, 218 uint32_t type ) 219 { 220 221 // the kcm_page descriptor must fit in the KCM slot 222 assert( (sizeof(kcm_page_t) <= CONFIG_KCM_SLOT_SIZE) , "KCM slot too small\n" ); 223 224 // the allocated object must fit in one single page 225 assert( (kmem_type_size(type) <= (CONFIG_PPM_PAGE_SIZE - CONFIG_KCM_SLOT_SIZE)), 226 "allocated object requires more than one single page\n" ); 220 uint32_t order) 221 { 222 223 assert( ((order > 5) && (order < 12)) , "order must be in [6,11]" ); 227 224 228 225 // initialize lock 229 busylock_init( &kcm->lock , LOCK_KCM_STATE ); 230 231 // initialize KCM type 232 kcm->type = type; 226 remote_busylock_init( XPTR( local_cxy , &kcm->lock ) , LOCK_KCM_STATE ); 233 227 234 228 // initialize KCM page lists 235 kcm->free_pages_nr = 0; 236 kcm->busy_pages_nr = 0; 229 kcm->full_pages_nr = 0; 237 230 kcm->active_pages_nr = 0; 238 list_root_init( &kcm->free_root ); 239 list_root_init( &kcm->busy_root ); 231 list_root_init( &kcm->full_root ); 240 232 list_root_init( &kcm->active_root ); 241 233 242 // initialize block size 243 uint32_t block_size = ARROUND_UP( kmem_type_size( type ) , CONFIG_KCM_SLOT_SIZE ); 244 kcm->block_size = block_size; 245 246 // initialize number of blocks per page 247 uint32_t blocks_nr = (CONFIG_PPM_PAGE_SIZE - CONFIG_KCM_SLOT_SIZE) / block_size; 248 kcm->blocks_nr = blocks_nr; 249 234 // initialize order and max_blocks 235 kcm->order = order; 236 kcm->max_blocks = ( CONFIG_PPM_PAGE_SIZE >> order ) - 1; 237 250 238 #if DEBUG_KCM 251 239 thread_t * this = CURRENT_THREAD; 252 240 uint32_t cycle = (uint32_t)hal_get_cycles(); 253 241 if( DEBUG_KCM < cycle ) 254 printk("\n[%s] thread[%x,%x] initialised KCM %s : block_size %d / blocks_nr %d\n", 255 __FUNCTION__, this->process->pid, this->trdid, 256 kmem_type_str( kcm->type ), block_size, blocks_nr ); 257 #endif 258 259 } 242 printk("\n[%s] thread[%x,%x] initialised KCM / order %d / max_blocks %d\n", 243 __FUNCTION__, this->process->pid, this->trdid, order, kcm->max_blocks ); 244 #endif 245 246 } // end kcm_init() 260 247 261 248 /////////////////////////////// … … 263 250 { 264 251 kcm_page_t * kcm_page; 265 list_entry_t * iter; 252 253 // build extended pointer on KCM lock 254 xptr_t lock_xp = XPTR( local_cxy , &kcm->lock ); 266 255 267 256 // get KCM lock 268 busylock_acquire( &kcm->lock ); 269 270 // release all free pages 271 LIST_FOREACH( &kcm->free_root , iter ) 272 { 273 kcm_page = (kcm_page_t *)LIST_ELEMENT( iter , kcm_page_t , list ); 274 list_unlink( iter ); 275 kcm->free_pages_nr --; 257 remote_busylock_acquire( lock_xp ); 258 259 // release all full pages 260 while( list_is_empty( &kcm->full_root ) == false ) 261 { 262 kcm_page = LIST_FIRST( &kcm->full_root , kcm_page_t , list ); 263 list_unlink( &kcm_page->list ); 276 264 ppm_free_pages( kcm_page->page ); 277 265 } 278 266 279 // release all active pages 280 LIST_FOREACH( &kcm->active_root , iter ) 281 { 282 kcm_page = (kcm_page_t *)LIST_ELEMENT( iter , kcm_page_t , list ); 283 list_unlink( iter ); 284 kcm->free_pages_nr --; 267 // release all empty pages 268 while( list_is_empty( &kcm->active_root ) == false ) 269 { 270 kcm_page = LIST_FIRST( &kcm->active_root , kcm_page_t , list ); 271 list_unlink( &kcm_page->list ); 285 272 ppm_free_pages( kcm_page->page ); 286 273 } 287 274 288 // release all busy pages289 LIST_FOREACH( &kcm->busy_root , iter )290 {291 kcm_page = (kcm_page_t *)LIST_ELEMENT( iter , kcm_page_t , list );292 list_unlink( iter );293 kcm->free_pages_nr --;294 ppm_free_pages( kcm_page->page );295 }296 297 275 // release KCM lock 298 busylock_release( &kcm->lock);276 remote_busylock_release( lock_xp ); 299 277 } 300 278 301 /////////////////////////////// 302 void * kcm_alloc( kcm_t * kcm ) 303 { 279 ////////////////////////////////// 280 void * kcm_alloc( uint32_t order ) 281 { 282 kcm_t * kcm_ptr; 304 283 kcm_page_t * kcm_page; 305 void * ptr = NULL; // pointer on block 284 void * block_ptr; 285 286 // min block size is 64 bytes 287 if( order < 6 ) order = 6; 288 289 assert( (order < 12) , "order = %d / must be less than 12" , order ); 290 291 // get local pointer on relevant KCM allocator 292 kcm_ptr = &LOCAL_CLUSTER->kcm[order - 6]; 293 294 // build extended pointer on local KCM lock 295 xptr_t lock_xp = XPTR( local_cxy , &kcm_ptr->lock ); 296 297 // get KCM lock 298 remote_busylock_acquire( lock_xp ); 299 300 // get a non-full kcm_page 301 kcm_page = kcm_get_page( kcm_ptr ); 302 303 if( kcm_page == NULL ) 304 { 305 remote_busylock_release( lock_xp ); 306 return NULL; 307 } 308 309 // get a block from selected active page 310 block_ptr = kcm_get_block( kcm_ptr , kcm_page ); 311 312 // release lock 313 remote_busylock_release( lock_xp ); 314 315 #if DEBUG_KCM 316 thread_t * this = CURRENT_THREAD; 317 uint32_t cycle = (uint32_t)hal_get_cycles(); 318 if( DEBUG_KCM < cycle ) 319 printk("\n[%s] thread[%x,%x] allocated block %x / order %d / kcm %x / status[%x,%x] / count %d\n", 320 __FUNCTION__, this->process->pid, this->trdid, block_ptr, order, kcm_ptr, 321 GET_CXY(kcm_page->status), GET_PTR(kcm_page->status), kcm_page->count ); 322 #endif 323 324 return block_ptr; 325 326 } // end kcm_alloc() 327 328 ///////////////////////////////// 329 void kcm_free( void * block_ptr ) 330 { 331 kcm_t * kcm_ptr; 332 kcm_page_t * kcm_page; 333 334 // check argument 335 assert( (block_ptr != NULL) , "block pointer cannot be NULL" ); 336 337 // get local pointer on KCM page 338 kcm_page = (kcm_page_t *)((intptr_t)block_ptr & ~CONFIG_PPM_PAGE_MASK); 339 340 // get local pointer on KCM descriptor 341 kcm_ptr = kcm_page->kcm; 342 343 #if DEBUG_KCM 344 thread_t * this = CURRENT_THREAD; 345 uint32_t cycle = (uint32_t)hal_get_cycles(); 346 if( DEBUG_KCM < cycle ) 347 printk("\n[%s] thread[%x,%x] release block %x / order %d / kcm %x / status [%x,%x] / count %d\n", 348 __FUNCTION__, this->process->pid, this->trdid, block_ptr, kcm_ptr->order, kcm_ptr, 349 GET_CXY(kcm_page->status), GET_PTR(kcm_page->status), kcm_page->count ); 350 #endif 351 352 // build extended pointer on local KCM lock 353 xptr_t lock_xp = XPTR( local_cxy , &kcm_ptr->lock ); 306 354 307 355 // get lock 308 busylock_acquire( &kcm->lock ); 309 310 // get an active page 311 if( list_is_empty( &kcm->active_root ) ) // no active page => get one 312 { 313 // get a page from free list 314 kcm_page = freelist_get( kcm ); 315 316 if( kcm_page == NULL ) 317 { 318 busylock_release( &kcm->lock ); 319 return NULL; 320 } 321 322 // insert page in active list 323 list_add_first( &kcm->active_root , &kcm_page->list ); 324 kcm->active_pages_nr ++; 325 kcm_page->active = 1; 326 } 327 else // get first page from active list 328 { 329 // get page pointer from active list 330 kcm_page = (kcm_page_t *)LIST_FIRST( &kcm->active_root , kcm_page_t , list ); 356 remote_busylock_acquire( lock_xp ); 357 358 // release block 359 kcm_put_block( kcm_ptr , kcm_page , block_ptr ); 360 361 // release lock 362 remote_busylock_release( lock_xp ); 363 } 364 365 ///////////////////////////////////////////////////////////////////////////////////// 366 // Remote access functions 367 ///////////////////////////////////////////////////////////////////////////////////// 368 369 ///////////////////////////////////////////////////////////////////////////////////// 370 // This static function can be called by any thread running in any cluster. 371 // It returns a local pointer on a block allocated from an non-full kcm_page. 372 // It makes a panic if no block available in selected page. 373 // It changes the page status as required. 374 ///////////////////////////////////////////////////////////////////////////////////// 375 // @ kcm_cxy : remote KCM cluster identidfier. 376 // @ kcm_ptr : local pointer on remote KCM allocator. 377 // @ kcm_page : pointer on active kcm page to use. 378 // @ return a local pointer on the allocated block. 379 ///////////////////////////////////////////////////////////////////////////////////// 380 static void * __attribute__((noinline)) kcm_remote_get_block( cxy_t kcm_cxy, 381 kcm_t * kcm_ptr, 382 kcm_page_t * kcm_page ) 383 { 384 uint32_t order = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->order ) ); 385 uint32_t max = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->max_blocks ) ); 386 uint32_t count = hal_remote_l32( XPTR( kcm_cxy , &kcm_page->count ) ); 387 uint64_t status = hal_remote_l64( XPTR( kcm_cxy , &kcm_page->status ) ); 388 uint32_t size = 1 << order; 389 390 assert( (count < max) , "kcm_page should not be full" ); 391 392 uint32_t index = 1; 393 uint64_t mask = (uint64_t)0x2; 394 uint32_t found = 0; 395 396 // allocate first free block in kcm_page, update status, 397 // and count , compute index of allocated block in kcm_page 398 while( index <= max ) 399 { 400 if( (status & mask) == 0 ) // block non allocated 401 { 402 hal_remote_s64( XPTR( kcm_cxy , &kcm_page->status ) , status | mask ); 403 hal_remote_s64( XPTR( kcm_cxy , &kcm_page->count ) , count + 1 ); 404 found = 1; 405 break; 406 } 407 408 index++; 409 mask <<= 1; 410 } 411 412 // change the page list if almost full 413 if( count == max-1 ) 414 { 415 list_remote_unlink( kcm_cxy , &kcm_page->list ); 416 hal_remote_atomic_add( XPTR( kcm_cxy , &kcm_ptr->active_pages_nr ) , -1 ); 417 418 list_remote_add_first( kcm_cxy , &kcm_ptr->full_root , &kcm_page->list ); 419 hal_remote_atomic_add( XPTR( kcm_cxy , &kcm_ptr->full_pages_nr ) , 1 ); 420 } 421 422 // compute return pointer 423 void * ptr = (void *)((intptr_t)kcm_page + (index * size) ); 424 425 #if DEBUG_KCM_REMOTE 426 thread_t * this = CURRENT_THREAD; 427 uint32_t cycle = (uint32_t)hal_get_cycles(); 428 if( DEBUG_KCM_REMOTE < cycle ) 429 printk("\n[%s] thread[%x,%x] get block %x in page %x / cluster %x / size %x / count %d\n", 430 __FUNCTION__, this->process->pid, this->trdid, 431 ptr, kcm_page, kcm_cxy, size, count + 1 ); 432 #endif 433 434 return ptr; 435 436 } // end kcm_remote_get_block() 437 438 ///////////////////////////////////////////////////////////////////////////////////// 439 // This private static function can be called by any thread running in any cluster. 440 // It releases a previously allocated block to the relevant kcm_page. 441 // It changes the kcm_page status as required. 442 ///////////////////////////////////////////////////////////////////////////////////// 443 // @ kcm_cxy : remote KCM cluster identifier 444 // @ kcm_ptr : local pointer on remote KCM. 445 // @ kcm_page : local pointer on kcm_page. 446 // @ block_ptr : pointer on block to be released. 447 ///////////////////////////////////////////////////////////////////////////////////// 448 static void __attribute__((noinline)) kcm_remote_put_block ( cxy_t kcm_cxy, 449 kcm_t * kcm_ptr, 450 kcm_page_t * kcm_page, 451 void * block_ptr ) 452 { 453 uint32_t max = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->max_blocks ) ); 454 uint32_t order = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->order ) ); 455 uint32_t count = hal_remote_l32( XPTR( kcm_cxy , &kcm_page->count ) ); 456 uint64_t status = hal_remote_l64( XPTR( kcm_cxy , &kcm_page->status ) ); 457 uint32_t size = 1 << order; 458 459 // compute block index from block pointer 460 uint32_t index = ((intptr_t)block_ptr - (intptr_t)kcm_page) / size; 461 462 // compute mask in bit vector 463 uint64_t mask = 1 << index; 464 465 assert( (status & mask) , "released page not allocated" ); 466 467 // update status & count in kcm_page 468 hal_remote_s64( XPTR( kcm_cxy , &kcm_page->status ) , status & ~mask ); 469 hal_remote_s32( XPTR( kcm_cxy , &kcm_page->count ) , count - 1 ); 470 471 // change the page list if page was full 472 if( count == max ) 473 { 474 list_remote_unlink( kcm_cxy , &kcm_page->list ); 475 hal_remote_atomic_add( XPTR( kcm_cxy , &kcm_ptr->full_pages_nr ) , -1 ); 476 477 list_remote_add_last( kcm_cxy , &kcm_ptr->active_root, &kcm_page->list ); 478 hal_remote_atomic_add( XPTR( kcm_cxy , &kcm_ptr->active_pages_nr ) , 1 ); 479 } 480 481 #if (DEBUG_KCM_REMOTE & 1) 482 thread_t * this = CURRENT_THREAD; 483 uint32_t cycle = (uint32_t)hal_get_cycles(); 484 if( DEBUG_KCM_REMOTE < cycle ) 485 printk("\n[%s] thread[%x,%x] released block %x in page %x / cluster %x / size %x / count %d\n", 486 __FUNCTION__, this->process->pid, this->trdid, block_ptr, kcm_page, size, count - 1 ) 487 #endif 488 489 } // end kcm_remote_put_block() 490 491 ///////////////////////////////////////////////////////////////////////////////////// 492 // This private static function can be called by any thread running in any cluster. 493 // It gets one non-full KCM page from the remote KCM. 494 // It allocates a page from remote PPM to populate the freelist, and initialises 495 // the kcm_page descriptor when required. 496 ///////////////////////////////////////////////////////////////////////////////////// 497 static kcm_page_t * __attribute__((noinline)) kcm_remote_get_page( cxy_t kcm_cxy, 498 kcm_t * kcm_ptr ) 499 { 500 kcm_page_t * kcm_page; // local pointer on remote KCM page 501 502 uint32_t active_pages_nr = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->active_pages_nr ) ); 503 504 if( active_pages_nr > 0 ) // return first active page 505 { 506 kcm_page = LIST_REMOTE_FIRST( kcm_cxy , &kcm_ptr->active_root , kcm_page_t , list ); 507 } 508 else // allocate a new page from PPM 509 { 510 // get one 4 Kbytes page from remote PPM 511 page_t * page = ppm_remote_alloc_pages( kcm_cxy , 0 ); 512 513 if( page == NULL ) 514 { 515 printk("\n[ERROR] in %s : failed to allocate page in cluster %x\n", 516 __FUNCTION__ , kcm_cxy ); 517 518 return NULL; 519 } 520 521 // get remote page base address 522 xptr_t base_xp = ppm_page2base( XPTR( kcm_cxy , page ) ); 523 524 // get local pointer on kcm_page 525 kcm_page = GET_PTR( base_xp ); 526 527 // initialize kcm_page descriptor 528 hal_remote_s32( XPTR( kcm_cxy , &kcm_page->count ) , 0 ); 529 hal_remote_s64( XPTR( kcm_cxy , &kcm_page->status ) , 0 ); 530 hal_remote_spt( XPTR( kcm_cxy , &kcm_page->kcm ) , kcm_ptr ); 531 hal_remote_spt( XPTR( kcm_cxy , &kcm_page->page ) , page ); 532 533 // introduce new page in remote KCM active_list 534 list_remote_add_first( kcm_cxy , &kcm_ptr->active_root , &kcm_page->list ); 535 hal_remote_atomic_add( XPTR( kcm_cxy , &kcm_ptr->active_pages_nr ) , 1 ); 536 } 537 538 return kcm_page; 539 540 } // end kcm_remote_get_page() 541 542 ///////////////////////////////////////// 543 void * kcm_remote_alloc( cxy_t kcm_cxy, 544 uint32_t order ) 545 { 546 kcm_t * kcm_ptr; 547 kcm_page_t * kcm_page; 548 void * block_ptr; 549 550 if( order < 6 ) order = 6; 551 552 assert( (order < 12) , "order = %d / must be less than 12" , order ); 553 554 // get local pointer on relevant KCM allocator 555 kcm_ptr = &LOCAL_CLUSTER->kcm[order - 6]; 556 557 // build extended pointer on remote KCM lock 558 xptr_t lock_xp = XPTR( kcm_cxy , &kcm_ptr->lock ); 559 560 // get lock 561 remote_busylock_acquire( lock_xp ); 562 563 // get a non-full kcm_page 564 kcm_page = kcm_remote_get_page( kcm_cxy , kcm_ptr ); 565 566 if( kcm_page == NULL ) 567 { 568 remote_busylock_release( lock_xp ); 569 return NULL; 331 570 } 332 571 333 572 // get a block from selected active page 334 // cannot fail, as an active page cannot be full... 335 ptr = kcm_get_block( kcm , kcm_page ); 573 block_ptr = kcm_remote_get_block( kcm_cxy , kcm_ptr , kcm_page ); 336 574 337 575 // release lock 338 busylock_release( &kcm->lock ); 339 340 return ptr; 576 remote_busylock_release( lock_xp ); 577 578 #if DEBUG_KCM_REMOTE 579 thread_t * this = CURRENT_THREAD; 580 uint32_t cycle = (uint32_t)hal_get_cycles(); 581 if( DEBUG_KCM_REMOTE < cycle ) 582 printk("\n[%s] thread[%x,%x] allocated block %x / order %d / kcm[%x,%x]\n", 583 __FUNCTION__, this->process->pid, this->trdid, block_ptr, order, kcm_cxy, kcm_ptr ); 584 #endif 585 586 return block_ptr; 587 588 } // end kcm_remote_alloc() 589 590 ///////////////////////////////////// 591 void kcm_remote_free( cxy_t kcm_cxy, 592 void * block_ptr ) 593 { 594 kcm_t * kcm_ptr; 595 kcm_page_t * kcm_page; 596 597 // check argument 598 assert( (block_ptr != NULL) , "block pointer cannot be NULL" ); 599 600 // get local pointer on remote KCM page 601 kcm_page = (kcm_page_t *)((intptr_t)block_ptr & ~CONFIG_PPM_PAGE_MASK); 602 603 // get local pointer on remote KCM 604 kcm_ptr = hal_remote_lpt( XPTR( kcm_cxy , &kcm_page->kcm ) ); 605 606 // build extended pointer on remote KCM lock 607 xptr_t lock_xp = XPTR( kcm_cxy , &kcm_ptr->lock ); 608 609 // get lock 610 remote_busylock_acquire( lock_xp ); 611 612 // release block 613 kcm_remote_put_block( kcm_cxy , kcm_ptr , kcm_page , block_ptr ); 614 615 // release lock 616 remote_busylock_release( lock_xp ); 617 618 #if DEBUG_KCM_REMOTE 619 thread_t * this = CURRENT_THREAD; 620 uint32_t cycle = (uint32_t)hal_get_cycles(); 621 uint32_t order = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->order ) ); 622 if( DEBUG_KCM_REMOTE < cycle ) 623 printk("\n[%s] thread[%x,%x] released block %x / order %d / kcm[%x,%x]\n", 624 __FUNCTION__, this->process->pid, this->trdid, block_ptr, order, kcm_cxy, kcm_ptr ); 625 #endif 626 627 } // end kcm_remote_free 628 629 ///////////////////////////////////////// 630 void kcm_remote_display( cxy_t kcm_cxy, 631 kcm_t * kcm_ptr ) 632 { 633 uint32_t order = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->order) ); 634 uint32_t full_pages_nr = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->full_pages_nr ) ); 635 uint32_t active_pages_nr = hal_remote_l32( XPTR( kcm_cxy , &kcm_ptr->active_pages_nr ) ); 636 637 printk("*** KCM / cxy %x / order %d / full_pages %d / empty_pages %d / active_pages %d\n", 638 kcm_cxy, order, full_pages_nr, active_pages_nr ); 341 639 } 342 343 ///////////////////////////344 void kcm_free( void * ptr )345 {346 kcm_page_t * kcm_page;347 kcm_t * kcm;348 349 // check argument350 assert( (ptr != NULL) , "pointer cannot be NULL" );351 352 kcm_page = (kcm_page_t *)((intptr_t)ptr & ~CONFIG_PPM_PAGE_MASK);353 kcm = kcm_page->kcm;354 355 // get lock356 busylock_acquire( &kcm->lock );357 358 // release block359 kcm_put_block( kcm , kcm_page , ptr );360 361 // release lock362 busylock_release( &kcm->lock );363 }364 365 ////////////////////////////366 void kcm_print (kcm_t * kcm)367 {368 printk("*** KCM type = %s / free_pages = %d / busy_pages = %d / active_pages = %d\n",369 kmem_type_str( kcm->type ) ,370 kcm->free_pages_nr ,371 kcm->busy_pages_nr ,372 kcm->active_pages_nr );373 }
Note: See TracChangeset
for help on using the changeset viewer.