[1] | 1 | /* |
---|
| 2 | * kcm.c - Per cluster & per type Kernel Cache Manager access functions |
---|
[18] | 3 | * |
---|
[1] | 4 | * Author Ghassan Almaless (2008,2009,2010,2011,2012) |
---|
[50] | 5 | * Alain Greiner (2016,2017) |
---|
[1] | 6 | * |
---|
| 7 | * Copyright (c) UPMC Sorbonne Universites |
---|
| 8 | * |
---|
| 9 | * This file is part of ALMOS-MKH. |
---|
| 10 | * |
---|
| 11 | * ALMOS-MKH is free software; you can redistribute it and/or modify it |
---|
| 12 | * under the terms of the GNU General Public License as published by |
---|
| 13 | * the Free Software Foundation; version 2.0 of the License. |
---|
| 14 | * |
---|
| 15 | * ALMOS-MKH is distributed in the hope that it will be useful, but |
---|
| 16 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
| 17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
---|
| 18 | * General Public License for more details. |
---|
| 19 | * |
---|
| 20 | * You should have received a copy of the GNU General Public License |
---|
| 21 | * along with ALMOS-MKH; if not, write to the Free Software Foundation, |
---|
| 22 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA |
---|
| 23 | */ |
---|
| 24 | |
---|
[14] | 25 | #include <kernel_config.h> |
---|
[1] | 26 | #include <hal_types.h> |
---|
| 27 | #include <hal_special.h> |
---|
| 28 | #include <list.h> |
---|
| 29 | #include <printk.h> |
---|
| 30 | #include <bits.h> |
---|
| 31 | #include <ppm.h> |
---|
| 32 | #include <thread.h> |
---|
| 33 | #include <page.h> |
---|
| 34 | #include <cluster.h> |
---|
[7] | 35 | #include <kmem.h> |
---|
[1] | 36 | #include <kcm.h> |
---|
| 37 | |
---|
| 38 | ////////////////////////////////////////////////////////////////////////////////////// |
---|
[7] | 39 | // This static function returns pointer on an allocated block from an active page. |
---|
[1] | 40 | // It returns NULL if no block available in selected page. |
---|
| 41 | // It changes the page status if required. |
---|
| 42 | ////////////////////////////////////////////////////////////////////////////////////// |
---|
[50] | 43 | // @ kcm : pointer on kcm allocator. |
---|
| 44 | // @ kcm_page : pointer on active kcm page to use. |
---|
[7] | 45 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
| 46 | static void * kcm_get_block( kcm_t * kcm, |
---|
[50] | 47 | kcm_page_t * kcm_page ) |
---|
[1] | 48 | { |
---|
[50] | 49 | kcm_dmsg("\n[INFO] %s : enters for %s / page %x / count = %d / active = %d\n", |
---|
| 50 | __FUNCTION__ , kmem_type_str( kcm->type ) , |
---|
[161] | 51 | (intptr_t)kcm_page , kcm_page->count , kcm_page->active ); |
---|
[1] | 52 | |
---|
[50] | 53 | assert( kcm_page->active , __FUNCTION__ , "kcm_page should be active" ); |
---|
| 54 | |
---|
[20] | 55 | // get first block available |
---|
[50] | 56 | int32_t index = bitmap_ffs( kcm_page->bitmap , kcm->blocks_nr ); |
---|
[1] | 57 | |
---|
[50] | 58 | assert( (index != -1) , __FUNCTION__ , "kcm_page should not be full" ); |
---|
[18] | 59 | |
---|
[20] | 60 | // allocate block |
---|
[50] | 61 | bitmap_clear( kcm_page->bitmap , index ); |
---|
[7] | 62 | |
---|
[50] | 63 | // increase kcm_page count |
---|
| 64 | kcm_page->count ++; |
---|
[1] | 65 | |
---|
[50] | 66 | // change the kcm_page to busy if no more free block in page |
---|
| 67 | if( kcm_page->count >= kcm->blocks_nr ) |
---|
[20] | 68 | { |
---|
[161] | 69 | kcm_page->active = 0; |
---|
[50] | 70 | list_unlink( &kcm_page->list); |
---|
[1] | 71 | kcm->active_pages_nr --; |
---|
| 72 | |
---|
[50] | 73 | list_add_first( &kcm->busy_root , &kcm_page->list); |
---|
[1] | 74 | kcm->busy_pages_nr ++; |
---|
[50] | 75 | kcm_page->busy = 1; |
---|
[20] | 76 | } |
---|
[1] | 77 | |
---|
[161] | 78 | // compute return pointer |
---|
| 79 | void * ptr = (void *)((intptr_t)kcm_page + CONFIG_KCM_SLOT_SIZE |
---|
| 80 | + (index * kcm->block_size) ); |
---|
[1] | 81 | |
---|
[65] | 82 | kcm_dmsg("\n[INFO] %s : allocated one block %s / ptr = %p / page = %x / count = %d\n", |
---|
| 83 | __FUNCTION__ , kmem_type_str( kcm->type ) , ptr , |
---|
[161] | 84 | (intptr_t)kcm_page , kcm_page->count ); |
---|
[50] | 85 | |
---|
| 86 | return ptr; |
---|
[161] | 87 | } |
---|
[50] | 88 | |
---|
[1] | 89 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
| 90 | // This static function releases a previously allocated block. |
---|
[50] | 91 | // It changes the kcm_page status if required. |
---|
[1] | 92 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
[352] | 93 | // @ kcm : pointer on kcm allocator. |
---|
| 94 | // @ kcm_page : pointer on kcm_page. |
---|
| 95 | // @ ptr : pointer on block to be released. |
---|
[7] | 96 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
[352] | 97 | static void kcm_put_block ( kcm_t * kcm, |
---|
| 98 | kcm_page_t * kcm_page, |
---|
| 99 | void * ptr ) |
---|
[1] | 100 | { |
---|
[20] | 101 | uint32_t index; |
---|
[18] | 102 | |
---|
[161] | 103 | // compute block index from block pointer |
---|
[50] | 104 | index = ((uint8_t *)ptr - (uint8_t *)kcm_page - CONFIG_KCM_SLOT_SIZE) / kcm->block_size; |
---|
[18] | 105 | |
---|
[176] | 106 | assert( !bitmap_state( kcm_page->bitmap , index ) , __FUNCTION__ , "page already freed" ); |
---|
| 107 | assert( (kcm_page->count > 0) , __FUNCTION__ , "count already zero" ); |
---|
| 108 | |
---|
[50] | 109 | bitmap_set( kcm_page->bitmap , index ); |
---|
| 110 | kcm_page->count --; |
---|
| 111 | |
---|
[20] | 112 | // change the page to active if it was busy |
---|
[50] | 113 | if( kcm_page->busy ) |
---|
[1] | 114 | { |
---|
[50] | 115 | kcm_page->busy = 0; |
---|
| 116 | list_unlink( &kcm_page->list ); |
---|
[1] | 117 | kcm->busy_pages_nr --; |
---|
| 118 | |
---|
[50] | 119 | list_add_last( &kcm->active_root, &kcm_page->list ); |
---|
[1] | 120 | kcm->active_pages_nr ++; |
---|
[50] | 121 | kcm_page->active = 1; |
---|
[1] | 122 | } |
---|
| 123 | |
---|
[50] | 124 | // change the kcm_page to free if last block in active page |
---|
| 125 | if( (kcm_page->active) && (kcm_page->count == 0) ) |
---|
[1] | 126 | { |
---|
[50] | 127 | kcm_page->active = 0; |
---|
| 128 | list_unlink( &kcm_page->list); |
---|
[1] | 129 | kcm->active_pages_nr --; |
---|
| 130 | |
---|
[50] | 131 | list_add_first( &kcm->free_root , &kcm_page->list); |
---|
[1] | 132 | kcm->free_pages_nr ++; |
---|
| 133 | } |
---|
[161] | 134 | } |
---|
[1] | 135 | |
---|
| 136 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
[7] | 137 | // This static function allocates one page from PPM. It initializes |
---|
[50] | 138 | // the kcm_page descriptor, and introduces the new kcm_page into freelist. |
---|
[1] | 139 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
| 140 | static error_t freelist_populate( kcm_t * kcm ) |
---|
| 141 | { |
---|
| 142 | page_t * page; |
---|
[50] | 143 | kcm_page_t * kcm_page; |
---|
[20] | 144 | kmem_req_t req; |
---|
[1] | 145 | |
---|
[20] | 146 | // get one page from local PPM |
---|
| 147 | req.type = KMEM_PAGE; |
---|
| 148 | req.size = 0; |
---|
| 149 | req.flags = AF_KERNEL; |
---|
| 150 | page = kmem_alloc( &req ); |
---|
[18] | 151 | |
---|
[7] | 152 | if( page == NULL ) |
---|
| 153 | { |
---|
[18] | 154 | printk("\n[ERROR] in %s : failed to allocate page in cluster %d\n", |
---|
[20] | 155 | __FUNCTION__ , local_cxy ); |
---|
| 156 | return ENOMEM; |
---|
[7] | 157 | } |
---|
| 158 | |
---|
[20] | 159 | // get page base address |
---|
[315] | 160 | xptr_t base_xp = ppm_page2base( XPTR( local_cxy , page ) ); |
---|
| 161 | kcm_page = (kcm_page_t *)GET_PTR( base_xp ); |
---|
[1] | 162 | |
---|
[20] | 163 | // initialize KCM-page descriptor |
---|
[50] | 164 | bitmap_set_range( kcm_page->bitmap , 0 , kcm->blocks_nr ); |
---|
[1] | 165 | |
---|
[50] | 166 | kcm_page->busy = 0; |
---|
| 167 | kcm_page->active = 0; |
---|
| 168 | kcm_page->count = 0; |
---|
| 169 | kcm_page->kcm = kcm; |
---|
| 170 | kcm_page->page = page; |
---|
[1] | 171 | |
---|
[20] | 172 | // introduce new page in free-list |
---|
[50] | 173 | list_add_first( &kcm->free_root , &kcm_page->list ); |
---|
[1] | 174 | kcm->free_pages_nr ++; |
---|
[18] | 175 | |
---|
[1] | 176 | return 0; |
---|
[161] | 177 | } |
---|
[1] | 178 | |
---|
| 179 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
[20] | 180 | // This private function gets one KCM page from the KCM freelist. |
---|
[1] | 181 | // It populates the freelist if required. |
---|
| 182 | ///////////////////////////////////////////////////////////////////////////////////// |
---|
| 183 | static kcm_page_t * freelist_get( kcm_t * kcm ) |
---|
| 184 | { |
---|
[7] | 185 | error_t error; |
---|
[50] | 186 | kcm_page_t * kcm_page; |
---|
[1] | 187 | |
---|
[20] | 188 | // get a new page from PPM if freelist empty |
---|
[1] | 189 | if( kcm->free_pages_nr == 0 ) |
---|
| 190 | { |
---|
[20] | 191 | error = freelist_populate( kcm ); |
---|
| 192 | if( error ) return NULL; |
---|
[1] | 193 | } |
---|
| 194 | |
---|
[50] | 195 | // get first KCM page from freelist and unlink it |
---|
| 196 | kcm_page = LIST_FIRST( &kcm->free_root, kcm_page_t , list ); |
---|
| 197 | list_unlink( &kcm_page->list ); |
---|
[1] | 198 | kcm->free_pages_nr --; |
---|
| 199 | |
---|
[50] | 200 | return kcm_page; |
---|
[161] | 201 | } |
---|
[1] | 202 | |
---|
[7] | 203 | ////////////////////////////// |
---|
| 204 | void kcm_init( kcm_t * kcm, |
---|
| 205 | uint32_t type ) |
---|
[1] | 206 | { |
---|
[161] | 207 | // the kcm_page descriptor mut fit in the KCM slot |
---|
| 208 | assert( (sizeof(kcm_page_t) <= CONFIG_KCM_SLOT_SIZE) , |
---|
| 209 | __FUNCTION__ , "KCM slot too small\n" ); |
---|
[1] | 210 | |
---|
[20] | 211 | // initialize lock |
---|
[1] | 212 | spinlock_init( &kcm->lock ); |
---|
| 213 | |
---|
[20] | 214 | // initialize KCM type |
---|
[1] | 215 | kcm->type = type; |
---|
| 216 | |
---|
[20] | 217 | // initialize KCM page lists |
---|
[1] | 218 | kcm->free_pages_nr = 0; |
---|
| 219 | kcm->busy_pages_nr = 0; |
---|
| 220 | kcm->active_pages_nr = 0; |
---|
| 221 | list_root_init( &kcm->free_root ); |
---|
| 222 | list_root_init( &kcm->busy_root ); |
---|
| 223 | list_root_init( &kcm->active_root ); |
---|
| 224 | |
---|
[161] | 225 | // initialize block size |
---|
[50] | 226 | uint32_t block_size = ARROUND_UP( kmem_type_size( type ) , CONFIG_KCM_SLOT_SIZE ); |
---|
[1] | 227 | kcm->block_size = block_size; |
---|
[18] | 228 | |
---|
[50] | 229 | // initialize number of blocks per page |
---|
| 230 | uint32_t blocks_nr = (CONFIG_PPM_PAGE_SIZE - CONFIG_KCM_SLOT_SIZE) / block_size; |
---|
[161] | 231 | kcm->blocks_nr = blocks_nr; |
---|
[50] | 232 | |
---|
[20] | 233 | kcm_dmsg("\n[INFO] %s : KCM %s initialised / block_size = %d / blocks_nr = %d\n", |
---|
[50] | 234 | __FUNCTION__ , kmem_type_str( type ) , kcm->block_size , kcm->blocks_nr ); |
---|
[161] | 235 | } |
---|
[1] | 236 | |
---|
| 237 | /////////////////////////////// |
---|
| 238 | void kcm_destroy( kcm_t * kcm ) |
---|
| 239 | { |
---|
[50] | 240 | kcm_page_t * kcm_page; |
---|
[1] | 241 | list_entry_t * iter; |
---|
[18] | 242 | |
---|
[20] | 243 | // get KCM lock |
---|
[1] | 244 | spinlock_lock( &kcm->lock ); |
---|
| 245 | |
---|
[20] | 246 | // release all free pages |
---|
[1] | 247 | LIST_FOREACH( &kcm->free_root , iter ) |
---|
| 248 | { |
---|
[50] | 249 | kcm_page = (kcm_page_t *)LIST_ELEMENT( iter , kcm_page_t , list ); |
---|
[1] | 250 | list_unlink( iter ); |
---|
| 251 | kcm->free_pages_nr --; |
---|
[50] | 252 | ppm_free_pages( kcm_page->page ); |
---|
[1] | 253 | } |
---|
| 254 | |
---|
[20] | 255 | // release all active pages |
---|
[1] | 256 | LIST_FOREACH( &kcm->active_root , iter ) |
---|
| 257 | { |
---|
[50] | 258 | kcm_page = (kcm_page_t *)LIST_ELEMENT( iter , kcm_page_t , list ); |
---|
[1] | 259 | list_unlink( iter ); |
---|
| 260 | kcm->free_pages_nr --; |
---|
[50] | 261 | ppm_free_pages( kcm_page->page ); |
---|
[1] | 262 | } |
---|
| 263 | |
---|
[20] | 264 | // release all busy pages |
---|
[1] | 265 | LIST_FOREACH( &kcm->busy_root , iter ) |
---|
| 266 | { |
---|
[50] | 267 | kcm_page = (kcm_page_t *)LIST_ELEMENT( iter , kcm_page_t , list ); |
---|
[1] | 268 | list_unlink( iter ); |
---|
| 269 | kcm->free_pages_nr --; |
---|
[50] | 270 | ppm_free_pages( kcm_page->page ); |
---|
[1] | 271 | } |
---|
| 272 | |
---|
[20] | 273 | // release KCM lock |
---|
| 274 | spinlock_unlock( &kcm->lock ); |
---|
[161] | 275 | } |
---|
[1] | 276 | |
---|
| 277 | /////////////////////////////// |
---|
| 278 | void * kcm_alloc( kcm_t * kcm ) |
---|
| 279 | { |
---|
[50] | 280 | kcm_page_t * kcm_page; |
---|
[1] | 281 | void * ptr = NULL; // pointer on block |
---|
| 282 | |
---|
[20] | 283 | // get lock |
---|
[1] | 284 | spinlock_lock( &kcm->lock ); |
---|
[18] | 285 | |
---|
[20] | 286 | // get an active page |
---|
| 287 | if( list_is_empty( &kcm->active_root ) ) // no active page => get one |
---|
| 288 | { |
---|
| 289 | // get a page from free list |
---|
[50] | 290 | kcm_page = freelist_get( kcm ); |
---|
[7] | 291 | |
---|
[182] | 292 | if( kcm_page == NULL ) |
---|
| 293 | { |
---|
| 294 | spinlock_unlock( &kcm->lock ); |
---|
| 295 | return NULL; |
---|
| 296 | } |
---|
[50] | 297 | |
---|
[20] | 298 | // insert page in active list |
---|
[50] | 299 | list_add_first( &kcm->active_root , &kcm_page->list ); |
---|
[20] | 300 | kcm->active_pages_nr ++; |
---|
[161] | 301 | kcm_page->active = 1; |
---|
[50] | 302 | |
---|
[161] | 303 | kcm_dmsg("\n[INFO] %s : enters for type %s at cycle %d / new page = %x / count = %d\n", |
---|
[101] | 304 | __FUNCTION__ , kmem_type_str( kcm->type ) , hal_get_cycles() , |
---|
[161] | 305 | (intptr_t)kcm_page , kcm_page->count ); |
---|
[50] | 306 | |
---|
[20] | 307 | } |
---|
[50] | 308 | else // get first page from active list |
---|
[20] | 309 | { |
---|
[50] | 310 | // get page pointer from active list |
---|
| 311 | kcm_page = (kcm_page_t *)LIST_FIRST( &kcm->active_root , kcm_page_t , list ); |
---|
[7] | 312 | |
---|
[50] | 313 | kcm_dmsg("\n[INFO] %s : enters for type %s at cycle %d / page = %x / count = %d\n", |
---|
[161] | 314 | __FUNCTION__ , kmem_type_str( kcm->type ) , hal_get_cycles() , |
---|
| 315 | (intptr_t)kcm_page , kcm_page->count ); |
---|
[20] | 316 | } |
---|
[1] | 317 | |
---|
[20] | 318 | // get a block from selected active page |
---|
| 319 | // cannot fail, as an active page cannot be full... |
---|
[50] | 320 | ptr = kcm_get_block( kcm , kcm_page ); |
---|
[7] | 321 | |
---|
[20] | 322 | // release lock |
---|
[50] | 323 | spinlock_unlock( &kcm->lock ); |
---|
[1] | 324 | |
---|
| 325 | return ptr; |
---|
[161] | 326 | } |
---|
[1] | 327 | |
---|
| 328 | /////////////////////////// |
---|
| 329 | void kcm_free( void * ptr ) |
---|
| 330 | { |
---|
[50] | 331 | kcm_page_t * kcm_page; |
---|
[1] | 332 | kcm_t * kcm; |
---|
[18] | 333 | |
---|
[50] | 334 | assert( (ptr != NULL) , __FUNCTION__ , "pointer cannot be NULL" ); |
---|
[18] | 335 | |
---|
[50] | 336 | kcm_page = (kcm_page_t *)((intptr_t)ptr & ~CONFIG_PPM_PAGE_MASK); |
---|
| 337 | kcm = kcm_page->kcm; |
---|
[1] | 338 | |
---|
[20] | 339 | // get lock |
---|
[1] | 340 | spinlock_lock( &kcm->lock ); |
---|
| 341 | |
---|
[20] | 342 | // release block |
---|
[352] | 343 | kcm_put_block( kcm , kcm_page , ptr ); |
---|
[1] | 344 | |
---|
[20] | 345 | // release lock |
---|
[1] | 346 | spinlock_unlock( &kcm->lock ); |
---|
[161] | 347 | } |
---|
[1] | 348 | |
---|
| 349 | //////////////////////////// |
---|
| 350 | void kcm_print (kcm_t * kcm) |
---|
| 351 | { |
---|
[7] | 352 | printk("*** KCM type = %s / free_pages = %d / busy_pages = %d / active_pages = %d\n", |
---|
[20] | 353 | kmem_type_str( kcm->type ) , |
---|
| 354 | kcm->free_pages_nr , |
---|
| 355 | kcm->busy_pages_nr , |
---|
| 356 | kcm->active_pages_nr ); |
---|
[1] | 357 | } |
---|