[1] | 1 | /* |
---|
| 2 | * ppm.c - Per-cluster Physical Pages Manager implementation |
---|
| 3 | * |
---|
| 4 | * Authors Ghassan Almaless (2008,2009,2010,2011,2012) |
---|
[50] | 5 | * Alain Greiner (2016,2017) |
---|
[1] | 6 | * |
---|
| 7 | * Copyright (c) UPMC Sorbonne Universites |
---|
| 8 | * |
---|
| 9 | * This file is part of ALMOS-MKH. |
---|
| 10 | * |
---|
| 11 | * ALMOS-MKH.is free software; you can redistribute it and/or modify it |
---|
| 12 | * under the terms of the GNU General Public License as published by |
---|
| 13 | * the Free Software Foundation; version 2.0 of the License. |
---|
| 14 | * |
---|
| 15 | * ALMOS-MKH.is distributed in the hope that it will be useful, but |
---|
| 16 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
| 17 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
---|
| 18 | * General Public License for more details. |
---|
| 19 | * |
---|
| 20 | * You should have received a copy of the GNU General Public License |
---|
| 21 | * along with ALMOS-MKH.; if not, write to the Free Software Foundation, |
---|
| 22 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA |
---|
| 23 | */ |
---|
| 24 | |
---|
[14] | 25 | #include <kernel_config.h> |
---|
[1] | 26 | #include <hal_types.h> |
---|
| 27 | #include <hal_special.h> |
---|
| 28 | #include <printk.h> |
---|
| 29 | #include <list.h> |
---|
| 30 | #include <bits.h> |
---|
| 31 | #include <page.h> |
---|
| 32 | #include <spinlock.h> |
---|
| 33 | #include <thread.h> |
---|
| 34 | #include <cluster.h> |
---|
| 35 | #include <kmem.h> |
---|
| 36 | #include <process.h> |
---|
| 37 | #include <dqdt.h> |
---|
| 38 | #include <ppm.h> |
---|
| 39 | |
---|
| 40 | //////////////////////////////////////////////// |
---|
| 41 | inline bool_t ppm_page_is_valid( page_t * page ) |
---|
| 42 | { |
---|
[160] | 43 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
[1] | 44 | uint32_t pgnr = (uint32_t)( page - ppm->pages_tbl ); |
---|
| 45 | return (pgnr <= ppm->pages_nr); |
---|
| 46 | } |
---|
| 47 | |
---|
[50] | 48 | |
---|
[315] | 49 | |
---|
[50] | 50 | ///////////////////////////////////////////// |
---|
[315] | 51 | inline xptr_t ppm_page2base( xptr_t page_xp ) |
---|
[1] | 52 | { |
---|
[315] | 53 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
[1] | 54 | |
---|
[315] | 55 | cxy_t page_cxy = GET_CXY( page_xp ); |
---|
| 56 | page_t * page_ptr = (page_t *)GET_PTR( page_xp ); |
---|
| 57 | |
---|
[406] | 58 | void * base_ptr = ppm->vaddr_base + |
---|
| 59 | ((page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT); |
---|
| 60 | |
---|
[315] | 61 | return XPTR( page_cxy , base_ptr ); |
---|
| 62 | |
---|
| 63 | } // end ppm_page2base() |
---|
| 64 | |
---|
| 65 | ///////////////////////////////////////////// |
---|
| 66 | inline xptr_t ppm_base2page( xptr_t base_xp ) |
---|
[1] | 67 | { |
---|
[315] | 68 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
[1] | 69 | |
---|
[315] | 70 | cxy_t base_cxy = GET_CXY( base_xp ); |
---|
| 71 | void * base_ptr = (void *)GET_PTR( base_xp ); |
---|
| 72 | |
---|
| 73 | page_t * page_ptr = ppm->pages_tbl + |
---|
| 74 | ((base_ptr - ppm->vaddr_base)>>CONFIG_PPM_PAGE_SHIFT); |
---|
| 75 | |
---|
| 76 | return XPTR( base_cxy , page_ptr ); |
---|
| 77 | |
---|
| 78 | } // end ppm_base2page() |
---|
| 79 | |
---|
| 80 | |
---|
| 81 | |
---|
[50] | 82 | /////////////////////////////////////////// |
---|
[315] | 83 | inline ppn_t ppm_page2ppn( xptr_t page_xp ) |
---|
| 84 | { |
---|
| 85 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
| 86 | |
---|
| 87 | cxy_t page_cxy = GET_CXY( page_xp ); |
---|
| 88 | page_t * page_ptr = (page_t *)GET_PTR( page_xp ); |
---|
| 89 | |
---|
| 90 | paddr_t paddr = PADDR( page_cxy , (page_ptr - ppm->pages_tbl)<<CONFIG_PPM_PAGE_SHIFT ); |
---|
| 91 | |
---|
| 92 | return paddr >> CONFIG_PPM_PAGE_SHIFT; |
---|
| 93 | |
---|
| 94 | } // end hal_page2ppn() |
---|
| 95 | |
---|
| 96 | /////////////////////////////////////// |
---|
| 97 | inline xptr_t ppm_ppn2page( ppn_t ppn ) |
---|
| 98 | { |
---|
| 99 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
| 100 | |
---|
| 101 | paddr_t paddr = ppn << CONFIG_PPM_PAGE_SHIFT; |
---|
| 102 | |
---|
| 103 | cxy_t page_cxy = CXY_FROM_PADDR( paddr ); |
---|
| 104 | lpa_t page_lpa = LPA_FROM_PADDR( paddr ); |
---|
| 105 | |
---|
| 106 | return XPTR( page_cxy , &ppm->pages_tbl[page_lpa>>CONFIG_PPM_PAGE_SHIFT] ); |
---|
| 107 | |
---|
| 108 | } // end hal_ppn2page |
---|
| 109 | |
---|
| 110 | |
---|
| 111 | |
---|
| 112 | /////////////////////////////////////// |
---|
| 113 | inline xptr_t ppm_ppn2base( ppn_t ppn ) |
---|
| 114 | { |
---|
| 115 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
| 116 | |
---|
| 117 | paddr_t paddr = ppn << CONFIG_PPM_PAGE_SHIFT; |
---|
| 118 | |
---|
| 119 | cxy_t page_cxy = CXY_FROM_PADDR( paddr ); |
---|
| 120 | lpa_t page_lpa = LPA_FROM_PADDR( paddr ); |
---|
| 121 | |
---|
| 122 | void * base_ptr = (void *)ppm->vaddr_base + (page_lpa & ~CONFIG_PPM_PAGE_SHIFT); |
---|
| 123 | |
---|
| 124 | return XPTR( page_cxy , base_ptr ); |
---|
| 125 | |
---|
| 126 | } // end ppm_ppn2base() |
---|
| 127 | |
---|
| 128 | /////////////////////////////////////////// |
---|
| 129 | inline ppn_t ppm_base2ppn( xptr_t base_xp ) |
---|
| 130 | { |
---|
| 131 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
| 132 | |
---|
| 133 | cxy_t base_cxy = GET_CXY( base_xp ); |
---|
| 134 | void * base_ptr = (void *)GET_PTR( base_xp ); |
---|
| 135 | |
---|
| 136 | paddr_t paddr = PADDR( base_cxy , (base_ptr - ppm->vaddr_base) ); |
---|
| 137 | |
---|
| 138 | return paddr >> CONFIG_PPM_PAGE_SHIFT; |
---|
| 139 | |
---|
| 140 | } // end ppm_base2ppn() |
---|
| 141 | |
---|
| 142 | |
---|
| 143 | |
---|
| 144 | /////////////////////////////////////////// |
---|
[50] | 145 | void ppm_free_pages_nolock( page_t * page ) |
---|
[1] | 146 | { |
---|
[7] | 147 | page_t * buddy; // searched buddy page descriptor |
---|
| 148 | uint32_t buddy_index; // buddy page index |
---|
| 149 | page_t * current; // current (merged) page descriptor |
---|
| 150 | uint32_t current_index; // current (merged) page index |
---|
[50] | 151 | uint32_t current_order; // current (merged) page order |
---|
[7] | 152 | |
---|
[160] | 153 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
| 154 | page_t * pages_tbl = ppm->pages_tbl; |
---|
[1] | 155 | |
---|
[407] | 156 | assert( !page_is_flag( page , PG_FREE ) , __FUNCTION__ , |
---|
| 157 | "page already released : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) ); |
---|
[177] | 158 | |
---|
[407] | 159 | assert( !page_is_flag( page , PG_RESERVED ) , __FUNCTION__ , |
---|
| 160 | "reserved page : ppn = %x\n" , ppm_page2ppn(XPTR(local_cxy,page)) ); |
---|
| 161 | |
---|
[160] | 162 | // update released page descriptor flags |
---|
[1] | 163 | page_set_flag( page , PG_FREE ); |
---|
| 164 | |
---|
[160] | 165 | // search the buddy page descriptor |
---|
| 166 | // - merge with current page descriptor if found |
---|
| 167 | // - exit to release the current page descriptor if not found |
---|
| 168 | current = page , |
---|
| 169 | current_index = (uint32_t)(page - ppm->pages_tbl); |
---|
[18] | 170 | for( current_order = page->order ; |
---|
[160] | 171 | current_order < CONFIG_PPM_MAX_ORDER ; |
---|
| 172 | current_order++ ) |
---|
| 173 | { |
---|
[7] | 174 | buddy_index = current_index ^ (1 << current_order); |
---|
| 175 | buddy = pages_tbl + buddy_index; |
---|
[18] | 176 | |
---|
[7] | 177 | if( !page_is_flag( buddy , PG_FREE ) || (buddy->order != current_order) ) break; |
---|
[1] | 178 | |
---|
[160] | 179 | // remove buddy from free list |
---|
[7] | 180 | list_unlink( &buddy->list ); |
---|
[1] | 181 | ppm->free_pages_nr[current_order] --; |
---|
[18] | 182 | |
---|
[160] | 183 | // merge buddy with current |
---|
[7] | 184 | buddy->order = 0; |
---|
| 185 | current_index &= buddy_index; |
---|
[1] | 186 | } |
---|
[18] | 187 | |
---|
[160] | 188 | // update merged page descriptor order |
---|
[7] | 189 | current = pages_tbl + current_index; |
---|
| 190 | current->order = current_order; |
---|
[1] | 191 | |
---|
[160] | 192 | // insert current in free list |
---|
[7] | 193 | list_add_first( &ppm->free_pages_root[current_order] , ¤t->list ); |
---|
[1] | 194 | ppm->free_pages_nr[current_order] ++; |
---|
| 195 | |
---|
[433] | 196 | } // end ppm_free_pages_nolock() |
---|
| 197 | |
---|
[1] | 198 | //////////////////////////////////////////// |
---|
| 199 | page_t * ppm_alloc_pages( uint32_t order ) |
---|
| 200 | { |
---|
[160] | 201 | uint32_t current_order; |
---|
[1] | 202 | page_t * remaining_block; |
---|
| 203 | uint32_t current_size; |
---|
[433] | 204 | |
---|
| 205 | #if CONFIG_DEBUG_PPM_ALLOC_PAGES |
---|
| 206 | uint32_t cycle = (uint32_t)hal_get_cycles(); |
---|
| 207 | if( CONFIG_DEBUG_PPM_ALLOC_PAGES < cycle ) |
---|
| 208 | printk("\n[DBG] in %s : thread %x enter for %d page(s) / cycle %d\n", |
---|
| 209 | __FUNCTION__ , CURRENT_THREAD , 1<<order, cycle ); |
---|
| 210 | #endif |
---|
[1] | 211 | |
---|
[433] | 212 | #if(CONFIG_DEBUG_PPM_ALLOC_PAGES & 0x1) |
---|
| 213 | if( CONFIG_DEBUG_PPM_ALLOC_PAGES < cycle ) |
---|
| 214 | ppm_print(); |
---|
| 215 | #endif |
---|
| 216 | |
---|
[160] | 217 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
[1] | 218 | |
---|
[407] | 219 | assert( (order < CONFIG_PPM_MAX_ORDER) , __FUNCTION__ , |
---|
| 220 | "illegal order argument = %x\n" , order ); |
---|
[1] | 221 | |
---|
[406] | 222 | page_t * block = NULL; |
---|
[1] | 223 | |
---|
[160] | 224 | // take lock protecting free lists |
---|
[1] | 225 | spinlock_lock( &ppm->free_lock ); |
---|
| 226 | |
---|
[160] | 227 | // find a free block equal or larger to requested size |
---|
[1] | 228 | for( current_order = order ; current_order < CONFIG_PPM_MAX_ORDER ; current_order ++ ) |
---|
| 229 | { |
---|
| 230 | if( !list_is_empty( &ppm->free_pages_root[current_order] ) ) |
---|
| 231 | { |
---|
| 232 | block = LIST_FIRST( &ppm->free_pages_root[current_order] , page_t , list ); |
---|
| 233 | list_unlink( &block->list ); |
---|
| 234 | break; |
---|
| 235 | } |
---|
| 236 | } |
---|
| 237 | |
---|
| 238 | if( block == NULL ) // return failure |
---|
| 239 | { |
---|
[160] | 240 | // release lock protecting free lists |
---|
| 241 | spinlock_unlock( &ppm->free_lock ); |
---|
[1] | 242 | |
---|
[433] | 243 | #if CONFIG_DEBUG_PPM_ALLOC_PAGES |
---|
| 244 | cycle = (uint32_t)hal_get_cycles(); |
---|
| 245 | if( CONFIG_DEBUG_PPM_ALLOC_PAGES < cycle ) |
---|
| 246 | printk("\n[DBG] in %s : thread %x cannot allocate %d page(s) at cycle %d\n", |
---|
| 247 | __FUNCTION__ , CURRENT_THREAD , 1<<order, cycle ); |
---|
| 248 | #endif |
---|
| 249 | |
---|
[160] | 250 | return NULL; |
---|
| 251 | } |
---|
[18] | 252 | |
---|
[160] | 253 | // update free-lists after removing a block |
---|
[18] | 254 | ppm->free_pages_nr[current_order] --; |
---|
[1] | 255 | current_size = (1 << current_order); |
---|
| 256 | |
---|
[160] | 257 | // split the removed block in smaller sub-blocks if required |
---|
| 258 | // and update the free-lists accordingly |
---|
[1] | 259 | while( current_order > order ) |
---|
| 260 | { |
---|
| 261 | current_order --; |
---|
| 262 | current_size >>= 1; |
---|
[18] | 263 | |
---|
[1] | 264 | remaining_block = block + current_size; |
---|
| 265 | remaining_block->order = current_order; |
---|
| 266 | |
---|
| 267 | list_add_first( &ppm->free_pages_root[current_order] , &remaining_block->list ); |
---|
| 268 | ppm->free_pages_nr[current_order] ++; |
---|
| 269 | } |
---|
[18] | 270 | |
---|
[160] | 271 | // update page descriptor |
---|
| 272 | page_clear_flag( block , PG_FREE ); |
---|
[1] | 273 | page_refcount_up( block ); |
---|
| 274 | block->order = order; |
---|
| 275 | |
---|
[160] | 276 | // release lock protecting free lists |
---|
[1] | 277 | spinlock_unlock( &ppm->free_lock ); |
---|
[18] | 278 | |
---|
[433] | 279 | #if CONFIG_DEBUG_PPM_ALLOC_PAGES |
---|
| 280 | cycle = (uint32_t)hal_get_cycles(); |
---|
| 281 | if( CONFIG_DEBUG_PPM_ALLOC_PAGES < cycle ) |
---|
| 282 | printk("\n[DBG] in %s : thread %x exit / %d page(s) allocated / ppn = %x / cycle %d\n", |
---|
| 283 | __FUNCTION__, CURRENT_THREAD, 1<<order, ppm_page2ppn(XPTR( local_cxy , block )), cycle ); |
---|
| 284 | #endif |
---|
[7] | 285 | |
---|
[1] | 286 | return block; |
---|
| 287 | |
---|
[433] | 288 | } // end ppm_alloc_pages() |
---|
[1] | 289 | |
---|
[433] | 290 | |
---|
[1] | 291 | //////////////////////////////////// |
---|
| 292 | void ppm_free_pages( page_t * page ) |
---|
| 293 | { |
---|
| 294 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
[18] | 295 | |
---|
[433] | 296 | #if CONFIG_DEBUG_PPM_FREE_PAGES |
---|
| 297 | uint32_t cycle = (uint32_t)hal_get_cycles(); |
---|
| 298 | if( CONFIG_DEBUG_PPM_FREE_PAGES < cycle ) |
---|
| 299 | printk("\n[DBG] in %s : thread %x enter for %d page(s) / cycle %d\n", |
---|
| 300 | __FUNCTION__ , CURRENT_THREAD , 1<<page->order , cycle ); |
---|
| 301 | #endif |
---|
| 302 | |
---|
| 303 | #if(CONFIG_DEBUG_PPM_FREE_PAGES & 0x1) |
---|
| 304 | if( CONFIG_DEBUG_PPM_FREE_PAGES < cycle ) |
---|
| 305 | ppm_print(); |
---|
| 306 | #endif |
---|
| 307 | |
---|
[160] | 308 | // get lock protecting free_pages[] array |
---|
[1] | 309 | spinlock_lock( &ppm->free_lock ); |
---|
| 310 | |
---|
[18] | 311 | ppm_free_pages_nolock( page ); |
---|
[1] | 312 | |
---|
[160] | 313 | // release lock protecting free_pages[] array |
---|
[1] | 314 | spinlock_unlock( &ppm->free_lock ); |
---|
[433] | 315 | |
---|
| 316 | #if CONFIG_DEBUG_PPM_FREE_PAGES |
---|
| 317 | cycle = (uint32_t)hal_get_cycles(); |
---|
| 318 | if( CONFIG_DEBUG_PPM_FREE_PAGES < cycle ) |
---|
| 319 | printk("\n[DBG] in %s : thread %x exit / %d page(s) released / ppn = %x / cycle %d\n", |
---|
| 320 | __FUNCTION__, CURRENT_THREAD, 1<<page->order, ppm_page2ppn(XPTR(local_cxy , page)), cycle ); |
---|
| 321 | #endif |
---|
| 322 | |
---|
[1] | 323 | } |
---|
| 324 | |
---|
[433] | 325 | //////////////// |
---|
| 326 | void ppm_print() |
---|
[1] | 327 | { |
---|
| 328 | uint32_t order; |
---|
| 329 | list_entry_t * iter; |
---|
| 330 | page_t * page; |
---|
| 331 | |
---|
[433] | 332 | ppm_t * ppm = &LOCAL_CLUSTER->ppm; |
---|
| 333 | |
---|
[160] | 334 | // get lock protecting free lists |
---|
[1] | 335 | spinlock_lock( &ppm->free_lock ); |
---|
| 336 | |
---|
[433] | 337 | printk("\n*** PPM in cluster %x : %d pages ***\n", local_cxy , ppm->pages_nr ); |
---|
[18] | 338 | |
---|
[1] | 339 | for( order = 0 ; order < CONFIG_PPM_MAX_ORDER ; order++ ) |
---|
| 340 | { |
---|
[433] | 341 | printk("- order = %d / free_pages = %d\t: ", |
---|
[160] | 342 | order , ppm->free_pages_nr[order] ); |
---|
[18] | 343 | |
---|
[1] | 344 | LIST_FOREACH( &ppm->free_pages_root[order] , iter ) |
---|
| 345 | { |
---|
| 346 | page = LIST_ELEMENT( iter , page_t , list ); |
---|
[433] | 347 | printk("%x," , page - ppm->pages_tbl ); |
---|
[1] | 348 | } |
---|
[18] | 349 | |
---|
[433] | 350 | printk("\n"); |
---|
[1] | 351 | } |
---|
| 352 | |
---|
[160] | 353 | // release lock protecting free lists |
---|
[1] | 354 | spinlock_unlock( &ppm->free_lock ); |
---|
[160] | 355 | } |
---|
[1] | 356 | |
---|
[53] | 357 | /////////////////////////////////////// |
---|
| 358 | error_t ppm_assert_order( ppm_t * ppm ) |
---|
[1] | 359 | { |
---|
| 360 | uint32_t order; |
---|
| 361 | list_entry_t * iter; |
---|
| 362 | page_t * page; |
---|
[18] | 363 | |
---|
[407] | 364 | for( order=0 ; order < CONFIG_PPM_MAX_ORDER ; order++ ) |
---|
[1] | 365 | { |
---|
| 366 | if( list_is_empty( &ppm->free_pages_root[order] ) ) continue; |
---|
[18] | 367 | |
---|
[1] | 368 | LIST_FOREACH( &ppm->free_pages_root[order] , iter ) |
---|
| 369 | { |
---|
| 370 | page = LIST_ELEMENT( iter , page_t , list ); |
---|
| 371 | |
---|
[160] | 372 | if( page->order != order ) return -1; |
---|
[1] | 373 | } |
---|
| 374 | } |
---|
| 375 | |
---|
[160] | 376 | return 0; |
---|
| 377 | } |
---|
[53] | 378 | |
---|