Context Navigation

source: trunk/kernel/mm/vmm.c @ 623

Last change on this file since 623 was 623, checked in by alain, 5 years ago
Introduce three new types of vsegs (KCODE,KDATA,KDEV) to map the kernel vsegs in the process VSL and GPT. This now used by both the TSAR and the I86 architectures.
File size: 76.4 KB

Rev	Line
[1]	1	/*
[611]	2	* vmm.c - virtual memory manager related operations definition.
[1]	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
[595]	6	* Alain Greiner (2016,2017,2018)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[577]	31	#include <hal_macros.h>
[1]	32	#include <printk.h>
[23]	33	#include <memcpy.h>
[567]	34	#include <remote_rwlock.h>
	35	#include <remote_queuelock.h>
[1]	36	#include <list.h>
[408]	37	#include <xlist.h>
[1]	38	#include <bits.h>
	39	#include <process.h>
	40	#include <thread.h>
	41	#include <vseg.h>
	42	#include <cluster.h>
	43	#include <scheduler.h>
	44	#include <vfs.h>
	45	#include <mapper.h>
	46	#include <page.h>
	47	#include <kmem.h>
	48	#include <vmm.h>
[585]	49	#include <hal_exception.h>
[1]	50
	51	//////////////////////////////////////////////////////////////////////////////////
	52	// Extern global variables
	53	//////////////////////////////////////////////////////////////////////////////////
	54
[567]	55	extern process_t process_zero; // allocated in cluster.c
[1]	56
[415]	57	///////////////////////////////////////
	58	error_t vmm_init( process_t * process )
[21]	59	{
[1]	60	error_t error;
	61	vseg_t * vseg_args;
	62	vseg_t * vseg_envs;
	63	intptr_t base;
	64	intptr_t size;
[614]	65	uint32_t i;
[1]	66
[438]	67	#if DEBUG_VMM_INIT
[567]	68	thread_t * this = CURRENT_THREAD;
[433]	69	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	70	if( DEBUG_VMM_INIT )
[614]	71	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	72	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	73	#endif
[204]	74
[1]	75	// get pointer on VMM
	76	vmm_t * vmm = &process->vmm;
	77
[407]	78	// initialize local list of vsegs
	79	vmm->vsegs_nr = 0;
[408]	80	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
[580]	81	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
[407]	82
[567]	83	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
	84	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
[21]	85
[567]	86	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
	87	"no more than 32 threads per cluster for a single process\n");
[1]	88
[567]	89	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
	90	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
	91	"STACK zone too small\n");
[1]	92
[409]	93	// register args vseg in VSL
[406]	94	base = (CONFIG_VMM_KENTRY_BASE +
	95	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	96	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	97
[407]	98	vseg_args = vmm_create_vseg( process,
	99	VSEG_TYPE_DATA,
	100	base,
	101	size,
	102	0, // file_offset unused
	103	0, // file_size unused
	104	XPTR_NULL, // mapper_xp unused
	105	local_cxy );
[204]	106
[415]	107	if( vseg_args == NULL )
	108	{
	109	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	110	return -1;
	111	}
[204]	112
[406]	113	vmm->args_vpn_base = base;
[1]	114
[409]	115	// register the envs vseg in VSL
[406]	116	base = (CONFIG_VMM_KENTRY_BASE +
	117	CONFIG_VMM_KENTRY_SIZE +
	118	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	119	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	120
[407]	121	vseg_envs = vmm_create_vseg( process,
	122	VSEG_TYPE_DATA,
	123	base,
	124	size,
	125	0, // file_offset unused
	126	0, // file_size unused
	127	XPTR_NULL, // mapper_xp unused
	128	local_cxy );
[204]	129
[415]	130	if( vseg_envs == NULL )
	131	{
	132	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	133	return -1;
	134	}
[204]	135
[406]	136	vmm->envs_vpn_base = base;
[1]	137
[409]	138	// create GPT (empty)
[1]	139	error = hal_gpt_create( &vmm->gpt );
	140
[415]	141	if( error )
[623]	142	{
	143	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	144	return -1;
	145	}
[204]	146
[585]	147	// initialize GPT lock
	148	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
	149
[623]	150	// update process VMM with kernel vsegs
	151	error = hal_vmm_kernel_update( process );
[409]	152
[623]	153	if( error )
	154	{
	155	printk("\n[ERROR] in %s : cannot update GPT for kernel vsegs\n", __FUNCTION__ );
	156	return -1;
	157	}
[409]	158
[1]	159	// initialize STACK allocator
	160	vmm->stack_mgr.bitmap = 0;
	161	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[567]	162	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
[1]	163
	164	// initialize MMAP allocator
[407]	165	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	166	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	167	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[567]	168	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
[1]	169	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
	170
[21]	171	// initialize instrumentation counters
[409]	172	vmm->pgfault_nr = 0;
[1]	173
[124]	174	hal_fence();
[1]	175
[438]	176	#if DEBUG_VMM_INIT
[433]	177	cycle = (uint32_t)hal_get_cycles();
[438]	178	if( DEBUG_VMM_INIT )
[614]	179	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	180	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	181	#endif
[204]	182
[415]	183	return 0;
	184
[204]	185	} // end vmm_init()
	186
[407]	187	//////////////////////////////////////
	188	void vmm_display( process_t * process,
	189	bool_t mapping )
	190	{
	191	vmm_t * vmm = &process->vmm;
	192	gpt_t * gpt = &vmm->gpt;
	193
[457]	194	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
	195	process->vmm.gpt.ptr , process->pid , local_cxy );
[407]	196
[585]	197	// get lock protecting the VSL and the GPT
[567]	198	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
[585]	199	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
[407]	200
	201	// scan the list of vsegs
[408]	202	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	203	xptr_t iter_xp;
	204	xptr_t vseg_xp;
[407]	205	vseg_t * vseg;
[408]	206	XLIST_FOREACH( root_xp , iter_xp )
[407]	207	{
[408]	208	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	209	vseg = GET_PTR( vseg_xp );
[408]	210
[407]	211	printk(" - %s : base = %X / size = %X / npages = %d\n",
	212	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
	213
	214	if( mapping )
	215	{
	216	vpn_t vpn;
	217	ppn_t ppn;
	218	uint32_t attr;
	219	vpn_t base = vseg->vpn_base;
	220	vpn_t size = vseg->vpn_size;
	221	for( vpn = base ; vpn < (base+size) ; vpn++ )
	222	{
[585]	223	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
[407]	224	if( attr & GPT_MAPPED )
	225	{
	226	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
	227	}
	228	}
	229	}
	230	}
	231
[585]	232	// release the locks
[567]	233	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
[585]	234	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
[407]	235
[408]	236	} // vmm_display()
	237
[611]	238	//////////////////////////////////////////
	239	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
	240	vseg_t * vseg )
[567]	241	{
	242	// build extended pointer on rwlock protecting VSL
	243	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	244
	245	// get rwlock in write mode
	246	remote_rwlock_wr_acquire( lock_xp );
	247
	248	// update vseg descriptor
	249	vseg->vmm = vmm;
	250
	251	// add vseg in vmm list
	252	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
	253	XPTR( local_cxy , &vseg->xlist ) );
	254
	255	// release rwlock in write mode
	256	remote_rwlock_wr_release( lock_xp );
	257	}
	258
[611]	259	////////////////////////////////////////////
	260	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
	261	vseg_t * vseg )
[567]	262	{
[611]	263	// get vseg type
	264	uint32_t type = vseg->type;
	265
[567]	266	// build extended pointer on rwlock protecting VSL
	267	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	268
	269	// get rwlock in write mode
	270	remote_rwlock_wr_acquire( lock_xp );
	271
	272	// update vseg descriptor
	273	vseg->vmm = NULL;
	274
[611]	275	// remove vseg from VSL
[567]	276	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	277
	278	// release rwlock in write mode
	279	remote_rwlock_wr_release( lock_xp );
	280
[611]	281	// release the stack slot to VMM stack allocator if STACK type
	282	if( type == VSEG_TYPE_STACK )
	283	{
	284	// get pointer on stack allocator
	285	stack_mgr_t * mgr = &vmm->stack_mgr;
	286
	287	// compute slot index
	288	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
	289
	290	// update stacks_bitmap
	291	busylock_acquire( &mgr->lock );
	292	bitmap_clear( &mgr->bitmap , index );
	293	busylock_release( &mgr->lock );
	294	}
	295
	296	// release the vseg to VMM mmap allocator if MMAP type
	297	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
	298	{
	299	// get pointer on mmap allocator
	300	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	301
	302	// compute zombi_list index
	303	uint32_t index = bits_log2( vseg->vpn_size );
	304
	305	// update zombi_list
	306	busylock_acquire( &mgr->lock );
	307	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
	308	busylock_release( &mgr->lock );
	309	}
	310
[623]	311	// release physical memory allocated for vseg if no MMAP and no kernel type
	312	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) &&
	313	(type != VSEG_TYPE_KCODE) && (type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
[611]	314	{
	315	vseg_free( vseg );
	316	}
	317
	318	} // end vmm_remove_vseg_from_vsl()
	319
[595]	320	////////////////////////////////////////////////
[433]	321	void vmm_global_update_pte( process_t * process,
	322	vpn_t vpn,
	323	uint32_t attr,
	324	ppn_t ppn )
[23]	325	{
[408]	326	xlist_entry_t * process_root_ptr;
	327	xptr_t process_root_xp;
	328	xptr_t process_iter_xp;
[23]	329
[408]	330	xptr_t remote_process_xp;
	331	cxy_t remote_process_cxy;
	332	process_t * remote_process_ptr;
	333	xptr_t remote_gpt_xp;
[23]	334
[408]	335	pid_t pid;
	336	cxy_t owner_cxy;
	337	lpid_t owner_lpid;
[23]	338
[438]	339	#if DEBUG_VMM_UPDATE_PTE
[433]	340	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	341	thread_t * this = CURRENT_THREAD;
[438]	342	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	343	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
	344	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
[433]	345	#endif
	346
[567]	347	// check cluster is reference
[585]	348	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
[433]	349
[408]	350	// get extended pointer on root of process copies xlist in owner cluster
	351	pid = process->pid;
	352	owner_cxy = CXY_FROM_PID( pid );
	353	owner_lpid = LPID_FROM_PID( pid );
	354	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	355	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	356
[408]	357	// loop on destination process copies
	358	XLIST_FOREACH( process_root_xp , process_iter_xp )
	359	{
	360	// get cluster and local pointer on remote process
	361	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	362	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	363	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	364
[438]	365	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
	366	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	367	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
	368	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
[433]	369	#endif
	370
[408]	371	// get extended pointer on remote gpt
	372	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	373
[433]	374	// update remote GPT
	375	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	376	}
	377
[438]	378	#if DEBUG_VMM_UPDATE_PTE
[433]	379	cycle = (uint32_t)hal_get_cycles();
[438]	380	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	381	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
	382	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
[433]	383	#endif
	384
	385	} // end vmm_global_update_pte()
	386
[408]	387	///////////////////////////////////////
	388	void vmm_set_cow( process_t * process )
	389	{
	390	vmm_t * vmm;
	391
	392	xlist_entry_t * process_root_ptr;
	393	xptr_t process_root_xp;
	394	xptr_t process_iter_xp;
	395
	396	xptr_t remote_process_xp;
	397	cxy_t remote_process_cxy;
	398	process_t * remote_process_ptr;
	399	xptr_t remote_gpt_xp;
	400
	401	xptr_t vseg_root_xp;
	402	xptr_t vseg_iter_xp;
	403
	404	xptr_t vseg_xp;
	405	vseg_t * vseg;
	406
	407	pid_t pid;
	408	cxy_t owner_cxy;
	409	lpid_t owner_lpid;
	410
[438]	411	#if DEBUG_VMM_SET_COW
[595]	412	uint32_t cycle = (uint32_t)hal_get_cycles();
	413	thread_t * this = CURRENT_THREAD;
[438]	414	if( DEBUG_VMM_SET_COW < cycle )
[595]	415	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
	416	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
[433]	417	#endif
[408]	418
[567]	419	// check cluster is reference
	420	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
	421	"local cluster is not process reference cluster\n");
[408]	422
	423	// get pointer on reference VMM
	424	vmm = &process->vmm;
	425
	426	// get extended pointer on root of process copies xlist in owner cluster
	427	pid = process->pid;
	428	owner_cxy = CXY_FROM_PID( pid );
	429	owner_lpid = LPID_FROM_PID( pid );
	430	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	431	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	432
	433	// get extended pointer on root of vsegs xlist from reference VMM
	434	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	435
	436	// loop on destination process copies
	437	XLIST_FOREACH( process_root_xp , process_iter_xp )
	438	{
	439	// get cluster and local pointer on remote process
	440	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	441	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	442	remote_process_cxy = GET_CXY( remote_process_xp );
	443
[595]	444	#if (DEBUG_VMM_SET_COW & 1)
[438]	445	if( DEBUG_VMM_SET_COW < cycle )
[595]	446	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
	447	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
[433]	448	#endif
[408]	449
	450	// get extended pointer on remote gpt
	451	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	452
	453	// loop on vsegs in (local) reference process VSL
	454	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	455	{
	456	// get pointer on vseg
	457	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	458	vseg = GET_PTR( vseg_xp );
[408]	459
[567]	460	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
	461	"all vsegs in reference VSL must be local\n" );
[408]	462
	463	// get vseg type, base and size
	464	uint32_t type = vseg->type;
	465	vpn_t vpn_base = vseg->vpn_base;
	466	vpn_t vpn_size = vseg->vpn_size;
	467
[595]	468	#if (DEBUG_VMM_SET_COW & 1)
[438]	469	if( DEBUG_VMM_SET_COW < cycle )
[595]	470	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	471	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
[433]	472	#endif
	473	// only DATA, ANON and REMOTE vsegs
[408]	474	if( (type == VSEG_TYPE_DATA) \|\|
	475	(type == VSEG_TYPE_ANON) \|\|
	476	(type == VSEG_TYPE_REMOTE) )
	477	{
[433]	478	vpn_t vpn;
	479	uint32_t attr;
	480	ppn_t ppn;
	481	xptr_t page_xp;
	482	cxy_t page_cxy;
	483	page_t * page_ptr;
	484	xptr_t forks_xp;
[469]	485	xptr_t lock_xp;
[433]	486
	487	// update flags in remote GPT
	488	hal_gpt_set_cow( remote_gpt_xp,
	489	vpn_base,
	490	vpn_size );
	491
	492	// atomically increment pending forks counter in physical pages,
	493	// for all vseg pages that are mapped in reference cluster
	494	if( remote_process_cxy == local_cxy )
	495	{
	496	// scan all pages in vseg
	497	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	498	{
	499	// get page attributes and PPN from reference GPT
[585]	500	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
[433]	501
	502	// atomically update pending forks counter if page is mapped
	503	if( attr & GPT_MAPPED )
	504	{
[469]	505	// get pointers and cluster on page descriptor
[433]	506	page_xp = ppm_ppn2page( ppn );
	507	page_cxy = GET_CXY( page_xp );
	508	page_ptr = GET_PTR( page_xp );
[469]	509
	510	// get extended pointers on "forks" and "lock"
[433]	511	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	512	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	513
[567]	514	// take lock protecting "forks" counter
	515	remote_busylock_acquire( lock_xp );
	516
[469]	517	// increment "forks"
[433]	518	hal_remote_atomic_add( forks_xp , 1 );
[567]	519
	520	// release lock protecting "forks" counter
	521	remote_busylock_release( lock_xp );
[433]	522	}
	523	} // end loop on vpn
	524	} // end if local
	525	} // end if vseg type
	526	} // end loop on vsegs
[408]	527	} // end loop on process copies
	528
[438]	529	#if DEBUG_VMM_SET_COW
[433]	530	cycle = (uint32_t)hal_get_cycles();
[438]	531	if( DEBUG_VMM_SET_COW < cycle )
[595]	532	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
	533	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
[433]	534	#endif
[408]	535
	536	} // end vmm_set-cow()
	537
	538	/////////////////////////////////////////////////
	539	error_t vmm_fork_copy( process_t * child_process,
	540	xptr_t parent_process_xp )
	541	{
	542	error_t error;
	543	cxy_t parent_cxy;
	544	process_t * parent_process;
	545	vmm_t * parent_vmm;
	546	xptr_t parent_lock_xp;
	547	vmm_t * child_vmm;
	548	xptr_t iter_xp;
	549	xptr_t parent_vseg_xp;
	550	vseg_t * parent_vseg;
	551	vseg_t * child_vseg;
	552	uint32_t type;
	553	bool_t cow;
	554	vpn_t vpn;
	555	vpn_t vpn_base;
	556	vpn_t vpn_size;
[469]	557	xptr_t page_xp; // extended pointer on page descriptor
[408]	558	page_t * page_ptr;
	559	cxy_t page_cxy;
[469]	560	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
	561	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
[408]	562	xptr_t parent_root_xp;
	563	bool_t mapped;
	564	ppn_t ppn;
	565
[438]	566	#if DEBUG_VMM_FORK_COPY
[433]	567	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	568	thread_t * this = CURRENT_THREAD;
[438]	569	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	570	printk("\n[%s] thread %x enter / cycle %d\n",
	571	__FUNCTION__ , this->process->pid, this->trdid, cycle );
[433]	572	#endif
[408]	573
	574	// get parent process cluster and local pointer
	575	parent_cxy = GET_CXY( parent_process_xp );
[433]	576	parent_process = GET_PTR( parent_process_xp );
[408]	577
	578	// get local pointers on parent and child VMM
	579	parent_vmm = &parent_process->vmm;
	580	child_vmm = &child_process->vmm;
	581
	582	// get extended pointer on lock protecting the parent VSL
	583	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
	584
	585	// initialize the lock protecting the child VSL
[567]	586	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
[408]	587
	588	// initialize the child VSL as empty
	589	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	590	child_vmm->vsegs_nr = 0;
	591
[623]	592	// create the child GPT
[408]	593	error = hal_gpt_create( &child_vmm->gpt );
[415]	594
[407]	595	if( error )
	596	{
[408]	597	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	598	return -1;
[407]	599	}
	600
[408]	601	// build extended pointer on parent VSL
	602	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
	603
[567]	604	// take the lock protecting the parent VSL in read mode
	605	remote_rwlock_rd_acquire( parent_lock_xp );
[415]	606
[408]	607	// loop on parent VSL xlist
	608	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	609	{
[408]	610	// get local and extended pointers on current parent vseg
	611	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	612	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	613
[408]	614	// get vseg type
[567]	615	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
[408]	616
[438]	617	#if DEBUG_VMM_FORK_COPY
[433]	618	cycle = (uint32_t)hal_get_cycles();
[438]	619	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	620	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
	621	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
[567]	622	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	623	#endif
[23]	624
[623]	625	// all parent vsegs - but STACK and kernel vsegs - must be copied in child VSL
	626	if( (type != VSEG_TYPE_STACK) && (type != VSEG_TYPE_KCODE) &&
	627	(type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
[23]	628	{
[408]	629	// allocate memory for a new child vseg
	630	child_vseg = vseg_alloc();
	631	if( child_vseg == NULL ) // release all allocated vsegs
[23]	632	{
[408]	633	vmm_destroy( child_process );
	634	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	635	return -1;
[23]	636	}
	637
[408]	638	// copy parent vseg to child vseg
	639	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	640
[408]	641	// register child vseg in child VSL
[611]	642	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
[407]	643
[438]	644	#if DEBUG_VMM_FORK_COPY
[433]	645	cycle = (uint32_t)hal_get_cycles();
[438]	646	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	647	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	648	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
[567]	649	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	650	#endif
[23]	651
[408]	652	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
	653	if( type != VSEG_TYPE_CODE )
	654	{
	655	// activate the COW for DATA, MMAP, REMOTE vsegs only
	656	cow = ( type != VSEG_TYPE_FILE );
[23]	657
[408]	658	vpn_base = child_vseg->vpn_base;
	659	vpn_size = child_vseg->vpn_size;
[23]	660
[408]	661	// scan pages in parent vseg
	662	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	663	{
	664	error = hal_gpt_pte_copy( &child_vmm->gpt,
	665	XPTR( parent_cxy , &parent_vmm->gpt ),
	666	vpn,
	667	cow,
	668	&ppn,
	669	&mapped );
	670	if( error )
	671	{
	672	vmm_destroy( child_process );
	673	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	674	return -1;
	675	}
	676
[433]	677	// increment pending forks counter in page if mapped
[408]	678	if( mapped )
	679	{
[469]	680	// get pointers and cluster on page descriptor
	681	page_xp = ppm_ppn2page( ppn );
[408]	682	page_cxy = GET_CXY( page_xp );
[433]	683	page_ptr = GET_PTR( page_xp );
[408]	684
[469]	685	// get extended pointers on "forks" and "lock"
	686	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	687	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	688
[567]	689	// get lock protecting "forks" counter
	690	remote_busylock_acquire( lock_xp );
	691
[469]	692	// increment "forks"
	693	hal_remote_atomic_add( forks_xp , 1 );
	694
[567]	695	// release lock protecting "forks" counter
	696	remote_busylock_release( lock_xp );
	697
[438]	698	#if DEBUG_VMM_FORK_COPY
[433]	699	cycle = (uint32_t)hal_get_cycles();
[438]	700	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	701	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
	702	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
[433]	703	#endif
[408]	704	}
	705	}
	706	} // end if no code & no stack
	707	} // end if no stack
	708	} // end loop on vsegs
	709
[567]	710	// release the parent VSL lock in read mode
	711	remote_rwlock_rd_release( parent_lock_xp );
[408]	712
[623]	713	// update child VMM with kernel vsegs
	714	error = hal_vmm_kernel_update( child_process );
[415]	715
	716	if( error )
	717	{
[623]	718	printk("\n[ERROR] in %s : cannot update child VMM\n", __FUNCTION__ );
[415]	719	return -1;
	720	}
	721
[408]	722	// initialize the child VMM STACK allocator
	723	child_vmm->stack_mgr.bitmap = 0;
	724	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	725
	726	// initialize the child VMM MMAP allocator
[23]	727	uint32_t i;
[408]	728	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	729	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	730	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
	731	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
[23]	732
[178]	733	// initialize instrumentation counters
[408]	734	child_vmm->pgfault_nr = 0;
[23]	735
[408]	736	// copy base addresses from parent VMM to child VMM
	737	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
	738	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	739	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	740	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	741	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	742	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	743
[408]	744	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	745
[124]	746	hal_fence();
[23]	747
[438]	748	#if DEBUG_VMM_FORK_COPY
[433]	749	cycle = (uint32_t)hal_get_cycles();
[438]	750	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	751	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
	752	__FUNCTION__ , this->process->pid, this->trdid , cycle );
[433]	753	#endif
	754
[23]	755	return 0;
	756
[408]	757	} // vmm_fork_copy()
[204]	758
[1]	759	///////////////////////////////////////
	760	void vmm_destroy( process_t * process )
	761	{
[408]	762	xptr_t vseg_xp;
[1]	763	vseg_t * vseg;
	764
[438]	765	#if DEBUG_VMM_DESTROY
[433]	766	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	767	thread_t * this = CURRENT_THREAD;
[438]	768	if( DEBUG_VMM_DESTROY < cycle )
[595]	769	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	770	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	771	#endif
[416]	772
[438]	773	#if (DEBUG_VMM_DESTROY & 1 )
[443]	774	if( DEBUG_VMM_DESTROY < cycle )
[437]	775	vmm_display( process , true );
	776	#endif
	777
[433]	778	// get pointer on local VMM
[1]	779	vmm_t * vmm = &process->vmm;
	780
[408]	781	// get extended pointer on VSL root and VSL lock
	782	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	783
[611]	784	// scan the VSL to delete all registered vsegs
	785	// (don't use a FOREACH for item deletion in xlist)
[619]	786
	787	uint32_t count = 0;
	788
	789	while( !xlist_is_empty( root_xp ) && (count < 10 ) )
[1]	790	{
[409]	791	// get pointer on first vseg in VSL
[567]	792	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
[433]	793	vseg = GET_PTR( vseg_xp );
[409]	794
[611]	795	// delete vseg and release physical pages
	796	vmm_delete_vseg( process->pid , vseg->min );
[409]	797
[443]	798	#if( DEBUG_VMM_DESTROY & 1 )
	799	if( DEBUG_VMM_DESTROY < cycle )
[611]	800	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
[443]	801	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	802	#endif
	803
[619]	804	count++;
	805
[1]	806	}
	807
	808	// remove all vsegs from zombi_lists in MMAP allocator
	809	uint32_t i;
	810	for( i = 0 ; i<32 ; i++ )
	811	{
	812	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
	813	{
[408]	814	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
[443]	815
	816	#if( DEBUG_VMM_DESTROY & 1 )
	817	if( DEBUG_VMM_DESTROY < cycle )
[595]	818	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
[443]	819	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	820	#endif
[611]	821	// clean vseg descriptor
	822	vseg->vmm = NULL;
	823
	824	// remove vseg from xlist
	825	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	826
	827	// release vseg descriptor
[1]	828	vseg_free( vseg );
[443]	829
	830	#if( DEBUG_VMM_DESTROY & 1 )
	831	if( DEBUG_VMM_DESTROY < cycle )
[595]	832	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
[443]	833	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	834	#endif
[1]	835	}
	836	}
	837
[409]	838	// release memory allocated to the GPT itself
[1]	839	hal_gpt_destroy( &vmm->gpt );
	840
[438]	841	#if DEBUG_VMM_DESTROY
[433]	842	cycle = (uint32_t)hal_get_cycles();
[438]	843	if( DEBUG_VMM_DESTROY < cycle )
[595]	844	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	845	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
[433]	846	#endif
[416]	847
[204]	848	} // end vmm_destroy()
	849
[1]	850	/////////////////////////////////////////////////
	851	vseg_t * vmm_check_conflict( process_t * process,
[21]	852	vpn_t vpn_base,
[1]	853	vpn_t vpn_size )
	854	{
	855	vmm_t * vmm = &process->vmm;
[408]	856
	857	// scan the VSL
[1]	858	vseg_t * vseg;
[408]	859	xptr_t iter_xp;
	860	xptr_t vseg_xp;
	861	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	862
[408]	863	XLIST_FOREACH( root_xp , iter_xp )
[1]	864	{
[408]	865	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	866	vseg = GET_PTR( vseg_xp );
[204]	867
[21]	868	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	869	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	870	}
	871	return NULL;
	872
[204]	873	} // end vmm_check_conflict()
	874
[1]	875	////////////////////////////////////////////////////////////////////////////////////////////
	876	// This static function is called by the vmm_create_vseg() function, and implements
	877	// the VMM stack_vseg specific allocator.
	878	////////////////////////////////////////////////////////////////////////////////////////////
	879	// @ vmm : pointer on VMM.
[21]	880	// @ vpn_base : (return value) first allocated page
[1]	881	// @ vpn_size : (return value) number of allocated pages
	882	////////////////////////////////////////////////////////////////////////////////////////////
	883	static error_t vmm_stack_alloc( vmm_t * vmm,
	884	vpn_t * vpn_base,
	885	vpn_t * vpn_size )
	886	{
	887	// get stack allocator pointer
	888	stack_mgr_t * mgr = &vmm->stack_mgr;
	889
	890	// get lock on stack allocator
[567]	891	busylock_acquire( &mgr->lock );
[1]	892
	893	// get first free slot index in bitmap
	894	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
[179]	895	if( (index < 0) \|\| (index > 31) )
	896	{
[567]	897	busylock_release( &mgr->lock );
	898	return 0xFFFFFFFF;
[179]	899	}
[1]	900
	901	// update bitmap
	902	bitmap_set( &mgr->bitmap , index );
[21]	903
[1]	904	// release lock on stack allocator
[567]	905	busylock_release( &mgr->lock );
[1]	906
[21]	907	// returns vpn_base, vpn_size (one page non allocated)
[1]	908	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
	909	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	910	return 0;
	911
[204]	912	} // end vmm_stack_alloc()
	913
[1]	914	////////////////////////////////////////////////////////////////////////////////////////////
	915	// This static function is called by the vmm_create_vseg() function, and implements
	916	// the VMM MMAP specific allocator.
	917	////////////////////////////////////////////////////////////////////////////////////////////
	918	// @ vmm : [in] pointer on VMM.
	919	// @ npages : [in] requested number of pages.
[21]	920	// @ vpn_base : [out] first allocated page.
[1]	921	// @ vpn_size : [out] actual number of allocated pages.
	922	////////////////////////////////////////////////////////////////////////////////////////////
	923	static error_t vmm_mmap_alloc( vmm_t * vmm,
	924	vpn_t npages,
	925	vpn_t * vpn_base,
	926	vpn_t * vpn_size )
	927	{
	928	uint32_t index;
	929	vseg_t * vseg;
	930	vpn_t base;
	931	vpn_t size;
[21]	932	vpn_t free;
[1]	933
[614]	934	#if DEBUG_VMM_MMAP_ALLOC
	935	thread_t * this = CURRENT_THREAD;
	936	uint32_t cycle = (uint32_t)hal_get_cycles();
	937	if( DEBUG_VMM_MMAP_ALLOC < cycle )
	938	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
	939	__FUNCTION__, this->process->pid, this->trdid, cycle );
	940	#endif
	941
	942	// vseg size must be power of 2
[1]	943	// compute actual size and index in zombi_list array
	944	size = POW2_ROUNDUP( npages );
	945	index = bits_log2( size );
	946
	947	// get mmap allocator pointer
	948	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	949
	950	// get lock on mmap allocator
[567]	951	busylock_acquire( &mgr->lock );
[1]	952
	953	// get vseg from zombi_list or from mmap zone
	954	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
	955	{
	956	// check overflow
	957	free = mgr->first_free_vpn;
[614]	958	if( (free + size) > mgr->vpn_size ) return -1;
[1]	959
[614]	960	// update MMAP allocator
[1]	961	mgr->first_free_vpn += size;
	962
	963	// compute base
	964	base = free;
	965	}
	966	else // from zombi_list
	967	{
	968	// get pointer on zombi vseg from zombi_list
[408]	969	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
[1]	970
	971	// remove vseg from free-list
[408]	972	list_unlink( &vseg->zlist );
[1]	973
	974	// compute base
	975	base = vseg->vpn_base;
[21]	976	}
	977
[1]	978	// release lock on mmap allocator
[567]	979	busylock_release( &mgr->lock );
[1]	980
[614]	981	#if DEBUG_VMM_MMAP_ALLOC
	982	cycle = (uint32_t)hal_get_cycles();
	983	if( DEBUG_VMM_DESTROY < cycle )
	984	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
	985	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
	986	#endif
	987
[1]	988	// returns vpn_base, vpn_size
	989	*vpn_base = base;
	990	*vpn_size = size;
	991	return 0;
	992
[204]	993	} // end vmm_mmap_alloc()
	994
[407]	995	////////////////////////////////////////////////
	996	vseg_t * vmm_create_vseg( process_t * process,
	997	vseg_type_t type,
	998	intptr_t base,
	999	uint32_t size,
	1000	uint32_t file_offset,
	1001	uint32_t file_size,
	1002	xptr_t mapper_xp,
	1003	cxy_t cxy )
[1]	1004	{
	1005	vseg_t * vseg; // created vseg pointer
[204]	1006	vpn_t vpn_base; // first page index
[595]	1007	vpn_t vpn_size; // number of pages covered by vseg
[1]	1008	error_t error;
	1009
[438]	1010	#if DEBUG_VMM_CREATE_VSEG
[595]	1011	thread_t * this = CURRENT_THREAD;
	1012	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1013	if( DEBUG_VMM_CREATE_VSEG < cycle )
[614]	1014	printk("\n[%s] thread[%x,%x] enter for process %x / %s / cxy %x / cycle %d\n",
	1015	__FUNCTION__, this->process->pid, this->trdid, process->pid, vseg_type_str(type), cxy, cycle );
[433]	1016	#endif
[21]	1017
[407]	1018	// get pointer on VMM
	1019	vmm_t * vmm = &process->vmm;
[21]	1020
[204]	1021	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	1022	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[595]	1023
[1]	1024	if( type == VSEG_TYPE_STACK )
	1025	{
	1026	// get vpn_base and vpn_size from STACK allocator
	1027	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
	1028	if( error )
	1029	{
[407]	1030	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
	1031	__FUNCTION__ , process->pid , local_cxy );
[1]	1032	return NULL;
	1033	}
	1034
	1035	// compute vseg base and size from vpn_base and vpn_size
	1036	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1037	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	1038	}
[595]	1039	else if( type == VSEG_TYPE_FILE )
	1040	{
	1041	// compute page index (in mapper) for first byte
	1042	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
	1043
	1044	// compute page index (in mapper) for last byte
	1045	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1046
	1047	// compute offset in first page
	1048	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
	1049
	1050	// compute number of pages required in virtual space
	1051	vpn_t npages = vpn_max - vpn_min + 1;
	1052
	1053	// get vpn_base and vpn_size from MMAP allocator
	1054	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	1055	if( error )
	1056	{
	1057	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	1058	__FUNCTION__ , process->pid , local_cxy );
	1059	return NULL;
	1060	}
	1061
	1062	// set the vseg base (not always aligned for FILE)
	1063	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
	1064	}
[21]	1065	else if( (type == VSEG_TYPE_ANON) \|\|
[1]	1066	(type == VSEG_TYPE_REMOTE) )
	1067	{
[595]	1068	// compute number of required pages in virtual space
	1069	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	1070	if( size & CONFIG_PPM_PAGE_MASK) npages++;
	1071
[1]	1072	// get vpn_base and vpn_size from MMAP allocator
	1073	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	1074	if( error )
	1075	{
	1076	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	1077	__FUNCTION__ , process->pid , local_cxy );
	1078	return NULL;
	1079	}
	1080
[595]	1081	// set vseg base (always aligned for ANON or REMOTE)
[1]	1082	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1083	}
[623]	1084	else // VSEG_TYPE_DATA, VSEG_TYPE_CODE or KERNEL vseg
[1]	1085	{
[204]	1086	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	1087	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1088
	1089	vpn_base = vpn_min;
	1090	vpn_size = vpn_max - vpn_min + 1;
[1]	1091	}
	1092
	1093	// check collisions
	1094	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
	1095	if( vseg != NULL )
	1096	{
[614]	1097	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
	1098	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
[407]	1099	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	1100	return NULL;
	1101	}
	1102
	1103	// allocate physical memory for vseg descriptor
	1104	vseg = vseg_alloc();
	1105	if( vseg == NULL )
	1106	{
	1107	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	1108	__FUNCTION__ , process->pid );
[1]	1109	return NULL;
	1110	}
	1111
[614]	1112	#if DEBUG_VMM_CREATE_VSEG
	1113	if( DEBUG_VMM_CREATE_VSEG < cycle )
	1114	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
	1115	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
	1116	#endif
	1117
[1]	1118	// initialize vseg descriptor
[407]	1119	vseg_init( vseg,
	1120	type,
	1121	base,
	1122	size,
	1123	vpn_base,
	1124	vpn_size,
	1125	file_offset,
	1126	file_size,
	1127	mapper_xp,
	1128	cxy );
[1]	1129
[408]	1130	// attach vseg to VSL
[611]	1131	vmm_attach_vseg_to_vsl( vmm , vseg );
[1]	1132
[438]	1133	#if DEBUG_VMM_CREATE_VSEG
[433]	1134	cycle = (uint32_t)hal_get_cycles();
[438]	1135	if( DEBUG_VMM_CREATE_VSEG < cycle )
[595]	1136	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
	1137	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
[433]	1138	#endif
[21]	1139
[1]	1140	return vseg;
	1141
[406]	1142	} // vmm_create_vseg()
	1143
[611]	1144	///////////////////////////////////
	1145	void vmm_delete_vseg( pid_t pid,
	1146	intptr_t vaddr )
[1]	1147	{
[611]	1148	process_t * process; // local pointer on local process
	1149	vmm_t * vmm; // local pointer on local process VMM
	1150	vseg_t * vseg; // local pointer on local vseg containing vaddr
	1151	gpt_t * gpt; // local pointer on local process GPT
[21]	1152	vpn_t vpn; // VPN of current PTE
	1153	vpn_t vpn_min; // VPN of first PTE
[1]	1154	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1155	ppn_t ppn; // current PTE ppn value
	1156	uint32_t attr; // current PTE attributes
	1157	kmem_req_t req; // request to release memory
	1158	xptr_t page_xp; // extended pointer on page descriptor
	1159	cxy_t page_cxy; // page descriptor cluster
	1160	page_t * page_ptr; // page descriptor pointer
[433]	1161	xptr_t forks_xp; // extended pointer on pending forks counter
[469]	1162	xptr_t lock_xp; // extended pointer on lock protecting forks counter
	1163	uint32_t forks; // actual number of pendinf forks
[623]	1164	uint32_t type; // vseg type
[1]	1165
[611]	1166	#if DEBUG_VMM_DELETE_VSEG
[595]	1167	uint32_t cycle = (uint32_t)hal_get_cycles();
	1168	thread_t * this = CURRENT_THREAD;
[611]	1169	if( DEBUG_VMM_DELETE_VSEG < cycle )
	1170	printk("\n[%s] thread[%x,%x] enter / process %x / vaddr %x / cycle %d\n",
	1171	__FUNCTION__, this->process->pid, this->trdid, pid, vaddr, cycle );
[433]	1172	#endif
[409]	1173
[611]	1174	// get local pointer on local process descriptor
	1175	process = cluster_get_local_process_from_pid( pid );
[1]	1176
[623]	1177	if( process == NULL )
	1178	{
	1179	printk("\n[ERRORR] in %s : cannot get local process descriptor\n",
	1180	__FUNCTION__ );
	1181	return;
	1182	}
[611]	1183
	1184	// get pointers on local process VMM an GPT
	1185	vmm = &process->vmm;
	1186	gpt = &process->vmm.gpt;
	1187
	1188	// get local pointer on vseg containing vaddr
	1189	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
	1190
[623]	1191	if( vseg == NULL )
	1192	{
	1193	printk("\n[ERRORR] in %s : cannot get vseg descriptor\n",
	1194	__FUNCTION__ );
	1195	return;
	1196	}
[611]	1197
[623]	1198	// get relevant vseg infos
	1199	type = vseg->type;
[1]	1200	vpn_min = vseg->vpn_base;
	1201	vpn_max = vpn_min + vseg->vpn_size;
[623]	1202
	1203	// loop to invalidate all vseg PTEs in GPT
[1]	1204	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1205	{
[409]	1206	// get GPT entry
[585]	1207	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
[409]	1208
	1209	if( attr & GPT_MAPPED ) // entry is mapped
	1210	{
[437]	1211
[611]	1212	#if( DEBUG_VMM_DELETE_VSEG & 1 )
	1213	if( DEBUG_VMM_DELETE_VSEG < cycle )
	1214	printk("- unmap vpn %x / ppn %x / vseg %s \n" , vpn , ppn, vseg_type_str(vseg->type) );
[437]	1215	#endif
[585]	1216	// unmap GPT entry in local GPT
[409]	1217	hal_gpt_reset_pte( gpt , vpn );
	1218
[623]	1219	// the allocated page is not released to KMEM for kernel vseg
	1220	if( (type != VSEG_TYPE_KCODE) &&
	1221	(type != VSEG_TYPE_KDATA) &&
	1222	(type != VSEG_TYPE_KDEV ) )
[409]	1223	{
[623]	1224
	1225	// FIXME This code must be completely re-written, as the actual release must depend on
	1226	// - the vseg type
	1227	// - the reference cluster
	1228	// - the page refcount and/or the forks counter
	1229
[433]	1230	// get extended pointer on physical page descriptor
[409]	1231	page_xp = ppm_ppn2page( ppn );
	1232	page_cxy = GET_CXY( page_xp );
[433]	1233	page_ptr = GET_PTR( page_xp );
[409]	1234
[469]	1235	// get extended pointers on forks and lock fields
	1236	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1237	lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1238
[623]	1239	// get the lock protecting the page
[621]	1240	remote_busylock_acquire( lock_xp );
[623]	1241
[433]	1242	// get pending forks counter
[567]	1243	forks = hal_remote_l32( forks_xp );
[623]	1244
[469]	1245	if( forks ) // decrement pending forks counter
[409]	1246	{
[433]	1247	hal_remote_atomic_add( forks_xp , -1 );
	1248	}
	1249	else // release physical page to relevant cluster
[409]	1250	{
[433]	1251	if( page_cxy == local_cxy ) // local cluster
	1252	{
	1253	req.type = KMEM_PAGE;
	1254	req.ptr = page_ptr;
	1255	kmem_free( &req );
	1256	}
	1257	else // remote cluster
	1258	{
	1259	rpc_pmem_release_pages_client( page_cxy , page_ptr );
	1260	}
[611]	1261
	1262	#if( DEBUG_VMM_DELETE_VSEG & 1 )
	1263	if( DEBUG_VMM_DELETE_VSEG < cycle )
	1264	printk("- release ppn %x\n", ppn );
	1265	#endif
[409]	1266	}
[623]	1267
	1268	// release the lock protecting the page
[621]	1269	remote_busylock_release( lock_xp );
[409]	1270	}
	1271	}
[1]	1272	}
[433]	1273
[611]	1274	// remove vseg from VSL and release vseg descriptor (if not MMAP)
	1275	vmm_detach_vseg_from_vsl( vmm , vseg );
	1276
	1277	#if DEBUG_VMM_DELETE_VSEG
[433]	1278	cycle = (uint32_t)hal_get_cycles();
[611]	1279	if( DEBUG_VMM_DELETE_VSEG < cycle )
[595]	1280	printk("\n[%s] thread[%x,%x] exit / process %x / vseg %s / base %x / cycle %d\n",
[611]	1281	__FUNCTION__, this->process->pid, this->trdid, pid, vseg_type_str(vseg->type), vaddr, cycle );
[433]	1282	#endif
	1283
[611]	1284	} // end vmm_delete_vseg()
[1]	1285
[611]	1286	/////////////////////////////////////////////
	1287	vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
	1288	intptr_t vaddr )
[406]	1289	{
[408]	1290	xptr_t iter_xp;
	1291	xptr_t vseg_xp;
	1292	vseg_t * vseg;
[406]	1293
[408]	1294	// get extended pointers on VSL lock and root
	1295	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1296	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1297
[408]	1298	// get lock protecting the VSL
[567]	1299	remote_rwlock_rd_acquire( lock_xp );
[408]	1300
	1301	// scan the list of vsegs in VSL
	1302	XLIST_FOREACH( root_xp , iter_xp )
[406]	1303	{
[408]	1304	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1305	vseg = GET_PTR( vseg_xp );
[595]	1306
[408]	1307	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
[595]	1308	{
[408]	1309	// return success
[567]	1310	remote_rwlock_rd_release( lock_xp );
[408]	1311	return vseg;
	1312	}
[406]	1313	}
	1314
[408]	1315	// return failure
[567]	1316	remote_rwlock_rd_release( lock_xp );
[623]	1317
[408]	1318	return NULL;
[406]	1319
[595]	1320	} // end vmm_vseg_from_vaddr()
[406]	1321
[1]	1322	/////////////////////////////////////////////
	1323	error_t vmm_resize_vseg( process_t * process,
	1324	intptr_t base,
	1325	intptr_t size )
	1326	{
[406]	1327	error_t error;
	1328	vseg_t * new;
	1329	vpn_t vpn_min;
	1330	vpn_t vpn_max;
[1]	1331
[623]	1332	#if DEBUG_VMM_RESIZE_VSEG
	1333	uint32_t cycle = (uint32_t)hal_get_cycles();
	1334	thread_t * this = CURRENT_THREAD;
	1335	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1336	printk("\n[%s] thread[%x,%x] enter / process %x / base %x / size %d / cycle %d\n",
	1337	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
	1338	#endif
	1339
[1]	1340	// get pointer on process VMM
	1341	vmm_t * vmm = &process->vmm;
	1342
	1343	intptr_t addr_min = base;
	1344	intptr_t addr_max = base + size;
	1345
	1346	// get pointer on vseg
[595]	1347	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
[1]	1348
[623]	1349	if( vseg == NULL)
	1350	{
	1351	printk("\n[ERROR] in %s : vseg(%x,%d) not found\n",
	1352	__FUNCTION__, base , size );
	1353	return -1;
	1354	}
[21]	1355
[623]	1356	// resize depends on unmapped region base and size
[611]	1357	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // not included in vseg
[1]	1358	{
[623]	1359	printk("\n[ERROR] in %s : unmapped region[%x->%x[ not included in vseg[%x->%x[\n",
	1360	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1361
[611]	1362	error = -1;
[1]	1363	}
[611]	1364	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be deleted
[1]	1365	{
[623]	1366
	1367	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1368	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1369	printk("\n[%s] unmapped region[%x->%x[ equal vseg[%x->%x[\n",
	1370	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1371	#endif
[611]	1372	vmm_delete_vseg( process->pid , vseg->min );
[623]	1373
	1374	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1375	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1376	printk("\n[%s] thread[%x,%x] deleted vseg\n",
	1377	__FUNCTION__, this->process->pid, this->trdid );
	1378	#endif
[1]	1379	error = 0;
	1380	}
[611]	1381	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1382	{
[623]	1383
	1384	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1385	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1386	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1387	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1388	#endif
	1389	// update vseg min address
[406]	1390	vseg->min = addr_max;
	1391
	1392	// update vpn_base and vpn_size
	1393	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1394	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1395	vseg->vpn_base = vpn_min;
	1396	vseg->vpn_size = vpn_max - vpn_min + 1;
[623]	1397
	1398	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1399	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1400	printk("\n[%s] thread[%x,%x] changed vseg_min\n",
	1401	__FUNCTION__, this->process->pid, this->trdid );
	1402	#endif
[406]	1403	error = 0;
[1]	1404	}
[611]	1405	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1406	{
[623]	1407
	1408	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1409	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1410	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1411	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1412	#endif
[406]	1413	// update vseg max address
	1414	vseg->max = addr_min;
	1415
	1416	// update vpn_base and vpn_size
	1417	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1418	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1419	vseg->vpn_base = vpn_min;
	1420	vseg->vpn_size = vpn_max - vpn_min + 1;
[623]	1421
	1422	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1423	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1424	printk("\n[%s] thread[%x,%x] changed vseg_max\n",
	1425	__FUNCTION__, this->process->pid, this->trdid );
	1426	#endif
[406]	1427	error = 0;
[623]	1428
[1]	1429	}
[611]	1430	else // vseg cut in three regions
[1]	1431	{
[623]	1432
	1433	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1434	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1435	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1436	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1437	#endif
[406]	1438	// resize existing vseg
	1439	vseg->max = addr_min;
	1440
	1441	// update vpn_base and vpn_size
	1442	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1443	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1444	vseg->vpn_base = vpn_min;
	1445	vseg->vpn_size = vpn_max - vpn_min + 1;
	1446
	1447	// create new vseg
[407]	1448	new = vmm_create_vseg( process,
	1449	vseg->type,
	1450	addr_min,
	1451	(vseg->max - addr_max),
	1452	vseg->file_offset,
	1453	vseg->file_size,
	1454	vseg->mapper_xp,
	1455	vseg->cxy );
	1456
[623]	1457	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1458	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1459	printk("\n[%s] thread[%x,%x] replaced vseg by two smal vsegs\n",
	1460	__FUNCTION__, this->process->pid, this->trdid );
	1461	#endif
	1462
	1463	if( new == NULL ) error = -1;
[406]	1464	else error = 0;
[1]	1465	}
	1466
[623]	1467	#if DEBUG_VMM_RESIZE_VSEG
	1468	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1469	printk("\n[%s] thread[%x,%x] exit / process %x / base %x / size %d / cycle %d\n",
	1470	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
	1471	#endif
[1]	1472
	1473	return error;
	1474
[406]	1475	} // vmm_resize_vseg()
	1476
[1]	1477	///////////////////////////////////////////
[388]	1478	error_t vmm_get_vseg( process_t * process,
[394]	1479	intptr_t vaddr,
[388]	1480	vseg_t ** found_vseg )
[1]	1481	{
[595]	1482	xptr_t vseg_xp;
	1483	vseg_t * vseg;
	1484	vmm_t * vmm;
	1485	error_t error;
[1]	1486
[440]	1487	// get pointer on local VMM
	1488	vmm = &process->vmm;
[1]	1489
[440]	1490	// try to get vseg from local VMM
[595]	1491	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
[440]	1492
[388]	1493	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1494	{
	1495	// get extended pointer on reference process
	1496	xptr_t ref_xp = process->ref_xp;
[1]	1497
[388]	1498	// get cluster and local pointer on reference process
	1499	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1500	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1501
	1502	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1503
	1504	// get extended pointer on reference vseg
[394]	1505	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1506
[440]	1507	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1508
	1509	// allocate a vseg in local cluster
	1510	vseg = vseg_alloc();
	1511
[440]	1512	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1513
	1514	// initialise local vseg from reference
	1515	vseg_init_from_ref( vseg , vseg_xp );
	1516
[611]	1517	// register local vseg in local VSL
	1518	vmm_attach_vseg_to_vsl( vmm , vseg );
[388]	1519	}
[595]	1520
[388]	1521	// success
	1522	*found_vseg = vseg;
[394]	1523	return 0;
[388]	1524
	1525	} // end vmm_get_vseg()
	1526
[407]	1527	//////////////////////////////////////////////////////////////////////////////////////
	1528	// This static function compute the target cluster to allocate a physical page
	1529	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
	1530	// and returns an extended pointer on the allocated page descriptor.
[585]	1531	// It can be called by a thread running in any cluster.
[407]	1532	// The vseg cannot have the FILE type.
	1533	//////////////////////////////////////////////////////////////////////////////////////
	1534	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1535	vpn_t vpn )
	1536	{
[433]	1537
[438]	1538	#if DEBUG_VMM_ALLOCATE_PAGE
[619]	1539	uint32_t cycle = (uint32_t)hal_get_cycles();
	1540	thread_t * this = CURRENT_THREAD;
	1541	xptr_t this_xp = XPTR( local_cxy , this );
[438]	1542	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[595]	1543	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
	1544	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
[433]	1545	#endif
	1546
[407]	1547	page_t * page_ptr;
	1548	cxy_t page_cxy;
	1549	kmem_req_t req;
[577]	1550	uint32_t index;
[407]	1551
[577]	1552	uint32_t type = vseg->type;
	1553	uint32_t flags = vseg->flags;
	1554	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1555	uint32_t y_size = LOCAL_CLUSTER->y_size;
[407]	1556
[567]	1557	// check vseg type
	1558	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
[407]	1559
	1560	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1561	{
[577]	1562	index = vpn & ((x_size * y_size) - 1);
	1563	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
[561]	1564
[577]	1565	// If the cluster selected from VPN's LSBs is empty, we select one randomly
	1566	if ( cluster_is_active( page_cxy ) == false )
	1567	{
	1568	page_cxy = cluster_random_select();
[561]	1569	}
[407]	1570	}
	1571	else // other cases => cxy specified in vseg
	1572	{
[561]	1573	page_cxy = vseg->cxy;
[407]	1574	}
	1575
	1576	// allocate a physical page from target cluster
	1577	if( page_cxy == local_cxy ) // target cluster is the local cluster
	1578	{
	1579	req.type = KMEM_PAGE;
	1580	req.size = 0;
	1581	req.flags = AF_NONE;
	1582	page_ptr = (page_t *)kmem_alloc( &req );
	1583	}
	1584	else // target cluster is not the local cluster
	1585	{
	1586	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
	1587	}
	1588
[438]	1589	#if DEBUG_VMM_ALLOCATE_PAGE
[595]	1590	cycle = (uint32_t)hal_get_cycles();
[438]	1591	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[595]	1592	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
	1593	__FUNCTION__ , this->process->pid, this->trdid, vpn,
	1594	ppm_page2ppn( XPTR( page_cxy , page_ptr ) , cycle );
[433]	1595	#endif
	1596
[407]	1597	if( page_ptr == NULL ) return XPTR_NULL;
	1598	else return XPTR( page_cxy , page_ptr );
	1599
	1600	} // end vmm_page_allocate()
	1601
[313]	1602	////////////////////////////////////////
	1603	error_t vmm_get_one_ppn( vseg_t * vseg,
	1604	vpn_t vpn,
	1605	ppn_t * ppn )
	1606	{
	1607	error_t error;
[407]	1608	xptr_t page_xp; // extended pointer on physical page descriptor
[606]	1609	uint32_t page_id; // missing page index in vseg mapper
[406]	1610	uint32_t type; // vseg type;
[313]	1611
[406]	1612	type = vseg->type;
[606]	1613	page_id = vpn - vseg->vpn_base;
[313]	1614
[438]	1615	#if DEBUG_VMM_GET_ONE_PPN
[595]	1616	uint32_t cycle = (uint32_t)hal_get_cycles();
	1617	thread_t * this = CURRENT_THREAD;
	1618	if( DEBUG_VMM_GET_ONE_PPN < cycle )
[606]	1619	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
	1620	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
[433]	1621	#endif
[313]	1622
[406]	1623	// FILE type : get the physical page from the file mapper
[313]	1624	if( type == VSEG_TYPE_FILE )
	1625	{
[406]	1626	// get extended pointer on mapper
[407]	1627	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1628
[567]	1629	assert( (mapper_xp != XPTR_NULL),
	1630	"mapper not defined for a FILE vseg\n" );
[406]	1631
[606]	1632	// get extended pointer on page descriptor
	1633	page_xp = mapper_remote_get_page( mapper_xp , page_id );
[406]	1634
[606]	1635	if ( page_xp == XPTR_NULL ) return EINVAL;
[313]	1636	}
	1637
[406]	1638	// Other types : allocate a physical page from target cluster,
[407]	1639	// as defined by vseg type and vpn value
[313]	1640	else
	1641	{
[433]	1642	// allocate one physical page
[407]	1643	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1644
[407]	1645	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1646
[406]	1647	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1648	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1649	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1650	{
[406]	1651	// get extended pointer on mapper
	1652	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1653
[567]	1654	assert( (mapper_xp != XPTR_NULL),
	1655	"mapper not defined for a CODE or DATA vseg\n" );
[406]	1656
	1657	// compute missing page offset in vseg
[606]	1658	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
[406]	1659
[313]	1660	// compute missing page offset in .elf file
[406]	1661	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1662
[438]	1663	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1664	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[595]	1665	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
	1666	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
[433]	1667	#endif
[406]	1668	// compute extended pointer on page base
[407]	1669	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1670
[406]	1671	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1672	uint32_t file_size = vseg->file_size;
	1673
	1674	if( file_size < offset ) // missing page fully in BSS
[313]	1675	{
[406]	1676
[438]	1677	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1678	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[595]	1679	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
	1680	__FUNCTION__, this->process->pid, this->trdid, vpn );
[433]	1681	#endif
[407]	1682	if( GET_CXY( page_xp ) == local_cxy )
[313]	1683	{
[315]	1684	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1685	}
	1686	else
	1687	{
[315]	1688	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1689	}
	1690	}
[406]	1691	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1692	{
[406]	1693
[438]	1694	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1695	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[595]	1696	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
	1697	__FUNCTION__, this->process->pid, this->trdid, vpn );
[433]	1698	#endif
[606]	1699	error = mapper_move_kernel( mapper_xp,
	1700	true, // to_buffer
	1701	elf_offset,
	1702	base_xp,
	1703	CONFIG_PPM_PAGE_SIZE );
[313]	1704	if( error ) return EINVAL;
	1705	}
[406]	1706	else // both in mapper and in BSS :
	1707	// - (file_size - offset) bytes from mapper
	1708	// - (page_size + offset - file_size) bytes from BSS
[313]	1709	{
[406]	1710
[438]	1711	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1712	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[610]	1713	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
[433]	1714	" %d bytes from mapper / %d bytes from BSS\n",
[595]	1715	__FUNCTION__, this->process->pid, this->trdid, vpn,
[407]	1716	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1717	#endif
[313]	1718	// initialize mapper part
[606]	1719	error = mapper_move_kernel( mapper_xp,
	1720	true, // to buffer
	1721	elf_offset,
	1722	base_xp,
	1723	file_size - offset );
[313]	1724	if( error ) return EINVAL;
	1725
	1726	// initialize BSS part
[407]	1727	if( GET_CXY( page_xp ) == local_cxy )
[313]	1728	{
[406]	1729	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1730	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1731	}
	1732	else
	1733	{
[406]	1734	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1735	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1736	}
	1737	}
	1738	} // end initialisation for CODE or DATA types
	1739	}
	1740
	1741	// return ppn
[407]	1742	*ppn = ppm_page2ppn( page_xp );
[406]	1743
[438]	1744	#if DEBUG_VMM_GET_ONE_PPN
[595]	1745	cycle = (uint32_t)hal_get_cycles();
	1746	if( DEBUG_VMM_GET_ONE_PPN < cycle )
	1747	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
	1748	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
[433]	1749	#endif
[406]	1750
[313]	1751	return 0;
	1752
	1753	} // end vmm_get_one_ppn()
	1754
[585]	1755	///////////////////////////////////////////////////
	1756	error_t vmm_handle_page_fault( process_t * process,
	1757	vpn_t vpn )
[1]	1758	{
[585]	1759	vseg_t * vseg; // vseg containing vpn
	1760	uint32_t new_attr; // new PTE_ATTR value
	1761	ppn_t new_ppn; // new PTE_PPN value
	1762	uint32_t ref_attr; // PTE_ATTR value in reference GPT
	1763	ppn_t ref_ppn; // PTE_PPN value in reference GPT
	1764	cxy_t ref_cxy; // reference cluster for missing vpn
	1765	process_t * ref_ptr; // reference process for missing vpn
	1766	xptr_t local_gpt_xp; // extended pointer on local GPT
	1767	xptr_t local_lock_xp; // extended pointer on local GPT lock
	1768	xptr_t ref_gpt_xp; // extended pointer on reference GPT
	1769	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
	1770	error_t error; // value returned by called functions
[1]	1771
[585]	1772	// get local vseg (access to reference VSL can be required)
	1773	error = vmm_get_vseg( process,
	1774	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	1775	&vseg );
	1776	if( error )
	1777	{
[595]	1778	printk("\n[ERROR] in %s : vpn %x in process %x not in a registered vseg\n",
[585]	1779	__FUNCTION__ , vpn , process->pid );
	1780
	1781	return EXCP_USER_ERROR;
	1782	}
	1783
	1784	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	1785	uint32_t cycle = (uint32_t)hal_get_cycles();
[567]	1786	thread_t * this = CURRENT_THREAD;
[585]	1787	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[595]	1788	printk("\n[%s] threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
[585]	1789	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
[433]	1790	#endif
[406]	1791
[585]	1792	//////////////// private vseg => access only the local GPT
	1793	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[438]	1794	{
[585]	1795	// build extended pointer on local GPT and local GPT lock
	1796	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1797	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
[407]	1798
[585]	1799	// take local GPT lock in write mode
	1800	remote_rwlock_wr_acquire( local_lock_xp );
[407]	1801
[585]	1802	// check VPN still unmapped in local GPT
[595]	1803
[585]	1804	// do nothing if VPN has been mapped by a a concurrent page_fault
	1805	hal_gpt_get_pte( local_gpt_xp,
	1806	vpn,
	1807	&new_attr,
	1808	&new_ppn );
[407]	1809
[585]	1810	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
	1811	{
	1812	// allocate and initialise a physical page depending on the vseg type
	1813	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
[407]	1814
[585]	1815	if( error )
[408]	1816	{
	1817	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1818	__FUNCTION__ , process->pid , vpn );
[1]	1819
[585]	1820	// release local GPT lock in write mode
	1821	remote_rwlock_wr_release( local_lock_xp );
[406]	1822
[585]	1823	return EXCP_KERNEL_PANIC;
[407]	1824	}
	1825
[408]	1826	// define new_attr from vseg flags
[407]	1827	new_attr = GPT_MAPPED \| GPT_SMALL;
	1828	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1829	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1830	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1831	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
	1832
[585]	1833	// set PTE (PPN & attribute) to local GPT
	1834	error = hal_gpt_set_pte( local_gpt_xp,
[408]	1835	vpn,
	1836	new_attr,
	1837	new_ppn );
[585]	1838	if ( error )
[407]	1839	{
[585]	1840	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
[407]	1841	__FUNCTION__ , process->pid , vpn );
[585]	1842
	1843	// release local GPT lock in write mode
	1844	remote_rwlock_wr_release( local_lock_xp );
	1845
	1846	return EXCP_KERNEL_PANIC;
[407]	1847	}
	1848	}
[585]	1849
	1850	// release local GPT lock in write mode
	1851	remote_rwlock_wr_release( local_lock_xp );
	1852
	1853	#if DEBUG_VMM_HANDLE_PAGE_FAULT
	1854	cycle = (uint32_t)hal_get_cycles();
	1855	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[595]	1856	printk("\n[%s] private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
[585]	1857	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
	1858	#endif
	1859	return EXCP_NON_FATAL;
	1860
	1861	} // end local GPT access
	1862
	1863	//////////// public vseg => access reference GPT
	1864	else
	1865	{
	1866	// get reference process cluster and local pointer
	1867	ref_cxy = GET_CXY( process->ref_xp );
	1868	ref_ptr = GET_PTR( process->ref_xp );
	1869
	1870	// build extended pointer on reference GPT and reference GPT lock
	1871	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
	1872	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
	1873
	1874	// build extended pointer on local GPT and local GPT lock
	1875	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1876	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
	1877
	1878	// take reference GPT lock in read mode
	1879	remote_rwlock_rd_acquire( ref_lock_xp );
	1880
	1881	// get directly PPN & attributes from reference GPT
	1882	// this can avoids a costly RPC for a false page fault
	1883	hal_gpt_get_pte( ref_gpt_xp,
	1884	vpn,
	1885	&ref_attr,
	1886	&ref_ppn );
	1887
	1888	// release reference GPT lock in read mode
	1889	remote_rwlock_rd_release( ref_lock_xp );
	1890
	1891	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
[1]	1892	{
[585]	1893	// take local GPT lock in write mode
	1894	remote_rwlock_wr_acquire( local_lock_xp );
	1895
	1896	// check VPN still unmapped in local GPT
	1897	hal_gpt_get_pte( local_gpt_xp,
	1898	vpn,
	1899	&new_attr,
	1900	&new_ppn );
[1]	1901
[585]	1902	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
	1903	{
	1904	// update local GPT from reference GPT
	1905	error = hal_gpt_set_pte( local_gpt_xp,
	1906	vpn,
	1907	ref_attr,
	1908	ref_ppn );
	1909	if( error )
	1910	{
[595]	1911	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn %x\n",
[585]	1912	__FUNCTION__ , process->pid , vpn );
	1913
	1914	// release local GPT lock in write mode
	1915	remote_rwlock_wr_release( local_lock_xp );
	1916
	1917	return EXCP_KERNEL_PANIC;
	1918	}
	1919	}
	1920	else // VPN has been mapped by a a concurrent page_fault
	1921	{
	1922	// keep PTE from local GPT
	1923	ref_attr = new_attr;
	1924	ref_ppn = new_ppn;
	1925	}
	1926
	1927	// release local GPT lock in write mode
	1928	remote_rwlock_wr_release( local_lock_xp );
	1929
	1930	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[433]	1931	cycle = (uint32_t)hal_get_cycles();
[585]	1932	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[595]	1933	printk("\n[%s] false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
[585]	1934	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
[433]	1935	#endif
[585]	1936	return EXCP_NON_FATAL;
	1937	}
	1938	else // true page fault => update reference GPT
	1939	{
	1940	// take reference GPT lock in write mode
	1941	remote_rwlock_wr_acquire( ref_lock_xp );
	1942
	1943	// check VPN still unmapped in reference GPT
	1944	// do nothing if VPN has been mapped by a a concurrent page_fault
	1945	hal_gpt_get_pte( ref_gpt_xp,
	1946	vpn,
	1947	&ref_attr,
	1948	&ref_ppn );
[406]	1949
[585]	1950	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
	1951	{
	1952	// allocate and initialise a physical page depending on the vseg type
	1953	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
[1]	1954
[585]	1955	if( error )
	1956	{
	1957	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1958	__FUNCTION__ , process->pid , vpn );
[313]	1959
[585]	1960	// release reference GPT lock in write mode
	1961	remote_rwlock_wr_release( ref_lock_xp );
	1962
	1963	return EXCP_KERNEL_PANIC;
	1964	}
[1]	1965
[585]	1966	// define new_attr from vseg flags
	1967	new_attr = GPT_MAPPED \| GPT_SMALL;
	1968	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1969	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1970	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1971	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
[440]	1972
[585]	1973	// update reference GPT
	1974	error = hal_gpt_set_pte( ref_gpt_xp,
	1975	vpn,
	1976	new_attr,
	1977	new_ppn );
	1978
	1979	// update local GPT (protected by reference GPT lock)
	1980	error \|= hal_gpt_set_pte( local_gpt_xp,
	1981	vpn,
	1982	new_attr,
	1983	new_ppn );
	1984
	1985	if( error )
	1986	{
	1987	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
	1988	__FUNCTION__ , process->pid , vpn );
	1989
	1990	// release reference GPT lock in write mode
	1991	remote_rwlock_wr_release( ref_lock_xp );
	1992
	1993	return EXCP_KERNEL_PANIC;
	1994	}
	1995	}
	1996
	1997	// release reference GPT lock in write mode
	1998	remote_rwlock_wr_release( ref_lock_xp );
	1999
[440]	2000	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[585]	2001	cycle = (uint32_t)hal_get_cycles();
[469]	2002	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[595]	2003	printk("\n[%s] true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
[585]	2004	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
[435]	2005	#endif
[585]	2006	return EXCP_NON_FATAL;
	2007	}
	2008	}
	2009	} // end vmm_handle_page_fault()
[435]	2010
[585]	2011	////////////////////////////////////////////
	2012	error_t vmm_handle_cow( process_t * process,
	2013	vpn_t vpn )
	2014	{
	2015	vseg_t * vseg; // vseg containing vpn
	2016	cxy_t ref_cxy; // reference cluster for missing vpn
	2017	process_t * ref_ptr; // reference process for missing vpn
	2018	xptr_t gpt_xp; // extended pointer on GPT
	2019	xptr_t gpt_lock_xp; // extended pointer on GPT lock
	2020	uint32_t old_attr; // current PTE_ATTR value
	2021	ppn_t old_ppn; // current PTE_PPN value
	2022	uint32_t new_attr; // new PTE_ATTR value
	2023	ppn_t new_ppn; // new PTE_PPN value
	2024	error_t error;
[1]	2025
[585]	2026	#if DEBUG_VMM_HANDLE_COW
[619]	2027	uint32_t cycle = (uint32_t)hal_get_cycles();
	2028	thread_t * this = CURRENT_THREAD;
	2029	xptr_t this_xp = XPTR( local_cxy , this );
[585]	2030	if( DEBUG_VMM_HANDLE_COW < cycle )
[595]	2031	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
[619]	2032	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
[585]	2033	#endif
	2034
[610]	2035	// access local GPT to get GPT_COW flag
	2036	bool_t cow = hal_gpt_pte_is_cow( &(process->vmm.gpt), vpn );
	2037
	2038	if( cow == false ) return EXCP_USER_ERROR;
	2039
[585]	2040	// get local vseg
	2041	error = vmm_get_vseg( process,
	2042	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	2043	&vseg );
[440]	2044	if( error )
[1]	2045	{
[595]	2046	printk("\n[PANIC] in %s : vpn %x in process %x not in a registered vseg\n",
[585]	2047	__FUNCTION__, vpn, process->pid );
	2048
	2049	return EXCP_KERNEL_PANIC;
[440]	2050	}
[407]	2051
[619]	2052	#if( DEBUG_VMM_HANDLE_COW & 1)
	2053	if( DEBUG_VMM_HANDLE_COW < cycle )
	2054	printk("\n[%s] thread[%x,%x] get vseg for vpn %x\n",
	2055	__FUNCTION__, this->process->pid, this->trdid, vpn );
	2056	#endif
	2057
[585]	2058	// get reference GPT cluster and local pointer
	2059	ref_cxy = GET_CXY( process->ref_xp );
	2060	ref_ptr = GET_PTR( process->ref_xp );
[407]	2061
[610]	2062	// build relevant extended pointers on relevant GPT and GPT lock
[585]	2063	// - access local GPT for a private vseg
	2064	// - access reference GPT for a public vseg
	2065	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[440]	2066	{
[585]	2067	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	2068	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
[1]	2069	}
[440]	2070	else
[1]	2071	{
[585]	2072	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
	2073	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
[1]	2074	}
	2075
[585]	2076	// take GPT lock in write mode
	2077	remote_rwlock_wr_acquire( gpt_lock_xp );
[441]	2078
[585]	2079	// get current PTE from reference GPT
	2080	hal_gpt_get_pte( gpt_xp,
	2081	vpn,
	2082	&old_attr,
	2083	&old_ppn );
[441]	2084
[619]	2085	#if( DEBUG_VMM_HANDLE_COW & 1)
	2086	if( DEBUG_VMM_HANDLE_COW < cycle )
	2087	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
	2088	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
	2089	#endif
	2090
[585]	2091	// the PTE must be mapped for a COW
	2092	if( (old_attr & GPT_MAPPED) == 0 )
	2093	{
	2094	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
	2095	__FUNCTION__, vpn, process->pid );
[407]	2096
[585]	2097	// release GPT lock in write mode
[619]	2098	remote_rwlock_wr_release( gpt_lock_xp );
[407]	2099
[585]	2100	return EXCP_KERNEL_PANIC;
[407]	2101	}
	2102
[619]	2103	// get pointers on physical page descriptor
[585]	2104	xptr_t page_xp = ppm_ppn2page( old_ppn );
	2105	cxy_t page_cxy = GET_CXY( page_xp );
	2106	page_t * page_ptr = GET_PTR( page_xp );
[435]	2107
[585]	2108	// get extended pointers on forks and lock field in page descriptor
	2109	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	2110	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
[407]	2111
[585]	2112	// take lock protecting "forks" counter
	2113	remote_busylock_acquire( forks_lock_xp );
[407]	2114
[585]	2115	// get number of pending forks from page descriptor
	2116	uint32_t forks = hal_remote_l32( forks_xp );
[441]	2117
[619]	2118	#if( DEBUG_VMM_HANDLE_COW & 1)
	2119	if( DEBUG_VMM_HANDLE_COW < cycle )
	2120	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
	2121	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
	2122	#endif
	2123
[585]	2124	if( forks ) // pending fork => allocate a new page, and copy old to new
	2125	{
[619]	2126	// decrement pending forks counter in page descriptor
	2127	hal_remote_atomic_add( forks_xp , -1 );
	2128
	2129	// release lock protecting "forks" counter
	2130	remote_busylock_release( forks_lock_xp );
	2131
	2132	// allocate a new page
[585]	2133	page_xp = vmm_page_allocate( vseg , vpn );
[619]	2134
[585]	2135	if( page_xp == XPTR_NULL )
	2136	{
	2137	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
	2138	__FUNCTION__ , vpn, process->pid );
[441]	2139
[585]	2140	// release GPT lock in write mode
	2141	remote_rwlock_wr_acquire( gpt_lock_xp );
[441]	2142
[585]	2143	return EXCP_KERNEL_PANIC;
	2144	}
[441]	2145
[585]	2146	// compute allocated page PPN
	2147	new_ppn = ppm_page2ppn( page_xp );
[441]	2148
[619]	2149	#if( DEBUG_VMM_HANDLE_COW & 1)
	2150	if( DEBUG_VMM_HANDLE_COW < cycle )
	2151	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
	2152	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
	2153	#endif
	2154
[585]	2155	// copy old page content to new page
[619]	2156	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
	2157	ppm_ppn2base( old_ppn ),
	2158	CONFIG_PPM_PAGE_SIZE );
[441]	2159
[585]	2160	#if(DEBUG_VMM_HANDLE_COW & 1)
	2161	if( DEBUG_VMM_HANDLE_COW < cycle )
[619]	2162	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
	2163	__FUNCTION__, this->process->pid, this->trdid );
[585]	2164	#endif
[440]	2165
[585]	2166	}
	2167	else // no pending fork => keep the existing page
	2168	{
[619]	2169	// release lock protecting "forks" counter
	2170	remote_busylock_release( forks_lock_xp );
[1]	2171
[585]	2172	#if(DEBUG_VMM_HANDLE_COW & 1)
	2173	if( DEBUG_VMM_HANDLE_COW < cycle )
[619]	2174	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
	2175	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
[585]	2176	#endif
	2177	new_ppn = old_ppn;
	2178	}
[1]	2179
[585]	2180	// build new_attr : reset COW and set WRITABLE,
	2181	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
	2182
[619]	2183	// update the relevant GPT
[585]	2184	// - private vseg => update local GPT
	2185	// - public vseg => update all GPT copies
	2186	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[1]	2187	{
[585]	2188	hal_gpt_set_pte( gpt_xp,
	2189	vpn,
	2190	new_attr,
	2191	new_ppn );
[1]	2192	}
[585]	2193	else
[1]	2194	{
[585]	2195	if( ref_cxy == local_cxy ) // reference cluster is local
	2196	{
	2197	vmm_global_update_pte( process,
	2198	vpn,
	2199	new_attr,
	2200	new_ppn );
	2201	}
	2202	else // reference cluster is remote
	2203	{
	2204	rpc_vmm_global_update_pte_client( ref_cxy,
	2205	ref_ptr,
	2206	vpn,
	2207	new_attr,
	2208	new_ppn );
	2209	}
[1]	2210	}
	2211
[585]	2212	// release GPT lock in write mode
	2213	remote_rwlock_wr_release( gpt_lock_xp );
[21]	2214
[585]	2215	#if DEBUG_VMM_HANDLE_COW
	2216	cycle = (uint32_t)hal_get_cycles();
	2217	if( DEBUG_VMM_HANDLE_COW < cycle )
[595]	2218	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
[619]	2219	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
[585]	2220	#endif
[313]	2221
[585]	2222	return EXCP_NON_FATAL;
[1]	2223
[585]	2224	} // end vmm_handle_cow()
	2225

Note: See TracBrowser for help on using the repository browser.

Download in other formats: