Context Navigation

source: trunk/kernel/mm/vmm.c @ 484

Last change on this file since 484 was 473, checked in by alain, 6 years ago
Fix several GCC warning related to the -Wextra compilation option.
File size: 66.1 KB

Rev	Line
[1]	1	/*
	2	* vmm.c - virtual memory manager related operations interface.
	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
	6	* Alain Greiner (2016)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[1]	31	#include <printk.h>
[23]	32	#include <memcpy.h>
[1]	33	#include <rwlock.h>
	34	#include <list.h>
[408]	35	#include <xlist.h>
[1]	36	#include <bits.h>
	37	#include <process.h>
	38	#include <thread.h>
	39	#include <vseg.h>
	40	#include <cluster.h>
	41	#include <scheduler.h>
	42	#include <vfs.h>
	43	#include <mapper.h>
	44	#include <page.h>
	45	#include <kmem.h>
	46	#include <vmm.h>
	47
	48	//////////////////////////////////////////////////////////////////////////////////
	49	// Extern global variables
	50	//////////////////////////////////////////////////////////////////////////////////
	51
	52	extern process_t process_zero; // defined in cluster.c file
	53
	54
[415]	55	///////////////////////////////////////
	56	error_t vmm_init( process_t * process )
[21]	57	{
[1]	58	error_t error;
	59	vseg_t * vseg_kentry;
	60	vseg_t * vseg_args;
	61	vseg_t * vseg_envs;
	62	intptr_t base;
	63	intptr_t size;
	64
[438]	65	#if DEBUG_VMM_INIT
[433]	66	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	67	if( DEBUG_VMM_INIT )
[433]	68	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	69	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	70	#endif
[204]	71
[1]	72	// get pointer on VMM
	73	vmm_t * vmm = &process->vmm;
	74
[407]	75	// initialize local list of vsegs
	76	vmm->vsegs_nr = 0;
[408]	77	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
	78	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	79
[204]	80	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
	81	<= CONFIG_VMM_ELF_BASE) , __FUNCTION__ , "UTILS zone too small\n" );
[21]	82
[204]	83	assert( (CONFIG_THREAD_MAX_PER_CLUSTER <= 32) , __FUNCTION__ ,
	84	"no more than 32 threads per cluster for a single process\n");
[1]	85
[204]	86	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREAD_MAX_PER_CLUSTER) <=
	87	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) , __FUNCTION__ ,
	88	"STACK zone too small\n");
[1]	89
[409]	90	// register kentry vseg in VSL
[406]	91	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	92	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	93
[407]	94	vseg_kentry = vmm_create_vseg( process,
	95	VSEG_TYPE_CODE,
	96	base,
	97	size,
	98	0, // file_offset unused
	99	0, // file_size unused
	100	XPTR_NULL, // mapper_xp unused
	101	local_cxy );
[204]	102
[415]	103	if( vseg_kentry == NULL )
	104	{
	105	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
	106	return -1;
	107	}
[204]	108
[406]	109	vmm->kent_vpn_base = base;
[1]	110
[409]	111	// register args vseg in VSL
[406]	112	base = (CONFIG_VMM_KENTRY_BASE +
	113	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	114	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	115
[407]	116	vseg_args = vmm_create_vseg( process,
	117	VSEG_TYPE_DATA,
	118	base,
	119	size,
	120	0, // file_offset unused
	121	0, // file_size unused
	122	XPTR_NULL, // mapper_xp unused
	123	local_cxy );
[204]	124
[415]	125	if( vseg_args == NULL )
	126	{
	127	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	128	return -1;
	129	}
[204]	130
[406]	131	vmm->args_vpn_base = base;
[1]	132
[409]	133	// register the envs vseg in VSL
[406]	134	base = (CONFIG_VMM_KENTRY_BASE +
	135	CONFIG_VMM_KENTRY_SIZE +
	136	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
[1]	137	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	138
[407]	139	vseg_envs = vmm_create_vseg( process,
	140	VSEG_TYPE_DATA,
	141	base,
	142	size,
	143	0, // file_offset unused
	144	0, // file_size unused
	145	XPTR_NULL, // mapper_xp unused
	146	local_cxy );
[204]	147
[415]	148	if( vseg_envs == NULL )
	149	{
	150	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	151	return -1;
	152	}
[204]	153
[406]	154	vmm->envs_vpn_base = base;
[1]	155
[409]	156	// create GPT (empty)
[1]	157	error = hal_gpt_create( &vmm->gpt );
	158
[415]	159	if( error )
	160	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
[204]	161
[415]	162	// initialize GPT (architecture specic)
[409]	163	// (For TSAR, identity map the kentry_vseg)
	164	error = hal_vmm_init( vmm );
	165
[415]	166	if( error )
	167	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
[409]	168
[1]	169	// initialize STACK allocator
	170	vmm->stack_mgr.bitmap = 0;
	171	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[457]	172	spinlock_init( &vmm->stack_mgr.lock );
[1]	173
	174	// initialize MMAP allocator
[407]	175	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	176	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	177	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[457]	178	spinlock_init( &vmm->mmap_mgr.lock );
	179
[1]	180	uint32_t i;
	181	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
	182
[21]	183	// initialize instrumentation counters
[409]	184	vmm->pgfault_nr = 0;
[1]	185
[124]	186	hal_fence();
[1]	187
[438]	188	#if DEBUG_VMM_INIT
[433]	189	cycle = (uint32_t)hal_get_cycles();
[438]	190	if( DEBUG_VMM_INIT )
[433]	191	printk("\n[DBG] %s : thread %x exit for process %x / entry_point = %x / cycle %d\n",
	192	__FUNCTION__ , CURRENT_THREAD , process->pid , process->vmm.entry_point , cycle );
	193	#endif
[204]	194
[415]	195	return 0;
	196
[204]	197	} // end vmm_init()
	198
[407]	199	//////////////////////////////////////
	200	void vmm_display( process_t * process,
	201	bool_t mapping )
	202	{
	203	vmm_t * vmm = &process->vmm;
	204	gpt_t * gpt = &vmm->gpt;
	205
[457]	206	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
	207	process->vmm.gpt.ptr , process->pid , local_cxy );
[407]	208
	209	// get lock protecting the vseg list
[408]	210	remote_rwlock_rd_lock( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	211
	212	// scan the list of vsegs
[408]	213	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	214	xptr_t iter_xp;
	215	xptr_t vseg_xp;
[407]	216	vseg_t * vseg;
[408]	217	XLIST_FOREACH( root_xp , iter_xp )
[407]	218	{
[408]	219	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	220	vseg = GET_PTR( vseg_xp );
[408]	221
[407]	222	printk(" - %s : base = %X / size = %X / npages = %d\n",
	223	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
	224
	225	if( mapping )
	226	{
	227	vpn_t vpn;
	228	ppn_t ppn;
	229	uint32_t attr;
	230	vpn_t base = vseg->vpn_base;
	231	vpn_t size = vseg->vpn_size;
	232	for( vpn = base ; vpn < (base+size) ; vpn++ )
	233	{
	234	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	235	if( attr & GPT_MAPPED )
	236	{
	237	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
	238	}
	239	}
	240	}
	241	}
	242
	243	// release the lock
[408]	244	remote_rwlock_rd_unlock( XPTR( local_cxy , &vmm->vsegs_lock ) );
[407]	245
[408]	246	} // vmm_display()
	247
[433]	248	/////////////////////i//////////////////////////
	249	void vmm_global_update_pte( process_t * process,
	250	vpn_t vpn,
	251	uint32_t attr,
	252	ppn_t ppn )
[23]	253	{
	254
[408]	255	xlist_entry_t * process_root_ptr;
	256	xptr_t process_root_xp;
	257	xptr_t process_iter_xp;
[23]	258
[408]	259	xptr_t remote_process_xp;
	260	cxy_t remote_process_cxy;
	261	process_t * remote_process_ptr;
	262	xptr_t remote_gpt_xp;
[23]	263
[408]	264	pid_t pid;
	265	cxy_t owner_cxy;
	266	lpid_t owner_lpid;
[23]	267
[438]	268	#if DEBUG_VMM_UPDATE_PTE
[433]	269	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	270	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	271	printk("\n[DBG] %s : thread %x enter for process %x / vpn %x / cycle %d\n",
	272	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	273	#endif
	274
	275	// check cluster is reference
	276	assert( (GET_CXY( process->ref_xp ) == local_cxy) , __FUNCTION__,
	277	"not called in reference cluster\n");
	278
[408]	279	// get extended pointer on root of process copies xlist in owner cluster
	280	pid = process->pid;
	281	owner_cxy = CXY_FROM_PID( pid );
	282	owner_lpid = LPID_FROM_PID( pid );
	283	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	284	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	285
[408]	286	// loop on destination process copies
	287	XLIST_FOREACH( process_root_xp , process_iter_xp )
	288	{
	289	// get cluster and local pointer on remote process
	290	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	291	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	292	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	293
[438]	294	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
	295	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	296	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	297	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	298	#endif
	299
[408]	300	// get extended pointer on remote gpt
	301	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	302
[433]	303	// update remote GPT
	304	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	305	}
	306
[438]	307	#if DEBUG_VMM_UPDATE_PTE
[433]	308	cycle = (uint32_t)hal_get_cycles();
[438]	309	if( DEBUG_VMM_UPDATE_PTE < cycle )
[433]	310	printk("\n[DBG] %s : thread %x exit for process %x / vpn %x / cycle %d\n",
	311	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
	312	#endif
	313
	314	} // end vmm_global_update_pte()
	315
[408]	316	///////////////////////////////////////
	317	void vmm_set_cow( process_t * process )
	318	{
	319	vmm_t * vmm;
	320
	321	xlist_entry_t * process_root_ptr;
	322	xptr_t process_root_xp;
	323	xptr_t process_iter_xp;
	324
	325	xptr_t remote_process_xp;
	326	cxy_t remote_process_cxy;
	327	process_t * remote_process_ptr;
	328	xptr_t remote_gpt_xp;
	329
	330	xptr_t vseg_root_xp;
	331	xptr_t vseg_iter_xp;
	332
	333	xptr_t vseg_xp;
	334	vseg_t * vseg;
	335
	336	pid_t pid;
	337	cxy_t owner_cxy;
	338	lpid_t owner_lpid;
	339
[438]	340	#if DEBUG_VMM_SET_COW
[433]	341	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	342	if( DEBUG_VMM_SET_COW < cycle )
[433]	343	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
	344	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	345	#endif
[408]	346
	347	// check cluster is reference
	348	assert( (GET_CXY( process->ref_xp ) == local_cxy) , __FUNCTION__,
	349	"local cluster is not process reference cluster\n");
	350
	351	// get pointer on reference VMM
	352	vmm = &process->vmm;
	353
	354	// get extended pointer on root of process copies xlist in owner cluster
	355	pid = process->pid;
	356	owner_cxy = CXY_FROM_PID( pid );
	357	owner_lpid = LPID_FROM_PID( pid );
	358	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	359	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	360
	361	// get extended pointer on root of vsegs xlist from reference VMM
	362	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	363
	364	// loop on destination process copies
	365	XLIST_FOREACH( process_root_xp , process_iter_xp )
	366	{
	367	// get cluster and local pointer on remote process
	368	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	369	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	370	remote_process_cxy = GET_CXY( remote_process_xp );
	371
[438]	372	#if (DEBUG_VMM_SET_COW &0x1)
	373	if( DEBUG_VMM_SET_COW < cycle )
[433]	374	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
	375	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
	376	#endif
[408]	377
	378	// get extended pointer on remote gpt
	379	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	380
	381	// loop on vsegs in (local) reference process VSL
	382	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	383	{
	384	// get pointer on vseg
	385	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	386	vseg = GET_PTR( vseg_xp );
[408]	387
	388	assert( (GET_CXY( vseg_xp ) == local_cxy) , __FUNCTION__,
	389	"all vsegs in reference VSL must be local\n" );
	390
	391	// get vseg type, base and size
	392	uint32_t type = vseg->type;
	393	vpn_t vpn_base = vseg->vpn_base;
	394	vpn_t vpn_size = vseg->vpn_size;
	395
[438]	396	#if (DEBUG_VMM_SET_COW & 0x1)
	397	if( DEBUG_VMM_SET_COW < cycle )
[433]	398	printk("\n[DBG] %s : thread %x handling vseg %s / vpn_base = %x / vpn_size = %x\n",
	399	__FUNCTION__, CURRENT_THREAD , vseg_type_str(type), vpn_base, vpn_size );
	400	#endif
	401	// only DATA, ANON and REMOTE vsegs
[408]	402	if( (type == VSEG_TYPE_DATA) \|\|
	403	(type == VSEG_TYPE_ANON) \|\|
	404	(type == VSEG_TYPE_REMOTE) )
	405	{
[433]	406	vpn_t vpn;
	407	uint32_t attr;
	408	ppn_t ppn;
	409	xptr_t page_xp;
	410	cxy_t page_cxy;
	411	page_t * page_ptr;
	412	xptr_t forks_xp;
[469]	413	xptr_t lock_xp;
[433]	414
	415	// update flags in remote GPT
	416	hal_gpt_set_cow( remote_gpt_xp,
	417	vpn_base,
	418	vpn_size );
	419
	420	// atomically increment pending forks counter in physical pages,
	421	// for all vseg pages that are mapped in reference cluster
	422	if( remote_process_cxy == local_cxy )
	423	{
	424	// the reference GPT is the local GPT
	425	gpt_t * gpt = GET_PTR( remote_gpt_xp );
	426
	427	// scan all pages in vseg
	428	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	429	{
	430	// get page attributes and PPN from reference GPT
	431	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	432
	433	// atomically update pending forks counter if page is mapped
	434	if( attr & GPT_MAPPED )
	435	{
[469]	436	// get pointers and cluster on page descriptor
[433]	437	page_xp = ppm_ppn2page( ppn );
	438	page_cxy = GET_CXY( page_xp );
	439	page_ptr = GET_PTR( page_xp );
[469]	440
	441	// get extended pointers on "forks" and "lock"
[433]	442	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	443	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	444
	445	// increment "forks"
	446	remote_spinlock_lock( lock_xp );
[433]	447	hal_remote_atomic_add( forks_xp , 1 );
[469]	448	remote_spinlock_unlock( lock_xp );
[433]	449	}
	450	} // end loop on vpn
	451	} // end if local
	452	} // end if vseg type
	453	} // end loop on vsegs
[408]	454	} // end loop on process copies
	455
[438]	456	#if DEBUG_VMM_SET_COW
[433]	457	cycle = (uint32_t)hal_get_cycles();
[438]	458	if( DEBUG_VMM_SET_COW < cycle )
[433]	459	printk("\n[DBG] %s : thread %x exit for process %x / cycle %d\n",
	460	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
	461	#endif
[408]	462
	463	} // end vmm_set-cow()
	464
	465	/////////////////////////////////////////////////
	466	error_t vmm_fork_copy( process_t * child_process,
	467	xptr_t parent_process_xp )
	468	{
	469	error_t error;
	470	cxy_t parent_cxy;
	471	process_t * parent_process;
	472	vmm_t * parent_vmm;
	473	xptr_t parent_lock_xp;
	474	vmm_t * child_vmm;
	475	xptr_t iter_xp;
	476	xptr_t parent_vseg_xp;
	477	vseg_t * parent_vseg;
	478	vseg_t * child_vseg;
	479	uint32_t type;
	480	bool_t cow;
	481	vpn_t vpn;
	482	vpn_t vpn_base;
	483	vpn_t vpn_size;
[469]	484	xptr_t page_xp; // extended pointer on page descriptor
[408]	485	page_t * page_ptr;
	486	cxy_t page_cxy;
[469]	487	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
	488	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
[408]	489	xptr_t parent_root_xp;
	490	bool_t mapped;
	491	ppn_t ppn;
	492
[438]	493	#if DEBUG_VMM_FORK_COPY
[433]	494	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	495	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	496	printk("\n[DBG] %s : thread %x enter / cycle %d\n",
	497	__FUNCTION__ , CURRENT_THREAD, cycle );
	498	#endif
[408]	499
	500	// get parent process cluster and local pointer
	501	parent_cxy = GET_CXY( parent_process_xp );
[433]	502	parent_process = GET_PTR( parent_process_xp );
[408]	503
	504	// get local pointers on parent and child VMM
	505	parent_vmm = &parent_process->vmm;
	506	child_vmm = &child_process->vmm;
	507
	508	// get extended pointer on lock protecting the parent VSL
	509	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
	510
	511	// initialize the lock protecting the child VSL
	512	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ) );
	513
	514	// initialize the child VSL as empty
	515	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
	516	child_vmm->vsegs_nr = 0;
	517
[415]	518	// create child GPT
[408]	519	error = hal_gpt_create( &child_vmm->gpt );
[415]	520
[407]	521	if( error )
	522	{
[408]	523	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	524	return -1;
[407]	525	}
	526
[408]	527	// build extended pointer on parent VSL
	528	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
	529
[415]	530	// take the lock protecting the parent VSL
	531	remote_rwlock_rd_lock( parent_lock_xp );
	532
[408]	533	// loop on parent VSL xlist
	534	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	535	{
[408]	536	// get local and extended pointers on current parent vseg
	537	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	538	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	539
[408]	540	// get vseg type
	541	type = hal_remote_lw( XPTR( parent_cxy , &parent_vseg->type ) );
	542
[438]	543	#if DEBUG_VMM_FORK_COPY
[433]	544	cycle = (uint32_t)hal_get_cycles();
[438]	545	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	546	printk("\n[DBG] %s : thread %x found parent vseg %s / vpn_base = %x / cycle %d\n",
	547	__FUNCTION__ , CURRENT_THREAD, vseg_type_str(type),
	548	hal_remote_lw( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
	549	#endif
[23]	550
[408]	551	// all parent vsegs - but STACK - must be copied in child VSL
	552	if( type != VSEG_TYPE_STACK )
[23]	553	{
[408]	554	// allocate memory for a new child vseg
	555	child_vseg = vseg_alloc();
	556	if( child_vseg == NULL ) // release all allocated vsegs
[23]	557	{
[408]	558	vmm_destroy( child_process );
	559	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	560	return -1;
[23]	561	}
	562
[408]	563	// copy parent vseg to child vseg
	564	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	565
[408]	566	// register child vseg in child VSL
	567	vseg_attach( child_vmm , child_vseg );
[407]	568
[438]	569	#if DEBUG_VMM_FORK_COPY
[433]	570	cycle = (uint32_t)hal_get_cycles();
[438]	571	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	572	printk("\n[DBG] %s : thread %x copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	573	__FUNCTION__ , CURRENT_THREAD , vseg_type_str(type),
	574	hal_remote_lw( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
	575	#endif
[23]	576
[408]	577	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
	578	if( type != VSEG_TYPE_CODE )
	579	{
	580	// activate the COW for DATA, MMAP, REMOTE vsegs only
	581	cow = ( type != VSEG_TYPE_FILE );
[23]	582
[408]	583	vpn_base = child_vseg->vpn_base;
	584	vpn_size = child_vseg->vpn_size;
[23]	585
[408]	586	// scan pages in parent vseg
	587	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	588	{
	589	error = hal_gpt_pte_copy( &child_vmm->gpt,
	590	XPTR( parent_cxy , &parent_vmm->gpt ),
	591	vpn,
	592	cow,
	593	&ppn,
	594	&mapped );
	595	if( error )
	596	{
	597	vmm_destroy( child_process );
	598	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	599	return -1;
	600	}
	601
[433]	602	// increment pending forks counter in page if mapped
[408]	603	if( mapped )
	604	{
[469]	605	// get pointers and cluster on page descriptor
	606	page_xp = ppm_ppn2page( ppn );
[408]	607	page_cxy = GET_CXY( page_xp );
[433]	608	page_ptr = GET_PTR( page_xp );
[408]	609
[469]	610	// get extended pointers on "forks" and "lock"
	611	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	612	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	613
	614	// increment "forks"
	615	remote_spinlock_lock( lock_xp );
	616	hal_remote_atomic_add( forks_xp , 1 );
	617	remote_spinlock_unlock( lock_xp );
	618
[438]	619	#if DEBUG_VMM_FORK_COPY
[433]	620	cycle = (uint32_t)hal_get_cycles();
[438]	621	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	622	printk("\n[DBG] %s : thread %x copied vpn %x to child GPT / cycle %d\n",
	623	__FUNCTION__ , CURRENT_THREAD , vpn , cycle );
	624	#endif
[408]	625	}
	626	}
	627	} // end if no code & no stack
	628	} // end if no stack
	629	} // end loop on vsegs
	630
	631	// release the parent vsegs lock
	632	remote_rwlock_rd_unlock( parent_lock_xp );
	633
[415]	634	// initialize child GPT (architecture specic)
	635	// => For TSAR, identity map the kentry_vseg
	636	error = hal_vmm_init( child_vmm );
	637
	638	if( error )
	639	{
	640	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
	641	return -1;
	642	}
	643
[408]	644	// initialize the child VMM STACK allocator
	645	child_vmm->stack_mgr.bitmap = 0;
	646	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	647
	648	// initialize the child VMM MMAP allocator
[23]	649	uint32_t i;
[408]	650	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	651	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	652	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
	653	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
[23]	654
[178]	655	// initialize instrumentation counters
[408]	656	child_vmm->pgfault_nr = 0;
[23]	657
[408]	658	// copy base addresses from parent VMM to child VMM
	659	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
	660	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	661	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	662	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	663	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	664	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	665
[408]	666	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	667
[124]	668	hal_fence();
[23]	669
[438]	670	#if DEBUG_VMM_FORK_COPY
[433]	671	cycle = (uint32_t)hal_get_cycles();
[438]	672	if( DEBUG_VMM_FORK_COPY < cycle )
[433]	673	printk("\n[DBG] %s : thread %x exit successfully / cycle %d\n",
	674	__FUNCTION__ , CURRENT_THREAD , cycle );
	675	#endif
	676
[23]	677	return 0;
	678
[408]	679	} // vmm_fork_copy()
[204]	680
[1]	681	///////////////////////////////////////
	682	void vmm_destroy( process_t * process )
	683	{
[408]	684	xptr_t vseg_xp;
[1]	685	vseg_t * vseg;
	686
[438]	687	#if DEBUG_VMM_DESTROY
[433]	688	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	689	if( DEBUG_VMM_DESTROY < cycle )
[443]	690	printk("\n[DBG] %s : thread %x enter for process %x in cluster %x / cycle %d\n",
[469]	691	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy, cycle );
[433]	692	#endif
[416]	693
[438]	694	#if (DEBUG_VMM_DESTROY & 1 )
[443]	695	if( DEBUG_VMM_DESTROY < cycle )
[437]	696	vmm_display( process , true );
	697	#endif
	698
[433]	699	// get pointer on local VMM
[1]	700	vmm_t * vmm = &process->vmm;
	701
[408]	702	// get extended pointer on VSL root and VSL lock
	703	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	704	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	705
[1]	706	// get lock protecting vseg list
[408]	707	remote_rwlock_wr_lock( lock_xp );
[1]	708
[409]	709	// remove all user vsegs registered in VSL
[408]	710	while( !xlist_is_empty( root_xp ) )
[1]	711	{
[409]	712	// get pointer on first vseg in VSL
[408]	713	vseg_xp = XLIST_FIRST_ELEMENT( root_xp , vseg_t , xlist );
[433]	714	vseg = GET_PTR( vseg_xp );
[409]	715
[437]	716	// unmap and release physical pages
[409]	717	vmm_unmap_vseg( process , vseg );
	718
	719	// remove vseg from VSL
[473]	720	vseg_detach( vseg );
[409]	721
	722	// release memory allocated to vseg descriptor
[1]	723	vseg_free( vseg );
[443]	724
	725	#if( DEBUG_VMM_DESTROY & 1 )
	726	if( DEBUG_VMM_DESTROY < cycle )
	727	printk("\n[DBG] %s : %s vseg released / vpn_base %x / vpn_size %d\n",
	728	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	729	#endif
	730
[1]	731	}
	732
[433]	733	// release lock protecting VSL
[408]	734	remote_rwlock_wr_unlock( lock_xp );
[1]	735
	736	// remove all vsegs from zombi_lists in MMAP allocator
	737	uint32_t i;
	738	for( i = 0 ; i<32 ; i++ )
	739	{
	740	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
	741	{
[408]	742	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
[443]	743
	744	#if( DEBUG_VMM_DESTROY & 1 )
	745	if( DEBUG_VMM_DESTROY < cycle )
	746	printk("\n[DBG] %s : found zombi vseg / vpn_base %x / vpn_size %d\n",
	747	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	748	#endif
[473]	749	vseg_detach( vseg );
[1]	750	vseg_free( vseg );
[443]	751
	752	#if( DEBUG_VMM_DESTROY & 1 )
	753	if( DEBUG_VMM_DESTROY < cycle )
	754	printk("\n[DBG] %s : zombi vseg released / vpn_base %x / vpn_size %d\n",
	755	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	756	#endif
[1]	757	}
	758	}
	759
[409]	760	// release memory allocated to the GPT itself
[1]	761	hal_gpt_destroy( &vmm->gpt );
	762
[438]	763	#if DEBUG_VMM_DESTROY
[433]	764	cycle = (uint32_t)hal_get_cycles();
[438]	765	if( DEBUG_VMM_DESTROY < cycle )
[443]	766	printk("\n[DBG] %s : thread %x exit for process %x in cluster %x / cycle %d\n",
[469]	767	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy , cycle );
[433]	768	#endif
[416]	769
[204]	770	} // end vmm_destroy()
	771
[1]	772	/////////////////////////////////////////////////
	773	vseg_t * vmm_check_conflict( process_t * process,
[21]	774	vpn_t vpn_base,
[1]	775	vpn_t vpn_size )
	776	{
	777	vmm_t * vmm = &process->vmm;
[408]	778
	779	// scan the VSL
[1]	780	vseg_t * vseg;
[408]	781	xptr_t iter_xp;
	782	xptr_t vseg_xp;
	783	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	784
[408]	785	XLIST_FOREACH( root_xp , iter_xp )
[1]	786	{
[408]	787	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	788	vseg = GET_PTR( vseg_xp );
[204]	789
[21]	790	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	791	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	792	}
	793	return NULL;
	794
[204]	795	} // end vmm_check_conflict()
	796
[1]	797	////////////////////////////////////////////////////////////////////////////////////////////
	798	// This static function is called by the vmm_create_vseg() function, and implements
	799	// the VMM stack_vseg specific allocator.
	800	////////////////////////////////////////////////////////////////////////////////////////////
	801	// @ vmm : pointer on VMM.
[21]	802	// @ vpn_base : (return value) first allocated page
[1]	803	// @ vpn_size : (return value) number of allocated pages
	804	////////////////////////////////////////////////////////////////////////////////////////////
	805	static error_t vmm_stack_alloc( vmm_t * vmm,
	806	vpn_t * vpn_base,
	807	vpn_t * vpn_size )
	808	{
	809	// get stack allocator pointer
	810	stack_mgr_t * mgr = &vmm->stack_mgr;
	811
	812	// get lock on stack allocator
	813	spinlock_lock( &mgr->lock );
	814
	815	// get first free slot index in bitmap
	816	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
[179]	817	if( (index < 0) \|\| (index > 31) )
	818	{
	819	spinlock_unlock( &mgr->lock );
	820	return ENOMEM;
	821	}
[1]	822
	823	// update bitmap
	824	bitmap_set( &mgr->bitmap , index );
[21]	825
[1]	826	// release lock on stack allocator
	827	spinlock_unlock( &mgr->lock );
	828
[21]	829	// returns vpn_base, vpn_size (one page non allocated)
[1]	830	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
	831	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	832	return 0;
	833
[204]	834	} // end vmm_stack_alloc()
	835
[1]	836	////////////////////////////////////////////////////////////////////////////////////////////
	837	// This static function is called by the vmm_create_vseg() function, and implements
	838	// the VMM MMAP specific allocator.
	839	////////////////////////////////////////////////////////////////////////////////////////////
	840	// @ vmm : [in] pointer on VMM.
	841	// @ npages : [in] requested number of pages.
[21]	842	// @ vpn_base : [out] first allocated page.
[1]	843	// @ vpn_size : [out] actual number of allocated pages.
	844	////////////////////////////////////////////////////////////////////////////////////////////
	845	static error_t vmm_mmap_alloc( vmm_t * vmm,
	846	vpn_t npages,
	847	vpn_t * vpn_base,
	848	vpn_t * vpn_size )
	849	{
	850	uint32_t index;
	851	vseg_t * vseg;
	852	vpn_t base;
	853	vpn_t size;
[21]	854	vpn_t free;
[1]	855
[21]	856	// mmap vseg size must be power of 2
[1]	857	// compute actual size and index in zombi_list array
	858	size = POW2_ROUNDUP( npages );
	859	index = bits_log2( size );
	860
	861	// get mmap allocator pointer
	862	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	863
	864	// get lock on mmap allocator
	865	spinlock_lock( &mgr->lock );
	866
	867	// get vseg from zombi_list or from mmap zone
	868	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
	869	{
	870	// check overflow
	871	free = mgr->first_free_vpn;
	872	if( (free + size) > mgr->vpn_size ) return ENOMEM;
	873
	874	// update STACK allocator
	875	mgr->first_free_vpn += size;
	876
	877	// compute base
	878	base = free;
	879	}
	880	else // from zombi_list
	881	{
	882	// get pointer on zombi vseg from zombi_list
[408]	883	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
[1]	884
	885	// remove vseg from free-list
[408]	886	list_unlink( &vseg->zlist );
[1]	887
	888	// compute base
	889	base = vseg->vpn_base;
[21]	890	}
	891
[1]	892	// release lock on mmap allocator
	893	spinlock_unlock( &mgr->lock );
	894
	895	// returns vpn_base, vpn_size
	896	*vpn_base = base;
	897	*vpn_size = size;
	898	return 0;
	899
[204]	900	} // end vmm_mmap_alloc()
	901
[407]	902	////////////////////////////////////////////////
	903	vseg_t * vmm_create_vseg( process_t * process,
	904	vseg_type_t type,
	905	intptr_t base,
	906	uint32_t size,
	907	uint32_t file_offset,
	908	uint32_t file_size,
	909	xptr_t mapper_xp,
	910	cxy_t cxy )
[1]	911	{
	912	vseg_t * vseg; // created vseg pointer
[204]	913	vpn_t vpn_base; // first page index
[1]	914	vpn_t vpn_size; // number of pages
	915	error_t error;
	916
[438]	917	#if DEBUG_VMM_CREATE_VSEG
[433]	918	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	919	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	920	printk("\n[DBG] %s : thread %x enter / process %x / base %x / size %x / %s / cxy %x / cycle %d\n",
	921	__FUNCTION__, CURRENT_THREAD, process->pid, base, size, vseg_type_str(type), cxy, cycle );
	922	#endif
[21]	923
[407]	924	// get pointer on VMM
	925	vmm_t * vmm = &process->vmm;
[21]	926
[204]	927	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	928	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[1]	929	if( type == VSEG_TYPE_STACK )
	930	{
	931	// get vpn_base and vpn_size from STACK allocator
	932	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
	933	if( error )
	934	{
[407]	935	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
	936	__FUNCTION__ , process->pid , local_cxy );
[1]	937	return NULL;
	938	}
	939
	940	// compute vseg base and size from vpn_base and vpn_size
	941	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	942	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	943	}
[21]	944	else if( (type == VSEG_TYPE_ANON) \|\|
	945	(type == VSEG_TYPE_FILE) \|\|
[1]	946	(type == VSEG_TYPE_REMOTE) )
	947	{
	948	// get vpn_base and vpn_size from MMAP allocator
	949	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	950	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	951	if( error )
	952	{
	953	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	954	__FUNCTION__ , process->pid , local_cxy );
	955	return NULL;
	956	}
	957
	958	// compute vseg base and size from vpn_base and vpn_size
	959	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	960	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	961	}
	962	else
	963	{
[204]	964	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	965	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	966
	967	vpn_base = vpn_min;
	968	vpn_size = vpn_max - vpn_min + 1;
[1]	969	}
	970
	971	// check collisions
	972	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
	973	if( vseg != NULL )
	974	{
[21]	975	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
[1]	976	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
[407]	977	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	978	return NULL;
	979	}
	980
	981	// allocate physical memory for vseg descriptor
	982	vseg = vseg_alloc();
	983	if( vseg == NULL )
	984	{
	985	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	986	__FUNCTION__ , process->pid );
[1]	987	return NULL;
	988	}
	989
	990	// initialize vseg descriptor
[407]	991	vseg_init( vseg,
	992	type,
	993	base,
	994	size,
	995	vpn_base,
	996	vpn_size,
	997	file_offset,
	998	file_size,
	999	mapper_xp,
	1000	cxy );
[1]	1001
[408]	1002	// attach vseg to VSL
	1003	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1004	remote_rwlock_wr_lock( lock_xp );
[1]	1005	vseg_attach( vmm , vseg );
[408]	1006	remote_rwlock_wr_unlock( lock_xp );
[1]	1007
[438]	1008	#if DEBUG_VMM_CREATE_VSEG
[433]	1009	cycle = (uint32_t)hal_get_cycles();
[438]	1010	if( DEBUG_VMM_CREATE_VSEG < cycle )
[433]	1011	printk("\n[DBG] %s : thread %x exit / process %x / %s / cxy %x / cycle %d\n",
	1012	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str(type), cxy, cycle );
	1013	#endif
[21]	1014
[1]	1015	return vseg;
	1016
[406]	1017	} // vmm_create_vseg()
	1018
[1]	1019	/////////////////////////////////////
	1020	void vmm_remove_vseg( vseg_t * vseg )
	1021	{
	1022	// get pointers on calling process and VMM
	1023	thread_t * this = CURRENT_THREAD;
	1024	vmm_t * vmm = &this->process->vmm;
	1025	uint32_t type = vseg->type;
	1026
[408]	1027	// detach vseg from VSL
	1028	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1029	remote_rwlock_wr_lock( lock_xp );
[473]	1030	vseg_detach( vseg );
[408]	1031	remote_rwlock_wr_unlock( lock_xp );
[1]	1032
	1033	// release the stack slot to VMM stack allocator if STACK type
	1034	if( type == VSEG_TYPE_STACK )
	1035	{
	1036	// get pointer on stack allocator
	1037	stack_mgr_t * mgr = &vmm->stack_mgr;
	1038
	1039	// compute slot index
	1040	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
	1041
	1042	// update stacks_bitmap
	1043	spinlock_lock( &mgr->lock );
	1044	bitmap_clear( &mgr->bitmap , index );
	1045	spinlock_unlock( &mgr->lock );
	1046	}
	1047
	1048	// release the vseg to VMM mmap allocator if MMAP type
	1049	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
	1050	{
	1051	// get pointer on mmap allocator
	1052	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	1053
	1054	// compute zombi_list index
	1055	uint32_t index = bits_log2( vseg->vpn_size );
	1056
	1057	// update zombi_list
	1058	spinlock_lock( &mgr->lock );
[408]	1059	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
[1]	1060	spinlock_unlock( &mgr->lock );
	1061	}
	1062
	1063	// release physical memory allocated for vseg descriptor if no MMAP type
	1064	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
	1065	{
	1066	vseg_free( vseg );
	1067	}
[407]	1068	} // end vmm_remove_vseg()
[1]	1069
	1070	/////////////////////////////////////////
	1071	void vmm_unmap_vseg( process_t * process,
	1072	vseg_t * vseg )
	1073	{
[21]	1074	vpn_t vpn; // VPN of current PTE
	1075	vpn_t vpn_min; // VPN of first PTE
[1]	1076	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1077	ppn_t ppn; // current PTE ppn value
	1078	uint32_t attr; // current PTE attributes
	1079	kmem_req_t req; // request to release memory
	1080	xptr_t page_xp; // extended pointer on page descriptor
	1081	cxy_t page_cxy; // page descriptor cluster
	1082	page_t * page_ptr; // page descriptor pointer
[433]	1083	xptr_t forks_xp; // extended pointer on pending forks counter
[469]	1084	xptr_t lock_xp; // extended pointer on lock protecting forks counter
	1085	uint32_t forks; // actual number of pendinf forks
[1]	1086
[438]	1087	#if DEBUG_VMM_UNMAP_VSEG
[433]	1088	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1089	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1090	printk("\n[DBG] %s : thread %x enter / process %x / vseg %s / base %x / cycle %d\n",
	1091	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1092	#endif
[409]	1093
[433]	1094	// get pointer on local GPT
[1]	1095	gpt_t * gpt = &process->vmm.gpt;
	1096
	1097	// loop on pages in vseg
	1098	vpn_min = vseg->vpn_base;
	1099	vpn_max = vpn_min + vseg->vpn_size;
	1100	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1101	{
[409]	1102	// get GPT entry
	1103	hal_gpt_get_pte( gpt , vpn , &attr , &ppn );
	1104
	1105	if( attr & GPT_MAPPED ) // entry is mapped
	1106	{
[437]	1107
[438]	1108	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
	1109	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[437]	1110	printk("- vpn %x / ppn %x\n" , vpn , ppn );
	1111	#endif
	1112
[409]	1113	// check small page
	1114	assert( (attr & GPT_SMALL) , __FUNCTION__ ,
	1115	"an user vseg must use small pages" );
	1116
[433]	1117	// unmap GPT entry in all GPT copies
[409]	1118	hal_gpt_reset_pte( gpt , vpn );
	1119
[433]	1120	// handle pending forks counter if
	1121	// 1) not identity mapped
	1122	// 2) running in reference cluster
	1123	if( ((vseg->flags & VSEG_IDENT) == 0) &&
	1124	(GET_CXY( process->ref_xp ) == local_cxy) )
[409]	1125	{
[433]	1126	// get extended pointer on physical page descriptor
[409]	1127	page_xp = ppm_ppn2page( ppn );
	1128	page_cxy = GET_CXY( page_xp );
[433]	1129	page_ptr = GET_PTR( page_xp );
[409]	1130
[469]	1131	// get extended pointers on forks and lock fields
	1132	forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1133	lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1134
[469]	1135	// get lock protecting page descriptor
	1136	remote_spinlock_lock( lock_xp );
	1137
[433]	1138	// get pending forks counter
[469]	1139	forks = hal_remote_lw( forks_xp );
[433]	1140
[469]	1141	if( forks ) // decrement pending forks counter
[409]	1142	{
[433]	1143	hal_remote_atomic_add( forks_xp , -1 );
	1144	}
	1145	else // release physical page to relevant cluster
[409]	1146	{
[433]	1147	if( page_cxy == local_cxy ) // local cluster
	1148	{
	1149	req.type = KMEM_PAGE;
	1150	req.ptr = page_ptr;
	1151	kmem_free( &req );
	1152	}
	1153	else // remote cluster
	1154	{
	1155	rpc_pmem_release_pages_client( page_cxy , page_ptr );
	1156	}
[409]	1157	}
[433]	1158
[469]	1159	// release lock protecting page descriptor
	1160	remote_spinlock_unlock( lock_xp );
[409]	1161	}
	1162	}
[1]	1163	}
[433]	1164
[438]	1165	#if DEBUG_VMM_UNMAP_VSEG
[433]	1166	cycle = (uint32_t)hal_get_cycles();
[438]	1167	if( DEBUG_VMM_UNMAP_VSEG < cycle )
[433]	1168	printk("\n[DBG] %s : thread %x exit / process %x / vseg %s / base %x / cycle %d\n",
	1169	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
	1170	#endif
	1171
[409]	1172	} // end vmm_unmap_vseg()
[1]	1173
[407]	1174	//////////////////////////////////////////////////////////////////////////////////////////
[440]	1175	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
	1176	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
	1177	// containing a given virtual address.
[407]	1178	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1179	// @ vmm : pointer on the process VMM.
	1180	// @ vaddr : virtual address.
	1181	// @ return vseg pointer if success / return NULL if not found.
[407]	1182	//////////////////////////////////////////////////////////////////////////////////////////
[406]	1183	static vseg_t * vseg_from_vaddr( vmm_t * vmm,
	1184	intptr_t vaddr )
	1185	{
[408]	1186	xptr_t iter_xp;
	1187	xptr_t vseg_xp;
	1188	vseg_t * vseg;
[406]	1189
[408]	1190	// get extended pointers on VSL lock and root
	1191	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
	1192	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1193
[408]	1194	// get lock protecting the VSL
	1195	remote_rwlock_rd_lock( lock_xp );
	1196
	1197	// scan the list of vsegs in VSL
	1198	XLIST_FOREACH( root_xp , iter_xp )
[406]	1199	{
[408]	1200	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1201	vseg = GET_PTR( vseg_xp );
[408]	1202	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
	1203	{
	1204	// return success
	1205	remote_rwlock_rd_unlock( lock_xp );
	1206	return vseg;
	1207	}
[406]	1208	}
	1209
[408]	1210	// return failure
	1211	remote_rwlock_rd_unlock( lock_xp );
	1212	return NULL;
[406]	1213
[408]	1214	} // end vseg_from_vaddr()
[406]	1215
[1]	1216	/////////////////////////////////////////////
	1217	error_t vmm_resize_vseg( process_t * process,
	1218	intptr_t base,
	1219	intptr_t size )
	1220	{
[406]	1221	error_t error;
	1222	vseg_t * new;
	1223	vpn_t vpn_min;
	1224	vpn_t vpn_max;
[1]	1225
	1226	// get pointer on process VMM
	1227	vmm_t * vmm = &process->vmm;
	1228
	1229	intptr_t addr_min = base;
	1230	intptr_t addr_max = base + size;
	1231
	1232	// get pointer on vseg
[406]	1233	vseg_t * vseg = vseg_from_vaddr( vmm , base );
[1]	1234
	1235	if( vseg == NULL) return EINVAL;
[21]	1236
[408]	1237	// get extended pointer on VSL lock
	1238	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
[21]	1239
[408]	1240	// get lock protecting VSL
	1241	remote_rwlock_wr_lock( lock_xp );
	1242
[1]	1243	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
	1244	{
	1245	error = EINVAL;
	1246	}
	1247	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
	1248	{
	1249	vmm_remove_vseg( vseg );
	1250	error = 0;
	1251	}
[406]	1252	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1253	{
[406]	1254	// update vseg base address
	1255	vseg->min = addr_max;
	1256
	1257	// update vpn_base and vpn_size
	1258	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1259	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1260	vseg->vpn_base = vpn_min;
	1261	vseg->vpn_size = vpn_max - vpn_min + 1;
	1262	error = 0;
[1]	1263	}
[406]	1264	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1265	{
[406]	1266	// update vseg max address
	1267	vseg->max = addr_min;
	1268
	1269	// update vpn_base and vpn_size
	1270	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1271	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1272	vseg->vpn_base = vpn_min;
	1273	vseg->vpn_size = vpn_max - vpn_min + 1;
	1274	error = 0;
[1]	1275	}
[406]	1276	else // vseg cut in three regions
[1]	1277	{
[406]	1278	// resize existing vseg
	1279	vseg->max = addr_min;
	1280
	1281	// update vpn_base and vpn_size
	1282	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1283	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1284	vseg->vpn_base = vpn_min;
	1285	vseg->vpn_size = vpn_max - vpn_min + 1;
	1286
	1287	// create new vseg
[407]	1288	new = vmm_create_vseg( process,
	1289	vseg->type,
	1290	addr_min,
	1291	(vseg->max - addr_max),
	1292	vseg->file_offset,
	1293	vseg->file_size,
	1294	vseg->mapper_xp,
	1295	vseg->cxy );
	1296
[406]	1297	if( new == NULL ) error = EINVAL;
	1298	else error = 0;
[1]	1299	}
	1300
	1301	// release VMM lock
[408]	1302	remote_rwlock_wr_unlock( lock_xp );
[1]	1303
	1304	return error;
	1305
[406]	1306	} // vmm_resize_vseg()
	1307
[1]	1308	///////////////////////////////////////////
[388]	1309	error_t vmm_get_vseg( process_t * process,
[394]	1310	intptr_t vaddr,
[388]	1311	vseg_t ** found_vseg )
[1]	1312	{
[440]	1313	xptr_t vseg_xp;
	1314	error_t error;
	1315	vseg_t * vseg;
	1316	vmm_t * vmm;
[1]	1317
[440]	1318	// get pointer on local VMM
	1319	vmm = &process->vmm;
[1]	1320
[440]	1321	// try to get vseg from local VMM
	1322	vseg = vseg_from_vaddr( vmm , vaddr );
	1323
[388]	1324	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1325	{
	1326	// get extended pointer on reference process
	1327	xptr_t ref_xp = process->ref_xp;
[1]	1328
[388]	1329	// get cluster and local pointer on reference process
	1330	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1331	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1332
	1333	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1334
	1335	// get extended pointer on reference vseg
[394]	1336	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1337
[440]	1338	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1339
	1340	// allocate a vseg in local cluster
	1341	vseg = vseg_alloc();
	1342
[440]	1343	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1344
	1345	// initialise local vseg from reference
	1346	vseg_init_from_ref( vseg , vseg_xp );
	1347
	1348	// register local vseg in local VMM
[406]	1349	vseg_attach( &process->vmm , vseg );
[388]	1350	}
	1351
	1352	// success
	1353	*found_vseg = vseg;
[394]	1354	return 0;
[388]	1355
	1356	} // end vmm_get_vseg()
	1357
[407]	1358	//////////////////////////////////////////////////////////////////////////////////////
	1359	// This static function compute the target cluster to allocate a physical page
	1360	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
	1361	// and returns an extended pointer on the allocated page descriptor.
	1362	// The vseg cannot have the FILE type.
	1363	//////////////////////////////////////////////////////////////////////////////////////
	1364	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1365	vpn_t vpn )
	1366	{
[433]	1367
[438]	1368	#if DEBUG_VMM_ALLOCATE_PAGE
	1369	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1370	printk("\n[DBG] in %s : thread %x enter for vpn %x\n",
	1371	__FUNCTION__ , CURRENT_THREAD, vpn );
	1372	#endif
	1373
[407]	1374	// compute target cluster
	1375	page_t * page_ptr;
	1376	cxy_t page_cxy;
	1377	kmem_req_t req;
	1378
	1379	uint32_t type = vseg->type;
	1380	uint32_t flags = vseg->flags;
	1381
	1382	assert( ( type != VSEG_TYPE_FILE ) , __FUNCTION__ , "illegal vseg type\n" );
	1383
	1384	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1385	{
	1386	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1387	uint32_t y_size = LOCAL_CLUSTER->y_size;
	1388	uint32_t y_width = LOCAL_CLUSTER->y_width;
	1389	uint32_t index = vpn & ((x_size * y_size) - 1);
	1390	uint32_t x = index / y_size;
	1391	uint32_t y = index % y_size;
	1392	page_cxy = (x<<y_width) + y;
	1393	}
	1394	else // other cases => cxy specified in vseg
	1395	{
	1396	page_cxy = vseg->cxy;
	1397	}
	1398
	1399	// allocate a physical page from target cluster
	1400	if( page_cxy == local_cxy ) // target cluster is the local cluster
	1401	{
	1402	req.type = KMEM_PAGE;
	1403	req.size = 0;
	1404	req.flags = AF_NONE;
	1405	page_ptr = (page_t *)kmem_alloc( &req );
	1406	}
	1407	else // target cluster is not the local cluster
	1408	{
	1409	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
	1410	}
	1411
[438]	1412	#if DEBUG_VMM_ALLOCATE_PAGE
	1413	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
[433]	1414	printk("\n[DBG] in %s : thread %x exit for vpn = %d / ppn = %x\n",
	1415	__FUNCTION__ , CURRENT_THREAD, vpn, ppm_page2ppn( XPTR( page_cxy , page_ptr ) ) );
	1416	#endif
	1417
[407]	1418	if( page_ptr == NULL ) return XPTR_NULL;
	1419	else return XPTR( page_cxy , page_ptr );
	1420
	1421	} // end vmm_page_allocate()
	1422
[313]	1423	////////////////////////////////////////
	1424	error_t vmm_get_one_ppn( vseg_t * vseg,
	1425	vpn_t vpn,
	1426	ppn_t * ppn )
	1427	{
	1428	error_t error;
[407]	1429	xptr_t page_xp; // extended pointer on physical page descriptor
[313]	1430	page_t * page_ptr; // local pointer on physical page descriptor
[406]	1431	uint32_t index; // missing page index in vseg mapper
	1432	uint32_t type; // vseg type;
[313]	1433
[406]	1434	type = vseg->type;
	1435	index = vpn - vseg->vpn_base;
[313]	1436
[438]	1437	#if DEBUG_VMM_GET_ONE_PPN
[441]	1438	thread_t * this = CURRENT_THREAD;
[469]	1439	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1440	printk("\n[DBG] %s : thread %x enter for vpn = %x / type = %s / index = %d\n",
[441]	1441	__FUNCTION__, this, vpn, vseg_type_str(type), index );
[433]	1442	#endif
[313]	1443
[406]	1444	// FILE type : get the physical page from the file mapper
[313]	1445	if( type == VSEG_TYPE_FILE )
	1446	{
[406]	1447	// get extended pointer on mapper
[407]	1448	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1449
[406]	1450	assert( (mapper_xp != XPTR_NULL), __FUNCTION__,
	1451	"mapper not defined for a FILE vseg\n" );
	1452
	1453	// get mapper cluster and local pointer
	1454	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1455	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1456
[313]	1457	// get page descriptor from mapper
	1458	if( mapper_cxy == local_cxy ) // mapper is local
	1459	{
	1460	page_ptr = mapper_get_page( mapper_ptr , index );
	1461	}
	1462	else // mapper is remote
	1463	{
	1464	rpc_mapper_get_page_client( mapper_cxy , mapper_ptr , index , &page_ptr );
	1465	}
	1466
	1467	if ( page_ptr == NULL ) return EINVAL;
	1468
[407]	1469	page_xp = XPTR( mapper_cxy , page_ptr );
[313]	1470	}
	1471
[406]	1472	// Other types : allocate a physical page from target cluster,
[407]	1473	// as defined by vseg type and vpn value
[313]	1474	else
	1475	{
[433]	1476	// allocate one physical page
[407]	1477	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1478
[407]	1479	if( page_xp == XPTR_NULL ) return ENOMEM;
[313]	1480
[406]	1481	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1482	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1483	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1484	{
[406]	1485	// get extended pointer on mapper
	1486	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1487
[406]	1488	assert( (mapper_xp != XPTR_NULL), __FUNCTION__,
	1489	"mapper not defined for a CODE or DATA vseg\n" );
	1490
	1491	// get mapper cluster and local pointer
	1492	cxy_t mapper_cxy = GET_CXY( mapper_xp );
[433]	1493	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
[406]	1494
	1495	// compute missing page offset in vseg
	1496	uint32_t offset = index << CONFIG_PPM_PAGE_SHIFT;
	1497
[313]	1498	// compute missing page offset in .elf file
[406]	1499	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1500
[438]	1501	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1502	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1503	printk("\n[DBG] %s : thread %x for vpn = %x / elf_offset = %x\n",
[441]	1504	__FUNCTION__, this, vpn, elf_offset );
[433]	1505	#endif
[313]	1506
[440]	1507
[406]	1508	// compute extended pointer on page base
[407]	1509	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1510
[406]	1511	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1512	uint32_t file_size = vseg->file_size;
	1513
	1514	if( file_size < offset ) // missing page fully in BSS
[313]	1515	{
[406]	1516
[438]	1517	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1518	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1519	printk("\n[DBG] %s : thread%x for vpn = %x / fully in BSS\n",
[441]	1520	__FUNCTION__, this, vpn );
[433]	1521	#endif
	1522
[440]	1523
[407]	1524	if( GET_CXY( page_xp ) == local_cxy )
[313]	1525	{
[315]	1526	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1527	}
	1528	else
	1529	{
[315]	1530	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1531	}
	1532	}
[406]	1533	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1534	{
[406]	1535
[438]	1536	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1537	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1538	printk("\n[DBG] %s : thread %x, for vpn = %x / fully in mapper\n",
[441]	1539	__FUNCTION__, this, vpn );
[433]	1540	#endif
[313]	1541	if( mapper_cxy == local_cxy )
	1542	{
	1543	error = mapper_move_kernel( mapper_ptr,
	1544	true, // to_buffer
[406]	1545	elf_offset,
[313]	1546	base_xp,
[315]	1547	CONFIG_PPM_PAGE_SIZE );
[313]	1548	}
	1549	else
	1550	{
	1551	rpc_mapper_move_buffer_client( mapper_cxy,
	1552	mapper_ptr,
	1553	true, // to buffer
	1554	false, // kernel buffer
[406]	1555	elf_offset,
	1556	base_xp,
[315]	1557	CONFIG_PPM_PAGE_SIZE,
[313]	1558	&error );
	1559	}
	1560	if( error ) return EINVAL;
	1561	}
[406]	1562	else // both in mapper and in BSS :
	1563	// - (file_size - offset) bytes from mapper
	1564	// - (page_size + offset - file_size) bytes from BSS
[313]	1565	{
[406]	1566
[438]	1567	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[469]	1568	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1569	printk("\n[DBG] %s : thread %x for vpn = %x / both mapper & BSS\n"
	1570	" %d bytes from mapper / %d bytes from BSS\n",
[441]	1571	__FUNCTION__, this, vpn,
[407]	1572	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1573	#endif
[313]	1574	// initialize mapper part
[315]	1575	if( mapper_cxy == local_cxy )
[313]	1576	{
	1577	error = mapper_move_kernel( mapper_ptr,
[315]	1578	true, // to buffer
[406]	1579	elf_offset,
[313]	1580	base_xp,
[406]	1581	file_size - offset );
[313]	1582	}
[315]	1583	else
[313]	1584	{
	1585	rpc_mapper_move_buffer_client( mapper_cxy,
	1586	mapper_ptr,
[315]	1587	true, // to buffer
[313]	1588	false, // kernel buffer
[406]	1589	elf_offset,
	1590	base_xp,
	1591	file_size - offset,
[313]	1592	&error );
	1593	}
	1594	if( error ) return EINVAL;
	1595
	1596	// initialize BSS part
[407]	1597	if( GET_CXY( page_xp ) == local_cxy )
[313]	1598	{
[406]	1599	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1600	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1601	}
	1602	else
	1603	{
[406]	1604	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1605	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1606	}
	1607	}
	1608	} // end initialisation for CODE or DATA types
	1609	}
	1610
	1611	// return ppn
[407]	1612	*ppn = ppm_page2ppn( page_xp );
[406]	1613
[438]	1614	#if DEBUG_VMM_GET_ONE_PPN
[469]	1615	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
[433]	1616	printk("\n[DBG] %s : thread %x exit for vpn = %x / ppn = %x\n",
[441]	1617	__FUNCTION__ , this , vpn , *ppn );
[433]	1618	#endif
[406]	1619
[313]	1620	return 0;
	1621
	1622	} // end vmm_get_one_ppn()
	1623
[1]	1624	/////////////////////////////////////////
	1625	error_t vmm_get_pte( process_t * process,
	1626	vpn_t vpn,
[407]	1627	bool_t cow,
	1628	uint32_t * attr,
	1629	ppn_t * ppn )
[1]	1630	{
[440]	1631	ppn_t old_ppn; // current PTE_PPN
	1632	uint32_t old_attr; // current PTE_ATTR
	1633	ppn_t new_ppn; // new PTE_PPN
	1634	uint32_t new_attr; // new PTE_ATTR
	1635	vmm_t * vmm;
	1636	vseg_t * vseg;
	1637	error_t error;
[1]	1638
[441]	1639	thread_t * this = CURRENT_THREAD;
	1640
[438]	1641	#if DEBUG_VMM_GET_PTE
[441]	1642	uint32_t cycle = (uint32_t)hal_get_cycles();
[469]	1643	if( DEBUG_VMM_GET_PTE < cycle )
	1644	printk("\n[DBG] %s : thread %x in process %x enter / vpn %x / cow %d / cycle %d\n",
	1645	__FUNCTION__, this->trdid, process->pid, vpn, cow, cycle );
[433]	1646	#endif
[406]	1647
[1]	1648	// get VMM pointer
[440]	1649	vmm = &process->vmm;
[1]	1650
[440]	1651	// get local vseg descriptor
	1652	error = vmm_get_vseg( process,
	1653	((intptr_t)vpn << CONFIG_PPM_PAGE_SHIFT),
	1654	&vseg );
[1]	1655
[440]	1656	// vseg has been checked by the vmm_handle_page_fault() function
	1657	assert( (vseg != NULL) , __FUNCTION__,
[469]	1658	"vseg undefined / vpn %x / thread %x in process %x / core[%x,%d] / cycle %d\n",
	1659	vpn, this->trdid, process->pid, local_cxy, this->core->lid,
[440]	1660	(uint32_t)hal_get_cycles() );
[406]	1661
[438]	1662	if( cow ) //////////////// copy_on_write request //////////////////////
[440]	1663	// get PTE from local GPT
[438]	1664	// allocate a new physical page if there is pending forks,
	1665	// initialize it from old physical page content,
	1666	// update PTE in all GPT copies,
	1667	{
[440]	1668	// access local GPT to get current PTE attributes and PPN
[438]	1669	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
[407]	1670
[440]	1671	assert( (old_attr & GPT_MAPPED), __FUNCTION__,
[469]	1672	"PTE unmapped for a COW exception / vpn %x / thread %x in process %x / cycle %d\n",
[441]	1673	vpn, this, process->pid, (uint32_t)hal_get_cycles() );
[407]	1674
[438]	1675	#if( DEBUG_VMM_GET_PTE & 1 )
[469]	1676	if( DEBUG_VMM_GET_PTE < cycle )
	1677	printk("\n[DBG] %s : thread %x in process %x handling COW for vpn %x\n",
	1678	__FUNCTION__, this->trdid, process->pid, vpn );
[433]	1679	#endif
[407]	1680
[433]	1681	// get extended pointer, cluster and local pointer on physical page descriptor
[408]	1682	xptr_t page_xp = ppm_ppn2page( old_ppn );
	1683	cxy_t page_cxy = GET_CXY( page_xp );
[433]	1684	page_t * page_ptr = GET_PTR( page_xp );
[407]	1685
[469]	1686	// get extended pointers on forks and lock field in page descriptor
	1687	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1688	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
	1689
	1690	// take lock protecting page descriptor
	1691	remote_spinlock_lock( lock_xp );
	1692
[408]	1693	// get number of pending forks in page descriptor
[469]	1694	uint32_t forks = hal_remote_lw( forks_xp );
[408]	1695
[433]	1696	if( forks ) // pending fork => allocate a new page, copy old to new
[1]	1697	{
[408]	1698	// allocate a new physical page
	1699	page_xp = vmm_page_allocate( vseg , vpn );
	1700	if( page_xp == XPTR_NULL )
	1701	{
	1702	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1703	__FUNCTION__ , process->pid , vpn );
	1704	return -1;
	1705	}
[1]	1706
[408]	1707	// compute allocated page PPN
	1708	new_ppn = ppm_page2ppn( page_xp );
[406]	1709
[408]	1710	// copy old page content to new page
	1711	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
	1712	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
	1713	memcpy( GET_PTR( new_base_xp ),
	1714	GET_PTR( old_base_xp ),
	1715	CONFIG_PPM_PAGE_SIZE );
[469]	1716
	1717	// decrement pending forks counter in page descriptor
	1718	hal_remote_atomic_add( forks_xp , -1 );
[408]	1719	}
[469]	1720	else // no pending fork => keep the existing page
[408]	1721	{
	1722	new_ppn = old_ppn;
	1723	}
[1]	1724
[469]	1725	// release lock protecting page descriptor
	1726	remote_spinlock_unlock( lock_xp );
	1727
[408]	1728	// build new_attr : reset COW and set WRITABLE,
	1729	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
[407]	1730
[408]	1731	// update GPT[vpn] for all GPT copies
[433]	1732	vmm_global_update_pte( process, vpn, new_attr, new_ppn );
[407]	1733	}
[438]	1734	else //////////// page_fault request ///////////////////////////
[440]	1735	// get PTE from local GPT
[438]	1736	// allocate a physical page if it is a true page fault,
[440]	1737	// initialize it if type is FILE, CODE, or DATA,
[438]	1738	// register in reference GPT, but don't update GPT copies
[407]	1739	{
[440]	1740	// access local GPT to get current PTE
[438]	1741	hal_gpt_get_pte( &vmm->gpt , vpn , &old_attr , &old_ppn );
	1742
[408]	1743	if( (old_attr & GPT_MAPPED) == 0 ) // true page_fault => map it
[407]	1744	{
[1]	1745
[438]	1746	#if( DEBUG_VMM_GET_PTE & 1 )
[469]	1747	if( DEBUG_VMM_GET_PTE < cycle )
	1748	printk("\n[DBG] %s : thread %x in process %x handling page fault for vpn %x\n",
	1749	__FUNCTION__, this->trdid, process->pid, vpn );
[433]	1750	#endif
[440]	1751	// allocate new_ppn, and initialize the new page
[407]	1752	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
	1753	if( error )
	1754	{
	1755	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	1756	__FUNCTION__ , process->pid , vpn );
[408]	1757	return -1;
[407]	1758	}
	1759
[408]	1760	// define new_attr from vseg flags
[407]	1761	new_attr = GPT_MAPPED \| GPT_SMALL;
	1762	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
	1763	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
	1764	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
	1765	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
	1766
[408]	1767	// register new PTE in reference GPT
	1768	// on demand policy => no update of GPT copies
	1769	error = hal_gpt_set_pte( &vmm->gpt,
	1770	vpn,
	1771	new_attr,
	1772	new_ppn );
[407]	1773	if( error )
	1774	{
	1775	printk("\n[ERROR] in %s : cannot update GPT / process = %x / vpn = %x\n",
	1776	__FUNCTION__ , process->pid , vpn );
[408]	1777	return -1;
[407]	1778	}
	1779	}
[408]	1780	else // mapped in reference GPT => get it
[1]	1781	{
[408]	1782	new_ppn = old_ppn;
[407]	1783	new_attr = old_attr;
[1]	1784	}
[407]	1785	}
[1]	1786
[438]	1787	#if DEBUG_VMM_GET_PTE
[433]	1788	cycle = (uint32_t)hal_get_cycles();
[469]	1789	if( DEBUG_VMM_GET_PTE < cycle )
	1790	printk("\n[DBG] %s : thread %x in process %x exit / vpn %x / ppn %x / attr %x / cycle %d\n",
	1791	__FUNCTION__, this->trdid, process->pid, vpn, new_ppn, new_attr, cycle );
[433]	1792	#endif
[406]	1793
[438]	1794	// return PPN and flags
[407]	1795	*ppn = new_ppn;
	1796	*attr = new_attr;
[1]	1797	return 0;
	1798
[313]	1799	} // end vmm_get_pte()
	1800
[1]	1801	///////////////////////////////////////////////////
	1802	error_t vmm_handle_page_fault( process_t * process,
[440]	1803	vpn_t vpn,
	1804	bool_t is_cow )
[1]	1805	{
	1806	uint32_t attr; // missing page attributes
[21]	1807	ppn_t ppn; // missing page PPN
[440]	1808	vseg_t * vseg; // vseg containing vpn
	1809	uint32_t type; // vseg type
	1810	cxy_t ref_cxy; // reference cluster for missing vpn
	1811	process_t * ref_ptr; // reference process for missing vpn
[407]	1812	error_t error;
[1]	1813
[440]	1814	thread_t * this = CURRENT_THREAD;
	1815
	1816	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[435]	1817	uint32_t cycle = (uint32_t)hal_get_cycles();
[469]	1818	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1819	printk("\n[DBG] %s : thread %x in process %x enter for vpn %x / core[%x,%d] / cycle %d\n",
	1820	__FUNCTION__, this, process->pid, vpn, local_cxy, this->core->lid, cycle );
[435]	1821	#endif
	1822
[440]	1823	// get local vseg (access reference VSL if required)
	1824	error = vmm_get_vseg( process , vpn<<CONFIG_PPM_PAGE_SHIFT , &vseg );
[1]	1825
[440]	1826	if( error )
[1]	1827	{
[440]	1828	printk("\n[ERROR] in %s : vpn %x / process %x / thread %x / core[%x,%d] / cycle %d\n",
	1829	__FUNCTION__, vpn, process->pid, this->trdid, local_cxy, this->core->lid,
	1830	(uint32_t)hal_get_cycles() );
	1831	return error;
	1832	}
[407]	1833
[440]	1834	// get segment type
	1835	type = vseg->type;
[407]	1836
[440]	1837	// get reference process cluster and local pointer
	1838	// for private vsegs (CODE and DATA type),
	1839	// the reference is the local process descriptor.
	1840	if( (type == VSEG_TYPE_STACK) \|\| (type == VSEG_TYPE_CODE) )
	1841	{
	1842	ref_cxy = local_cxy;
	1843	ref_ptr = process;
[1]	1844	}
[440]	1845	else
[1]	1846	{
[440]	1847	ref_cxy = GET_CXY( process->ref_xp );
	1848	ref_ptr = GET_PTR( process->ref_xp );
[1]	1849	}
	1850
[440]	1851	// get missing PTE attributes and PPN
	1852	if( local_cxy != ref_cxy )
[407]	1853	{
[441]	1854
	1855	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[469]	1856	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1857	printk("\n[DBG] %s : thread %x in process %x call RPC_VMM_GET_PTE\n",
	1858	__FUNCTION__, this, process->pid );
	1859	#endif
	1860
[407]	1861	rpc_vmm_get_pte_client( ref_cxy,
	1862	ref_ptr,
	1863	vpn,
[440]	1864	is_cow,
[407]	1865	&attr,
	1866	&ppn,
	1867	&error );
	1868
	1869	// get local VMM pointer
	1870	vmm_t * vmm = &process->vmm;
	1871
	1872	// update local GPT
[408]	1873	error \|= hal_gpt_set_pte( &vmm->gpt,
	1874	vpn,
	1875	attr,
	1876	ppn );
[407]	1877	}
	1878	else // local cluster is the reference cluster
	1879	{
	1880	error = vmm_get_pte( process,
	1881	vpn,
[440]	1882	is_cow,
[407]	1883	&attr,
	1884	&ppn );
	1885	}
	1886
[440]	1887	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[435]	1888	cycle = (uint32_t)hal_get_cycles();
[469]	1889	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
[441]	1890	printk("\n[DBG] %s : thread %x in process %x exit for vpn %x / core[%x,%d] / cycle %d\n",
	1891	__FUNCTION__, this, process->pid, vpn, local_cxy, this->core->lid, cycle );
[435]	1892	#endif
	1893
[407]	1894	return error;
	1895
[441]	1896	} // end vmm_handle_page_fault()
[407]	1897
[441]	1898
	1899
	1900
	1901
	1902
	1903
	1904
	1905
[440]	1906	/* deprecated April 2018 [AG]
	1907
	1908	error_t vmm_v2p_translate( process_t * process,
	1909	void * ptr,
	1910	paddr_t * paddr )
[1]	1911	{
[21]	1912	// access page table
[1]	1913	error_t error;
	1914	vpn_t vpn;
	1915	uint32_t attr;
	1916	ppn_t ppn;
	1917	uint32_t offset;
	1918
[23]	1919	vpn = (vpn_t)( (intptr_t)ptr >> CONFIG_PPM_PAGE_SHIFT );
	1920	offset = (uint32_t)( ((intptr_t)ptr) & CONFIG_PPM_PAGE_MASK );
[1]	1921
[440]	1922	if( local_cxy == GET_CXY( process->ref_xp) ) // local process is reference process
[1]	1923	{
[407]	1924	error = vmm_get_pte( process, vpn , false , &attr , &ppn );
[1]	1925	}
[50]	1926	else // calling process is not reference process
[1]	1927	{
	1928	cxy_t ref_cxy = GET_CXY( process->ref_xp );
[433]	1929	process_t * ref_ptr = GET_PTR( process->ref_xp );
[407]	1930	rpc_vmm_get_pte_client( ref_cxy , ref_ptr , vpn , false , &attr , &ppn , &error );
[1]	1931	}
	1932
[23]	1933	// set paddr
[1]	1934	*paddr = (((paddr_t)ppn) << CONFIG_PPM_PAGE_SHIFT) \| offset;
[21]	1935
[23]	1936	return error;
[313]	1937
[315]	1938	} // end vmm_v2p_translate()
[1]	1939
[440]	1940	*/

Note: See TracBrowser for help on using the repository browser.

Download in other formats: