Context Navigation

source: trunk/kernel/mm/vmm.c @ 638

Last change on this file since 638 was 635, checked in by alain, 5 years ago

This version is a major evolution: The physical memory allocators,
defined in the kmem.c, ppm.c, and kcm.c files have been modified
to support remote accesses. The RPCs that were previously user
to allocate physical memory in a remote cluster have been removed.
This has been done to cure a dead-lock in case of concurrent page-faults.

This version 2.2 has been tested on a (4 clusters / 2 cores per cluster)
TSAR architecture, for both the "sort" and the "fft" applications.

File size: 81.9 KB

Rev	Line
[1]	1	/*
[611]	2	* vmm.c - virtual memory manager related operations definition.
[1]	3	*
	4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
	5	* Mohamed Lamine Karaoui (2015)
[625]	6	* Alain Greiner (2016,2017,2018,2019)
[21]	7	*
[1]	8	* Copyright (c) UPMC Sorbonne Universites
	9	*
	10	* This file is part of ALMOS-MKH.
	11	*
	12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
	13	* under the terms of the GNU General Public License as published by
	14	* the Free Software Foundation; version 2.0 of the License.
	15	*
	16	* ALMOS-MKH is distributed in the hope that it will be useful, but
	17	* WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License
	22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
	23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
[14]	26	#include <kernel_config.h>
[457]	27	#include <hal_kernel_types.h>
[1]	28	#include <hal_special.h>
	29	#include <hal_gpt.h>
[409]	30	#include <hal_vmm.h>
[577]	31	#include <hal_macros.h>
[1]	32	#include <printk.h>
[23]	33	#include <memcpy.h>
[567]	34	#include <remote_rwlock.h>
	35	#include <remote_queuelock.h>
[1]	36	#include <list.h>
[408]	37	#include <xlist.h>
[1]	38	#include <bits.h>
	39	#include <process.h>
	40	#include <thread.h>
	41	#include <vseg.h>
	42	#include <cluster.h>
	43	#include <scheduler.h>
	44	#include <vfs.h>
	45	#include <mapper.h>
	46	#include <page.h>
	47	#include <kmem.h>
	48	#include <vmm.h>
[585]	49	#include <hal_exception.h>
[1]	50
[635]	51	////////////////////////////////////////////////////////////////////////////////////////////
[1]	52	// Extern global variables
[635]	53	////////////////////////////////////////////////////////////////////////////////////////////
[1]	54
[567]	55	extern process_t process_zero; // allocated in cluster.c
[1]	56
[625]	57	////////////////////////////////////////////////////////////////////////////////////////////
	58	// This static function is called by the vmm_create_vseg() function, and implements
	59	// the VMM STACK specific allocator.
	60	////////////////////////////////////////////////////////////////////////////////////////////
	61	// @ vmm : [in] pointer on VMM.
	62	// @ ltid : [in] requested slot == local user thread identifier.
	63	// @ vpn_base : [out] first allocated page
	64	// @ vpn_size : [out] number of allocated pages
	65	////////////////////////////////////////////////////////////////////////////////////////////
	66	static void vmm_stack_alloc( vmm_t * vmm,
	67	ltid_t ltid,
	68	vpn_t * vpn_base,
	69	vpn_t * vpn_size )
[21]	70	{
[625]	71
	72	// check ltid argument
	73	assert( (ltid <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
	74	"slot index %d too large for an user stack vseg", ltid );
	75
	76	// get stack allocator pointer
	77	stack_mgr_t * mgr = &vmm->stack_mgr;
	78
	79	// get lock on stack allocator
	80	busylock_acquire( &mgr->lock );
	81
	82	// check requested slot is available
	83	assert( (bitmap_state( &mgr->bitmap , ltid ) == false),
	84	"slot index %d already allocated", ltid );
	85
	86	// update bitmap
	87	bitmap_set( &mgr->bitmap , ltid );
	88
	89	// release lock on stack allocator
	90	busylock_release( &mgr->lock );
	91
	92	// returns vpn_base, vpn_size (first page non allocated)
	93	vpn_base = mgr->vpn_base + ltid CONFIG_VMM_STACK_SIZE + 1;
	94	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
	95
	96	} // end vmm_stack_alloc()
	97
	98	////////////////////////////////////////////////////////////////////////////////////////////
	99	// This static function is called by the vmm_remove_vseg() function, and implements
	100	// the VMM STACK specific desallocator.
	101	////////////////////////////////////////////////////////////////////////////////////////////
	102	// @ vmm : [in] pointer on VMM.
	103	// @ vseg : [in] pointer on released vseg.
	104	////////////////////////////////////////////////////////////////////////////////////////////
	105	static void vmm_stack_free( vmm_t * vmm,
	106	vseg_t * vseg )
	107	{
	108	// get stack allocator pointer
	109	stack_mgr_t * mgr = &vmm->stack_mgr;
	110
	111	// compute slot index
	112	uint32_t index = (vseg->vpn_base - 1 - mgr->vpn_base) / CONFIG_VMM_STACK_SIZE;
	113
	114	// check index
	115	assert( (index <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
	116	"slot index %d too large for an user stack vseg", index );
	117
	118	// check released slot is allocated
	119	assert( (bitmap_state( &mgr->bitmap , index ) == true),
	120	"released slot index %d non allocated", index );
	121
	122	// get lock on stack allocator
	123	busylock_acquire( &mgr->lock );
	124
	125	// update stacks_bitmap
	126	bitmap_clear( &mgr->bitmap , index );
	127
	128	// release lock on stack allocator
	129	busylock_release( &mgr->lock );
	130
	131	} // end vmm_stack_free()
	132
	133	////////////////////////////////////////////////////////////////////////////////////////////
	134	// This static function is called by the vmm_create_vseg() function, and implements
	135	// the VMM MMAP specific allocator.
	136	////////////////////////////////////////////////////////////////////////////////////////////
	137	// @ vmm : [in] pointer on VMM.
	138	// @ npages : [in] requested number of pages.
	139	// @ vpn_base : [out] first allocated page.
	140	// @ vpn_size : [out] actual number of allocated pages.
	141	////////////////////////////////////////////////////////////////////////////////////////////
	142	static error_t vmm_mmap_alloc( vmm_t * vmm,
	143	vpn_t npages,
	144	vpn_t * vpn_base,
	145	vpn_t * vpn_size )
	146	{
	147	uint32_t order;
	148	xptr_t vseg_xp;
	149	vseg_t * vseg;
	150	vpn_t base;
	151	vpn_t size;
	152	vpn_t free;
	153
	154	#if DEBUG_VMM_MMAP_ALLOC
	155	thread_t * this = CURRENT_THREAD;
	156	uint32_t cycle = (uint32_t)hal_get_cycles();
	157	if( DEBUG_VMM_MMAP_ALLOC < cycle )
	158	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
	159	__FUNCTION__, this->process->pid, this->trdid, cycle );
	160	#endif
	161
	162	// number of allocated pages must be power of 2
	163	// compute actual size and order
	164	size = POW2_ROUNDUP( npages );
	165	order = bits_log2( size );
	166
	167	// get mmap allocator pointer
	168	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	169
	170	// build extended pointer on root of zombi_list[order]
	171	xptr_t root_xp = XPTR( local_cxy , &mgr->zombi_list[order] );
	172
	173	// take lock protecting zombi_lists
	174	busylock_acquire( &mgr->lock );
	175
	176	// get vseg from zombi_list or from mmap zone
	177	if( xlist_is_empty( root_xp ) ) // from mmap zone
	178	{
	179	// check overflow
	180	free = mgr->first_free_vpn;
	181	if( (free + size) > mgr->vpn_size ) return -1;
	182
	183	// update MMAP allocator
	184	mgr->first_free_vpn += size;
	185
	186	// compute base
	187	base = free;
	188	}
	189	else // from zombi_list
	190	{
	191	// get pointer on zombi vseg from zombi_list
	192	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
	193	vseg = GET_PTR( vseg_xp );
	194
	195	// remove vseg from free-list
	196	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	197
	198	// compute base
	199	base = vseg->vpn_base;
	200	}
	201
	202	// release lock
	203	busylock_release( &mgr->lock );
	204
	205	#if DEBUG_VMM_MMAP_ALLOC
	206	cycle = (uint32_t)hal_get_cycles();
	207	if( DEBUG_VMM_DESTROY < cycle )
	208	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
	209	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
	210	#endif
	211
	212	// returns vpn_base, vpn_size
	213	*vpn_base = base;
	214	*vpn_size = size;
	215	return 0;
	216
	217	} // end vmm_mmap_alloc()
	218
	219	////////////////////////////////////////////////////////////////////////////////////////////
	220	// This static function is called by the vmm_remove_vseg() function, and implements
	221	// the VMM MMAP specific desallocator.
	222	////////////////////////////////////////////////////////////////////////////////////////////
	223	// @ vmm : [in] pointer on VMM.
	224	// @ vseg : [in] pointer on released vseg.
	225	////////////////////////////////////////////////////////////////////////////////////////////
	226	static void vmm_mmap_free( vmm_t * vmm,
	227	vseg_t * vseg )
	228	{
	229	// get pointer on mmap allocator
	230	mmap_mgr_t * mgr = &vmm->mmap_mgr;
	231
	232	// compute zombi_list order
	233	uint32_t order = bits_log2( vseg->vpn_size );
	234
	235	// take lock protecting zombi lists
	236	busylock_acquire( &mgr->lock );
	237
	238	// update relevant zombi_list
	239	xlist_add_first( XPTR( local_cxy , &mgr->zombi_list[order] ),
	240	XPTR( local_cxy , &vseg->xlist ) );
	241
	242	// release lock
	243	busylock_release( &mgr->lock );
	244
	245	} // end of vmm_mmap_free()
	246
	247	////////////////////////////////////////////////////////////////////////////////////////////
	248	// This static function registers one vseg in the VSL of a local process descriptor.
	249	////////////////////////////////////////////////////////////////////////////////////////////
	250	// vmm : [in] pointer on VMM.
	251	// vseg : [in] pointer on vseg.
	252	////////////////////////////////////////////////////////////////////////////////////////////
	253	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
	254	vseg_t * vseg )
	255	{
	256	// update vseg descriptor
	257	vseg->vmm = vmm;
	258
	259	// increment vsegs number
	260	vmm->vsegs_nr++;
	261
	262	// add vseg in vmm list
	263	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
	264	XPTR( local_cxy , &vseg->xlist ) );
	265
	266	} // end vmm_attach_vseg_from_vsl()
	267
	268	////////////////////////////////////////////////////////////////////////////////////////////
	269	// This static function removes one vseg from the VSL of a local process descriptor.
	270	////////////////////////////////////////////////////////////////////////////////////////////
	271	// vmm : [in] pointer on VMM.
	272	// vseg : [in] pointer on vseg.
	273	////////////////////////////////////////////////////////////////////////////////////////////
	274	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
	275	vseg_t * vseg )
	276	{
	277	// update vseg descriptor
	278	vseg->vmm = NULL;
	279
	280	// decrement vsegs number
	281	vmm->vsegs_nr--;
	282
	283	// remove vseg from VSL
	284	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	285
	286	} // end vmm_detach_from_vsl()
	287
	288	////////////////////////////////////////////
	289	error_t vmm_user_init( process_t * process )
	290	{
[614]	291	uint32_t i;
[1]	292
[625]	293	#if DEBUG_VMM_USER_INIT
[567]	294	thread_t * this = CURRENT_THREAD;
[433]	295	uint32_t cycle = (uint32_t)hal_get_cycles();
[625]	296	if( DEBUG_VMM_USER_INIT )
[614]	297	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	298	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	299	#endif
[204]	300
[1]	301	// get pointer on VMM
	302	vmm_t * vmm = &process->vmm;
	303
[625]	304	// check UTILS zone
[624]	305	assert( ((CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE) <=
	306	(CONFIG_VMM_ELF_BASE - CONFIG_VMM_UTILS_BASE)) ,
	307	"UTILS zone too small\n" );
[21]	308
[625]	309	// check STACK zone
[567]	310	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
	311	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
	312	"STACK zone too small\n");
[1]	313
[635]	314	// initialize the lock protecting the VSL
	315	remote_rwlock_init( XPTR( local_cxy , &vmm->vsl_lock ) , LOCK_VMM_VSL );
	316
	317
	318	/*
[625]	319	// register "args" vseg in VSL
[624]	320	base = CONFIG_VMM_UTILS_BASE << CONFIG_PPM_PAGE_SHIFT;
[1]	321	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	322
[407]	323	vseg_args = vmm_create_vseg( process,
	324	VSEG_TYPE_DATA,
	325	base,
	326	size,
	327	0, // file_offset unused
	328	0, // file_size unused
	329	XPTR_NULL, // mapper_xp unused
	330	local_cxy );
[415]	331	if( vseg_args == NULL )
	332	{
	333	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
	334	return -1;
	335	}
[204]	336
[406]	337	vmm->args_vpn_base = base;
[1]	338
[625]	339	// register "envs" vseg in VSL
[624]	340	base = (CONFIG_VMM_UTILS_BASE + CONFIG_VMM_ARGS_SIZE) << CONFIG_PPM_PAGE_SHIFT;
[1]	341	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
[406]	342
[407]	343	vseg_envs = vmm_create_vseg( process,
	344	VSEG_TYPE_DATA,
	345	base,
	346	size,
	347	0, // file_offset unused
	348	0, // file_size unused
	349	XPTR_NULL, // mapper_xp unused
	350	local_cxy );
[415]	351	if( vseg_envs == NULL )
	352	{
	353	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
	354	return -1;
	355	}
[204]	356
[406]	357	vmm->envs_vpn_base = base;
[635]	358	*/
[1]	359	// initialize STACK allocator
	360	vmm->stack_mgr.bitmap = 0;
	361	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
[567]	362	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
[1]	363
	364	// initialize MMAP allocator
[407]	365	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	366	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	367	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[567]	368	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
[625]	369	for( i = 0 ; i < 32 ; i++ )
	370	{
	371	xlist_root_init( XPTR( local_cxy , &vmm->mmap_mgr.zombi_list[i] ) );
	372	}
[1]	373
[21]	374	// initialize instrumentation counters
[635]	375	vmm->false_pgfault_nr = 0;
	376	vmm->local_pgfault_nr = 0;
	377	vmm->global_pgfault_nr = 0;
	378	vmm->false_pgfault_cost = 0;
	379	vmm->local_pgfault_cost = 0;
	380	vmm->global_pgfault_cost = 0;
[1]	381
[124]	382	hal_fence();
[1]	383
[625]	384	#if DEBUG_VMM_USER_INIT
[433]	385	cycle = (uint32_t)hal_get_cycles();
[625]	386	if( DEBUG_VMM_USER_INIT )
[614]	387	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	388	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	389	#endif
[204]	390
[415]	391	return 0;
	392
[625]	393	} // end vmm_user_init()
[204]	394
[611]	395	//////////////////////////////////////////
[625]	396	void vmm_user_reset( process_t * process )
[567]	397	{
[625]	398	xptr_t vseg_xp;
	399	vseg_t * vseg;
	400	vseg_type_t vseg_type;
[567]	401
[625]	402	#if DEBUG_VMM_USER_RESET
[635]	403	uint32_t cycle;
[625]	404	thread_t * this = CURRENT_THREAD;
[635]	405	#endif
	406
	407	#if (DEBUG_VMM_USER_RESET & 1 )
	408	cycle = (uint32_t)hal_get_cycles();
[625]	409	if( DEBUG_VMM_USER_RESET < cycle )
	410	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	411	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
	412	#endif
[567]	413
[625]	414	#if (DEBUG_VMM_USER_RESET & 1 )
	415	if( DEBUG_VMM_USER_RESET < cycle )
[635]	416	hal_vmm_display( XPTR( local_cxy , process ) , true );
[625]	417	#endif
[567]	418
[625]	419	// get pointer on local VMM
	420	vmm_t * vmm = &process->vmm;
[624]	421
[625]	422	// build extended pointer on VSL root and VSL lock
	423	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	424	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
[567]	425
[625]	426	// take the VSL lock
	427	remote_rwlock_wr_acquire( lock_xp );
[567]	428
[625]	429	// scan the VSL to delete all non kernel vsegs
	430	// (we don't use a FOREACH in case of item deletion)
	431	xptr_t iter_xp;
	432	xptr_t next_xp;
	433	for( iter_xp = hal_remote_l64( root_xp ) ;
	434	iter_xp != root_xp ;
	435	iter_xp = next_xp )
	436	{
	437	// save extended pointer on next item in xlist
	438	next_xp = hal_remote_l64( iter_xp );
[611]	439
[625]	440	// get pointers on current vseg in VSL
	441	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
	442	vseg = GET_PTR( vseg_xp );
	443	vseg_type = vseg->type;
[567]	444
[625]	445	#if( DEBUG_VMM_USER_RESET & 1 )
	446	if( DEBUG_VMM_USER_RESET < cycle )
	447	printk("\n[%s] found %s vseg / vpn_base %x / vpn_size %d\n",
	448	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	449	#endif
	450	// delete non kernel vseg
	451	if( (vseg_type != VSEG_TYPE_KCODE) &&
	452	(vseg_type != VSEG_TYPE_KDATA) &&
	453	(vseg_type != VSEG_TYPE_KDEV ) )
	454	{
	455	// remove vseg from VSL
	456	vmm_remove_vseg( process , vseg );
[567]	457
[625]	458	#if( DEBUG_VMM_USER_RESET & 1 )
	459	if( DEBUG_VMM_USER_RESET < cycle )
	460	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
	461	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	462	#endif
	463	}
	464	else
	465	{
[567]	466
[625]	467	#if( DEBUG_VMM_USER_RESET & 1 )
	468	if( DEBUG_VMM_USER_RESET < cycle )
	469	printk("\n[%s] keep %s vseg / vpn_base %x / vpn_size %d\n",
	470	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	471	#endif
	472	}
	473	} // end loop on vsegs in VSL
[567]	474
[625]	475	// release the VSL lock
	476	remote_rwlock_wr_release( lock_xp );
[567]	477
[625]	478	// FIXME il faut gérer les process copies...
[611]	479
[625]	480	#if DEBUG_VMM_USER_RESET
	481	cycle = (uint32_t)hal_get_cycles();
	482	if( DEBUG_VMM_USER_RESET < cycle )
	483	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	484	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
	485	#endif
[611]	486
[635]	487	#if (DEBUG_VMM_USER_RESET & 1 )
	488	if( DEBUG_VMM_USER_RESET < cycle )
	489	hal_vmm_display( XPTR( local_cxy , process ) , true );
	490	#endif
	491
[625]	492	} // end vmm_user_reset()
[611]	493
[595]	494	////////////////////////////////////////////////
[433]	495	void vmm_global_update_pte( process_t * process,
	496	vpn_t vpn,
	497	uint32_t attr,
	498	ppn_t ppn )
[23]	499	{
[408]	500	xlist_entry_t * process_root_ptr;
	501	xptr_t process_root_xp;
	502	xptr_t process_iter_xp;
[23]	503
[408]	504	xptr_t remote_process_xp;
	505	cxy_t remote_process_cxy;
	506	process_t * remote_process_ptr;
	507	xptr_t remote_gpt_xp;
[23]	508
[408]	509	pid_t pid;
	510	cxy_t owner_cxy;
	511	lpid_t owner_lpid;
[23]	512
[438]	513	#if DEBUG_VMM_UPDATE_PTE
[433]	514	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	515	thread_t * this = CURRENT_THREAD;
[438]	516	if( DEBUG_VMM_UPDATE_PTE < cycle )
[635]	517	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / attr %x / ppn %x / ycle %d\n",
	518	__FUNCTION__, this->process->pid, this->trdid, process->pid, vpn, attr, ppn, cycle );
[433]	519	#endif
	520
[408]	521	// get extended pointer on root of process copies xlist in owner cluster
	522	pid = process->pid;
	523	owner_cxy = CXY_FROM_PID( pid );
	524	owner_lpid = LPID_FROM_PID( pid );
	525	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	526	process_root_xp = XPTR( owner_cxy , process_root_ptr );
[23]	527
[635]	528	// check local cluster is owner cluster
	529	assert( (owner_cxy == local_cxy) , "must be called in owner cluster\n");
	530
[408]	531	// loop on destination process copies
	532	XLIST_FOREACH( process_root_xp , process_iter_xp )
	533	{
	534	// get cluster and local pointer on remote process
	535	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	536	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	537	remote_process_cxy = GET_CXY( remote_process_xp );
[407]	538
[635]	539	#if (DEBUG_VMM_UPDATE_PTE & 1)
[438]	540	if( DEBUG_VMM_UPDATE_PTE < cycle )
[635]	541	printk("\n[%s] thread[%x,%x] handling vpn %x for process %x in cluster %x\n",
[595]	542	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
[433]	543	#endif
	544
[408]	545	// get extended pointer on remote gpt
	546	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	547
[433]	548	// update remote GPT
	549	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
[408]	550	}
	551
[438]	552	#if DEBUG_VMM_UPDATE_PTE
[433]	553	cycle = (uint32_t)hal_get_cycles();
[438]	554	if( DEBUG_VMM_UPDATE_PTE < cycle )
[595]	555	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
	556	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
[433]	557	#endif
	558
[635]	559	#if (DEBUG_VMM_UPDATE_PTE & 1)
	560	hal_vmm_display( process , true );
	561	#endif
	562
[433]	563	} // end vmm_global_update_pte()
	564
[408]	565	///////////////////////////////////////
	566	void vmm_set_cow( process_t * process )
	567	{
	568	vmm_t * vmm;
	569
	570	xlist_entry_t * process_root_ptr;
	571	xptr_t process_root_xp;
	572	xptr_t process_iter_xp;
	573
	574	xptr_t remote_process_xp;
	575	cxy_t remote_process_cxy;
	576	process_t * remote_process_ptr;
	577	xptr_t remote_gpt_xp;
	578
	579	xptr_t vseg_root_xp;
	580	xptr_t vseg_iter_xp;
	581
	582	xptr_t vseg_xp;
	583	vseg_t * vseg;
	584
	585	pid_t pid;
	586	cxy_t owner_cxy;
	587	lpid_t owner_lpid;
	588
[635]	589	// get target process PID
	590	pid = process->pid;
	591
[438]	592	#if DEBUG_VMM_SET_COW
[595]	593	uint32_t cycle = (uint32_t)hal_get_cycles();
	594	thread_t * this = CURRENT_THREAD;
[438]	595	if( DEBUG_VMM_SET_COW < cycle )
[595]	596	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
[635]	597	__FUNCTION__, this->process->pid, this->trdid, pid , cycle );
[433]	598	#endif
[408]	599
[635]	600	#if (DEBUG_VMM_SET_COW & 1)
	601	if( DEBUG_VMM_SET_COW < cycle )
	602	hal_vmm_display( process , true );
	603	#endif
	604
[567]	605	// check cluster is reference
[635]	606	assert( (XPTR( local_cxy , process ) == process->ref_xp),
	607	"local cluster must be process reference cluster\n");
[408]	608
	609	// get pointer on reference VMM
	610	vmm = &process->vmm;
	611
	612	// get extended pointer on root of process copies xlist in owner cluster
	613	owner_cxy = CXY_FROM_PID( pid );
	614	owner_lpid = LPID_FROM_PID( pid );
	615	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
	616	process_root_xp = XPTR( owner_cxy , process_root_ptr );
	617
	618	// get extended pointer on root of vsegs xlist from reference VMM
	619	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	620
[635]	621	// loop on target process copies
[408]	622	XLIST_FOREACH( process_root_xp , process_iter_xp )
	623	{
[635]	624	// get cluster and local pointer on remote process copy
[408]	625	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
[433]	626	remote_process_ptr = GET_PTR( remote_process_xp );
[408]	627	remote_process_cxy = GET_CXY( remote_process_xp );
	628
[595]	629	#if (DEBUG_VMM_SET_COW & 1)
[438]	630	if( DEBUG_VMM_SET_COW < cycle )
[635]	631	printk("\n[%s] thread[%x,%x] (%x) handles process %x in cluster %x\n",
	632	__FUNCTION__, this->process->pid, this->trdid, this, pid, remote_process_cxy );
[433]	633	#endif
[408]	634
	635	// get extended pointer on remote gpt
	636	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
	637
	638	// loop on vsegs in (local) reference process VSL
	639	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
	640	{
	641	// get pointer on vseg
	642	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
[433]	643	vseg = GET_PTR( vseg_xp );
[408]	644
	645	// get vseg type, base and size
	646	uint32_t type = vseg->type;
	647	vpn_t vpn_base = vseg->vpn_base;
	648	vpn_t vpn_size = vseg->vpn_size;
	649
[595]	650	#if (DEBUG_VMM_SET_COW & 1)
[438]	651	if( DEBUG_VMM_SET_COW < cycle )
[635]	652	printk("\n[%s] thread[%x,%x] found vseg %s / vpn_base = %x / vpn_size = %x\n",
[595]	653	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
[433]	654	#endif
	655	// only DATA, ANON and REMOTE vsegs
[408]	656	if( (type == VSEG_TYPE_DATA) \|\|
	657	(type == VSEG_TYPE_ANON) \|\|
	658	(type == VSEG_TYPE_REMOTE) )
	659	{
[433]	660	vpn_t vpn;
	661	uint32_t attr;
	662	ppn_t ppn;
	663	xptr_t page_xp;
	664	cxy_t page_cxy;
	665	page_t * page_ptr;
	666	xptr_t forks_xp;
[469]	667	xptr_t lock_xp;
[433]	668
	669	// update flags in remote GPT
	670	hal_gpt_set_cow( remote_gpt_xp,
	671	vpn_base,
	672	vpn_size );
	673
	674	// atomically increment pending forks counter in physical pages,
[635]	675	// this is only done once, when handling the reference copy
[433]	676	if( remote_process_cxy == local_cxy )
	677	{
[635]	678
	679	#if (DEBUG_VMM_SET_COW & 1)
	680	if( DEBUG_VMM_SET_COW < cycle )
	681	printk("\n[%s] thread[%x,%x] handles vseg %s / vpn_base = %x / vpn_size = %x\n",
	682	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
	683	#endif
[433]	684	// scan all pages in vseg
	685	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	686	{
	687	// get page attributes and PPN from reference GPT
[585]	688	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
[433]	689
	690	// atomically update pending forks counter if page is mapped
	691	if( attr & GPT_MAPPED )
	692	{
[469]	693	// get pointers and cluster on page descriptor
[433]	694	page_xp = ppm_ppn2page( ppn );
	695	page_cxy = GET_CXY( page_xp );
	696	page_ptr = GET_PTR( page_xp );
[469]	697
	698	// get extended pointers on "forks" and "lock"
[433]	699	forks_xp = XPTR( page_cxy , &page_ptr->forks );
[469]	700	lock_xp = XPTR( page_cxy , &page_ptr->lock );
	701
[567]	702	// take lock protecting "forks" counter
	703	remote_busylock_acquire( lock_xp );
	704
[469]	705	// increment "forks"
[433]	706	hal_remote_atomic_add( forks_xp , 1 );
[567]	707
	708	// release lock protecting "forks" counter
	709	remote_busylock_release( lock_xp );
[433]	710	}
	711	} // end loop on vpn
[635]	712
	713	#if (DEBUG_VMM_SET_COW & 1)
	714	if( DEBUG_VMM_SET_COW < cycle )
	715	printk("\n[%s] thread[%x,%x] completes vseg %s / vpn_base = %x / vpn_size = %x\n",
	716	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
	717	#endif
[433]	718	} // end if local
	719	} // end if vseg type
	720	} // end loop on vsegs
[408]	721	} // end loop on process copies
	722
[438]	723	#if DEBUG_VMM_SET_COW
[433]	724	cycle = (uint32_t)hal_get_cycles();
[438]	725	if( DEBUG_VMM_SET_COW < cycle )
[595]	726	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
	727	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
[433]	728	#endif
[408]	729
	730	} // end vmm_set-cow()
	731
	732	/////////////////////////////////////////////////
	733	error_t vmm_fork_copy( process_t * child_process,
	734	xptr_t parent_process_xp )
	735	{
	736	error_t error;
	737	cxy_t parent_cxy;
	738	process_t * parent_process;
	739	vmm_t * parent_vmm;
	740	xptr_t parent_lock_xp;
	741	vmm_t * child_vmm;
	742	xptr_t iter_xp;
	743	xptr_t parent_vseg_xp;
	744	vseg_t * parent_vseg;
	745	vseg_t * child_vseg;
	746	uint32_t type;
	747	vpn_t vpn;
	748	vpn_t vpn_base;
	749	vpn_t vpn_size;
	750	xptr_t parent_root_xp;
	751	bool_t mapped;
	752	ppn_t ppn;
	753
[438]	754	#if DEBUG_VMM_FORK_COPY
[433]	755	uint32_t cycle = (uint32_t)hal_get_cycles();
[595]	756	thread_t * this = CURRENT_THREAD;
[438]	757	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	758	printk("\n[%s] thread %x enter / cycle %d\n",
	759	__FUNCTION__ , this->process->pid, this->trdid, cycle );
[433]	760	#endif
[408]	761
	762	// get parent process cluster and local pointer
	763	parent_cxy = GET_CXY( parent_process_xp );
[433]	764	parent_process = GET_PTR( parent_process_xp );
[408]	765
	766	// get local pointers on parent and child VMM
	767	parent_vmm = &parent_process->vmm;
	768	child_vmm = &child_process->vmm;
	769
[625]	770	// build extended pointer on parent VSL root and lock
[408]	771	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
[625]	772	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsl_lock );
[408]	773
[567]	774	// take the lock protecting the parent VSL in read mode
	775	remote_rwlock_rd_acquire( parent_lock_xp );
[415]	776
[408]	777	// loop on parent VSL xlist
	778	XLIST_FOREACH( parent_root_xp , iter_xp )
[23]	779	{
[625]	780	// get pointers on current parent vseg
[408]	781	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	782	parent_vseg = GET_PTR( parent_vseg_xp );
[23]	783
[408]	784	// get vseg type
[567]	785	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
[408]	786
[438]	787	#if DEBUG_VMM_FORK_COPY
[433]	788	cycle = (uint32_t)hal_get_cycles();
[438]	789	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	790	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
	791	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
[567]	792	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	793	#endif
[23]	794
[623]	795	// all parent vsegs - but STACK and kernel vsegs - must be copied in child VSL
	796	if( (type != VSEG_TYPE_STACK) && (type != VSEG_TYPE_KCODE) &&
	797	(type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
[23]	798	{
[408]	799	// allocate memory for a new child vseg
	800	child_vseg = vseg_alloc();
	801	if( child_vseg == NULL ) // release all allocated vsegs
[23]	802	{
[408]	803	vmm_destroy( child_process );
	804	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
	805	return -1;
[23]	806	}
	807
[408]	808	// copy parent vseg to child vseg
	809	vseg_init_from_ref( child_vseg , parent_vseg_xp );
[23]	810
[625]	811	// build extended pointer on VSL lock
	812	xptr_t lock_xp = XPTR( local_cxy , &child_vmm->vsl_lock );
	813
	814	// take the VSL lock in write mode
	815	remote_rwlock_wr_acquire( lock_xp );
	816
[408]	817	// register child vseg in child VSL
[611]	818	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
[407]	819
[625]	820	// release the VSL lock
	821	remote_rwlock_wr_release( lock_xp );
	822
[438]	823	#if DEBUG_VMM_FORK_COPY
[433]	824	cycle = (uint32_t)hal_get_cycles();
[438]	825	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	826	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
	827	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
[567]	828	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
[433]	829	#endif
[625]	830	// copy DATA, ANON, REMOTE, FILE parent GPT entries to child GPT
[408]	831	if( type != VSEG_TYPE_CODE )
	832	{
[625]	833	// activate the COW for DATA, ANON, REMOTE vsegs only
[635]	834	// cow = ( type != VSEG_TYPE_FILE );
[23]	835
[408]	836	vpn_base = child_vseg->vpn_base;
	837	vpn_size = child_vseg->vpn_size;
[23]	838
[408]	839	// scan pages in parent vseg
	840	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
	841	{
	842	error = hal_gpt_pte_copy( &child_vmm->gpt,
[625]	843	vpn,
[408]	844	XPTR( parent_cxy , &parent_vmm->gpt ),
	845	vpn,
[635]	846	false, // does not handle COW flag
	847	&ppn, // unused
	848	&mapped ); // unused
[408]	849	if( error )
	850	{
	851	vmm_destroy( child_process );
	852	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
	853	return -1;
	854	}
	855
[438]	856	#if DEBUG_VMM_FORK_COPY
[433]	857	cycle = (uint32_t)hal_get_cycles();
[438]	858	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	859	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
	860	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
[433]	861	#endif
[408]	862	}
	863	} // end if no code & no stack
	864	} // end if no stack
	865	} // end loop on vsegs
	866
[567]	867	// release the parent VSL lock in read mode
	868	remote_rwlock_rd_release( parent_lock_xp );
[408]	869
	870	// initialize the child VMM STACK allocator
	871	child_vmm->stack_mgr.bitmap = 0;
	872	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
	873
	874	// initialize the child VMM MMAP allocator
[23]	875	uint32_t i;
[408]	876	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
	877	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
	878	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
[625]	879	for( i = 0 ; i < 32 ; i++ )
	880	{
	881	xlist_root_init( XPTR( local_cxy , &child_vmm->mmap_mgr.zombi_list[i] ) );
	882	}
[23]	883
[178]	884	// initialize instrumentation counters
[635]	885	child_vmm->false_pgfault_nr = 0;
	886	child_vmm->local_pgfault_nr = 0;
	887	child_vmm->global_pgfault_nr = 0;
	888	child_vmm->false_pgfault_cost = 0;
	889	child_vmm->local_pgfault_cost = 0;
	890	child_vmm->global_pgfault_cost = 0;
[23]	891
[408]	892	// copy base addresses from parent VMM to child VMM
	893	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
	894	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
	895	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
	896	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
	897	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
[23]	898
[408]	899	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
[23]	900
[124]	901	hal_fence();
[23]	902
[438]	903	#if DEBUG_VMM_FORK_COPY
[433]	904	cycle = (uint32_t)hal_get_cycles();
[438]	905	if( DEBUG_VMM_FORK_COPY < cycle )
[595]	906	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
	907	__FUNCTION__ , this->process->pid, this->trdid , cycle );
[433]	908	#endif
	909
[23]	910	return 0;
	911
[408]	912	} // vmm_fork_copy()
[204]	913
[1]	914	///////////////////////////////////////
	915	void vmm_destroy( process_t * process )
	916	{
[408]	917	xptr_t vseg_xp;
[1]	918	vseg_t * vseg;
	919
[438]	920	#if DEBUG_VMM_DESTROY
[635]	921	uint32_t cycle = (uint32_t)hal_get_cycles();
	922	thread_t * this = CURRENT_THREAD;
[438]	923	if( DEBUG_VMM_DESTROY < cycle )
[595]	924	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
	925	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
[433]	926	#endif
[416]	927
[438]	928	#if (DEBUG_VMM_DESTROY & 1 )
[443]	929	if( DEBUG_VMM_DESTROY < cycle )
[635]	930	hal_vmm_display( XPTR( local_cxy, process ) , true );
[437]	931	#endif
	932
[433]	933	// get pointer on local VMM
[1]	934	vmm_t * vmm = &process->vmm;
	935
[625]	936	// build extended pointer on VSL root, VSL lock and GPT lock
	937	xptr_t vsl_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
	938	xptr_t vsl_lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
[408]	939
[625]	940	// take the VSL lock
	941	remote_rwlock_wr_acquire( vsl_lock_xp );
	942
[611]	943	// scan the VSL to delete all registered vsegs
[625]	944	// (we don't use a FOREACH in case of item deletion)
	945	xptr_t iter_xp;
	946	xptr_t next_xp;
	947	for( iter_xp = hal_remote_l64( vsl_root_xp ) ;
	948	iter_xp != vsl_root_xp ;
	949	iter_xp = next_xp )
[1]	950	{
[625]	951	// save extended pointer on next item in xlist
	952	next_xp = hal_remote_l64( iter_xp );
[409]	953
[625]	954	// get pointers on current vseg in VSL
	955	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
	956	vseg = GET_PTR( vseg_xp );
	957
[611]	958	// delete vseg and release physical pages
[625]	959	vmm_remove_vseg( process , vseg );
[409]	960
[443]	961	#if( DEBUG_VMM_DESTROY & 1 )
	962	if( DEBUG_VMM_DESTROY < cycle )
[611]	963	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
[443]	964	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	965	#endif
	966
[1]	967	}
	968
[625]	969	// release the VSL lock
	970	remote_rwlock_wr_release( vsl_lock_xp );
	971
	972	// remove all registered MMAP vsegs
	973	// from zombi_lists in MMAP allocator
[1]	974	uint32_t i;
	975	for( i = 0 ; i<32 ; i++ )
	976	{
[625]	977	// build extended pointer on zombi_list[i]
	978	xptr_t root_xp = XPTR( local_cxy , &vmm->mmap_mgr.zombi_list[i] );
	979
	980	// scan zombi_list[i]
	981	while( !xlist_is_empty( root_xp ) )
[1]	982	{
[625]	983	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
	984	vseg = GET_PTR( vseg_xp );
[443]	985
	986	#if( DEBUG_VMM_DESTROY & 1 )
	987	if( DEBUG_VMM_DESTROY < cycle )
[595]	988	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
[443]	989	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	990	#endif
[611]	991	// clean vseg descriptor
	992	vseg->vmm = NULL;
	993
[625]	994	// remove vseg from zombi_list
[611]	995	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
	996
	997	// release vseg descriptor
[1]	998	vseg_free( vseg );
[443]	999
	1000	#if( DEBUG_VMM_DESTROY & 1 )
	1001	if( DEBUG_VMM_DESTROY < cycle )
[595]	1002	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
[443]	1003	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
	1004	#endif
[1]	1005	}
	1006	}
	1007
[409]	1008	// release memory allocated to the GPT itself
[1]	1009	hal_gpt_destroy( &vmm->gpt );
	1010
[438]	1011	#if DEBUG_VMM_DESTROY
[433]	1012	cycle = (uint32_t)hal_get_cycles();
[438]	1013	if( DEBUG_VMM_DESTROY < cycle )
[595]	1014	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
	1015	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
[433]	1016	#endif
[416]	1017
[204]	1018	} // end vmm_destroy()
	1019
[1]	1020	/////////////////////////////////////////////////
	1021	vseg_t * vmm_check_conflict( process_t * process,
[21]	1022	vpn_t vpn_base,
[1]	1023	vpn_t vpn_size )
	1024	{
	1025	vmm_t * vmm = &process->vmm;
[408]	1026
	1027	// scan the VSL
[1]	1028	vseg_t * vseg;
[408]	1029	xptr_t iter_xp;
	1030	xptr_t vseg_xp;
	1031	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[1]	1032
[408]	1033	XLIST_FOREACH( root_xp , iter_xp )
[1]	1034	{
[408]	1035	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1036	vseg = GET_PTR( vseg_xp );
[204]	1037
[21]	1038	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
	1039	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
[1]	1040	}
	1041	return NULL;
	1042
[204]	1043	} // end vmm_check_conflict()
	1044
[1]	1045
	1046
[407]	1047	////////////////////////////////////////////////
	1048	vseg_t * vmm_create_vseg( process_t * process,
	1049	vseg_type_t type,
[635]	1050	intptr_t base, // ltid for VSEG_TYPE_STACK
[407]	1051	uint32_t size,
	1052	uint32_t file_offset,
	1053	uint32_t file_size,
	1054	xptr_t mapper_xp,
	1055	cxy_t cxy )
[1]	1056	{
	1057	vseg_t * vseg; // created vseg pointer
[204]	1058	vpn_t vpn_base; // first page index
[595]	1059	vpn_t vpn_size; // number of pages covered by vseg
[1]	1060	error_t error;
	1061
[635]	1062	#if (DEBUG_VMM_CREATE_VSEG & 1)
[595]	1063	thread_t * this = CURRENT_THREAD;
	1064	uint32_t cycle = (uint32_t)hal_get_cycles();
[438]	1065	if( DEBUG_VMM_CREATE_VSEG < cycle )
[635]	1066	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cxy %x / cycle %d\n",
	1067	__FUNCTION__, this->process->pid, this->trdid,
	1068	process->pid, vseg_type_str(type), base, cxy, cycle );
[433]	1069	#endif
[21]	1070
[407]	1071	// get pointer on VMM
	1072	vmm_t * vmm = &process->vmm;
[21]	1073
[204]	1074	// compute base, size, vpn_base, vpn_size, depending on vseg type
[407]	1075	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
[595]	1076
[1]	1077	if( type == VSEG_TYPE_STACK )
	1078	{
	1079	// get vpn_base and vpn_size from STACK allocator
[625]	1080	vmm_stack_alloc( vmm , base , &vpn_base , &vpn_size );
[1]	1081
	1082	// compute vseg base and size from vpn_base and vpn_size
	1083	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1084	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
	1085	}
[595]	1086	else if( type == VSEG_TYPE_FILE )
	1087	{
	1088	// compute page index (in mapper) for first byte
	1089	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
	1090
	1091	// compute page index (in mapper) for last byte
	1092	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1093
	1094	// compute offset in first page
	1095	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
	1096
	1097	// compute number of pages required in virtual space
	1098	vpn_t npages = vpn_max - vpn_min + 1;
	1099
	1100	// get vpn_base and vpn_size from MMAP allocator
	1101	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	1102	if( error )
	1103	{
	1104	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	1105	__FUNCTION__ , process->pid , local_cxy );
	1106	return NULL;
	1107	}
	1108
	1109	// set the vseg base (not always aligned for FILE)
	1110	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
	1111	}
[21]	1112	else if( (type == VSEG_TYPE_ANON) \|\|
[1]	1113	(type == VSEG_TYPE_REMOTE) )
	1114	{
[595]	1115	// compute number of required pages in virtual space
	1116	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
	1117	if( size & CONFIG_PPM_PAGE_MASK) npages++;
	1118
[1]	1119	// get vpn_base and vpn_size from MMAP allocator
	1120	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
	1121	if( error )
	1122	{
	1123	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
	1124	__FUNCTION__ , process->pid , local_cxy );
	1125	return NULL;
	1126	}
	1127
[595]	1128	// set vseg base (always aligned for ANON or REMOTE)
[1]	1129	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
	1130	}
[623]	1131	else // VSEG_TYPE_DATA, VSEG_TYPE_CODE or KERNEL vseg
[1]	1132	{
[204]	1133	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
	1134	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1135
	1136	vpn_base = vpn_min;
	1137	vpn_size = vpn_max - vpn_min + 1;
[1]	1138	}
	1139
	1140	// check collisions
	1141	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
[624]	1142
[1]	1143	if( vseg != NULL )
	1144	{
[614]	1145	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
	1146	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
[407]	1147	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
[1]	1148	return NULL;
	1149	}
	1150
	1151	// allocate physical memory for vseg descriptor
	1152	vseg = vseg_alloc();
	1153	if( vseg == NULL )
	1154	{
	1155	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
[407]	1156	__FUNCTION__ , process->pid );
[1]	1157	return NULL;
	1158	}
	1159
[635]	1160	#if (DEBUG_VMM_CREATE_VSEG & 1)
[614]	1161	if( DEBUG_VMM_CREATE_VSEG < cycle )
	1162	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
	1163	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
	1164	#endif
	1165
[1]	1166	// initialize vseg descriptor
[407]	1167	vseg_init( vseg,
	1168	type,
	1169	base,
	1170	size,
	1171	vpn_base,
	1172	vpn_size,
	1173	file_offset,
	1174	file_size,
	1175	mapper_xp,
	1176	cxy );
[1]	1177
[625]	1178	// build extended pointer on VSL lock
	1179	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
	1180
	1181	// take the VSL lock in write mode
	1182	remote_rwlock_wr_acquire( lock_xp );
	1183
[408]	1184	// attach vseg to VSL
[611]	1185	vmm_attach_vseg_to_vsl( vmm , vseg );
[1]	1186
[625]	1187	// release the VSL lock
	1188	remote_rwlock_wr_release( lock_xp );
	1189
[438]	1190	#if DEBUG_VMM_CREATE_VSEG
[433]	1191	cycle = (uint32_t)hal_get_cycles();
[438]	1192	if( DEBUG_VMM_CREATE_VSEG < cycle )
[635]	1193	printk("\n[%s] thread[%x,%x] exit / process %x / %s / base %x / cxy %x / cycle %d\n",
	1194	__FUNCTION__, this->process->pid, this->trdid,
	1195	process->pid, vseg_type_str(type), base, cxy, cycle );
[433]	1196	#endif
[21]	1197
[1]	1198	return vseg;
	1199
[406]	1200	} // vmm_create_vseg()
	1201
[625]	1202
	1203	//////////////////////////////////////////
	1204	void vmm_remove_vseg( process_t * process,
	1205	vseg_t * vseg )
[1]	1206	{
[625]	1207	vmm_t * vmm; // local pointer on process VMM
[629]	1208	xptr_t gpt_xp; // extended pointer on GPT
[625]	1209	bool_t is_ref; // local process is reference process
	1210	uint32_t vseg_type; // vseg type
[21]	1211	vpn_t vpn; // VPN of current PTE
	1212	vpn_t vpn_min; // VPN of first PTE
[1]	1213	vpn_t vpn_max; // VPN of last PTE (excluded)
[409]	1214	ppn_t ppn; // current PTE ppn value
	1215	uint32_t attr; // current PTE attributes
	1216	xptr_t page_xp; // extended pointer on page descriptor
	1217	cxy_t page_cxy; // page descriptor cluster
	1218	page_t * page_ptr; // page descriptor pointer
[625]	1219	xptr_t count_xp; // extended pointer on page refcount
[1]	1220
[625]	1221	// check arguments
	1222	assert( (process != NULL), "process argument is NULL" );
	1223	assert( (vseg != NULL), "vseg argument is NULL" );
[409]	1224
[625]	1225	// compute is_ref
	1226	is_ref = (GET_CXY( process->ref_xp ) == local_cxy);
[1]	1227
[625]	1228	// get pointers on local process VMM
[611]	1229	vmm = &process->vmm;
	1230
[629]	1231	// build extended pointer on GPT
	1232	gpt_xp = XPTR( local_cxy , &vmm->gpt );
	1233
[623]	1234	// get relevant vseg infos
[624]	1235	vseg_type = vseg->type;
	1236	vpn_min = vseg->vpn_base;
	1237	vpn_max = vpn_min + vseg->vpn_size;
[623]	1238
[625]	1239	#if DEBUG_VMM_REMOVE_VSEG
	1240	uint32_t cycle = (uint32_t)hal_get_cycles();
	1241	thread_t * this = CURRENT_THREAD;
	1242	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1243	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cycle %d\n",
	1244	__FUNCTION__, this->process->pid, this->trdid,
	1245	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
	1246	#endif
	1247
	1248	// loop on PTEs in GPT
[1]	1249	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
	1250	{
[625]	1251	// get ppn and attr
[629]	1252	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
[409]	1253
[625]	1254	if( attr & GPT_MAPPED ) // PTE is mapped
[409]	1255	{
[437]	1256
[625]	1257	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
	1258	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1259	printk("- unmap vpn %x / ppn %x / %s" , vpn , ppn, vseg_type_str(vseg_type) );
[437]	1260	#endif
[585]	1261	// unmap GPT entry in local GPT
[629]	1262	hal_gpt_reset_pte( gpt_xp , vpn );
[409]	1263
[625]	1264	// get pointers on physical page descriptor
	1265	page_xp = ppm_ppn2page( ppn );
	1266	page_cxy = GET_CXY( page_xp );
	1267	page_ptr = GET_PTR( page_xp );
[409]	1268
[625]	1269	// decrement page refcount
	1270	count_xp = XPTR( page_cxy , &page_ptr->refcount );
[633]	1271	hal_remote_atomic_add( count_xp , -1 );
[624]	1272
[625]	1273	// compute the ppn_release condition depending on vseg type
	1274	bool_t ppn_release;
	1275	if( (vseg_type == VSEG_TYPE_FILE) \|\|
	1276	(vseg_type == VSEG_TYPE_KCODE) \|\|
	1277	(vseg_type == VSEG_TYPE_KDATA) \|\|
	1278	(vseg_type == VSEG_TYPE_KDEV) )
	1279	{
	1280	// no physical page release for FILE and KERNEL
	1281	ppn_release = false;
	1282	}
	1283	else if( (vseg_type == VSEG_TYPE_CODE) \|\|
	1284	(vseg_type == VSEG_TYPE_STACK) )
	1285	{
	1286	// always release physical page for private vsegs
	1287	ppn_release = true;
	1288	}
	1289	else if( (vseg_type == VSEG_TYPE_ANON) \|\|
	1290	(vseg_type == VSEG_TYPE_REMOTE) )
	1291	{
	1292	// release physical page if reference cluster
	1293	ppn_release = is_ref;
	1294	}
	1295	else if( is_ref ) // vseg_type == DATA in reference cluster
	1296	{
	1297	// get extended pointers on forks and lock field in page descriptor
	1298	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	1299	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
[433]	1300
[625]	1301	// take lock protecting "forks" counter
[621]	1302	remote_busylock_acquire( lock_xp );
[623]	1303
[625]	1304	// get number of pending forks from page descriptor
	1305	uint32_t forks = hal_remote_l32( forks_xp );
[623]	1306
[625]	1307	// decrement pending forks counter if required
	1308	if( forks ) hal_remote_atomic_add( forks_xp , -1 );
[624]	1309
[625]	1310	// release lock protecting "forks" counter
	1311	remote_busylock_release( lock_xp );
[624]	1312
[625]	1313	// release physical page if forks == 0
	1314	ppn_release = (forks == 0);
	1315	}
	1316	else // vseg_type == DATA not in reference cluster
	1317	{
	1318	// no physical page release if not in reference cluster
	1319	ppn_release = false;
	1320	}
[611]	1321
[625]	1322	// release physical page to relevant kmem when required
[632]	1323	if( ppn_release ) ppm_remote_free_pages( page_cxy , page_ptr );
[623]	1324
[625]	1325	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
	1326	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1327	{
	1328	if( ppn_release ) printk(" / released to kmem\n" );
	1329	else printk("\n");
	1330	}
	1331	#endif
[409]	1332	}
[1]	1333	}
[433]	1334
[625]	1335	// remove vseg from VSL
[611]	1336	vmm_detach_vseg_from_vsl( vmm , vseg );
	1337
[625]	1338	// release vseg descriptor depending on vseg type
	1339	if( vseg_type == VSEG_TYPE_STACK )
	1340	{
	1341	// release slot to local stack allocator
	1342	vmm_stack_free( vmm , vseg );
	1343
	1344	// release vseg descriptor to local kmem
	1345	vseg_free( vseg );
	1346	}
	1347	else if( (vseg_type == VSEG_TYPE_ANON) \|\|
	1348	(vseg_type == VSEG_TYPE_FILE) \|\|
	1349	(vseg_type == VSEG_TYPE_REMOTE) )
	1350	{
	1351	// release vseg to local mmap allocator
	1352	vmm_mmap_free( vmm , vseg );
	1353	}
	1354	else
	1355	{
	1356	// release vseg descriptor to local kmem
	1357	vseg_free( vseg );
	1358	}
	1359
	1360	#if DEBUG_VMM_REMOVE_VSEG
[433]	1361	cycle = (uint32_t)hal_get_cycles();
[625]	1362	if( DEBUG_VMM_REMOVE_VSEG < cycle )
	1363	printk("[%s] thread[%x,%x] exit / process %x / %s / base %x / cycle %d\n",
	1364	__FUNCTION__, this->process->pid, this->trdid,
	1365	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
[433]	1366	#endif
	1367
[625]	1368	} // end vmm_remove_vseg()
[1]	1369
[625]	1370
	1371	///////////////////////////////////
	1372	void vmm_delete_vseg( pid_t pid,
	1373	intptr_t vaddr )
	1374	{
	1375	process_t * process; // local pointer on local process
	1376	vseg_t * vseg; // local pointer on local vseg containing vaddr
	1377
	1378	// get local pointer on local process descriptor
	1379	process = cluster_get_local_process_from_pid( pid );
	1380
	1381	if( process == NULL )
	1382	{
	1383	printk("\n[WARNING] in %s : cannot get local process descriptor\n",
	1384	__FUNCTION__ );
	1385	return;
	1386	}
	1387
	1388	// get local pointer on local vseg containing vaddr
	1389	vseg = vmm_vseg_from_vaddr( &process->vmm , vaddr );
	1390
	1391	if( vseg == NULL )
	1392	{
	1393	printk("\n[WARNING] in %s : cannot get vseg descriptor\n",
	1394	__FUNCTION__ );
	1395	return;
	1396	}
	1397
	1398	// call relevant function
	1399	vmm_remove_vseg( process , vseg );
	1400
	1401	} // end vmm_delete_vseg
	1402
	1403
[611]	1404	/////////////////////////////////////////////
	1405	vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
	1406	intptr_t vaddr )
[406]	1407	{
[408]	1408	xptr_t vseg_xp;
	1409	vseg_t * vseg;
[625]	1410	xptr_t iter_xp;
[406]	1411
[408]	1412	// get extended pointers on VSL lock and root
[625]	1413	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
[408]	1414	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
[406]	1415
[408]	1416	// get lock protecting the VSL
[567]	1417	remote_rwlock_rd_acquire( lock_xp );
[408]	1418
	1419	// scan the list of vsegs in VSL
	1420	XLIST_FOREACH( root_xp , iter_xp )
[406]	1421	{
[625]	1422	// get pointers on vseg
[408]	1423	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
[433]	1424	vseg = GET_PTR( vseg_xp );
[595]	1425
[625]	1426	// return success when match
[408]	1427	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
[595]	1428	{
[408]	1429	// return success
[567]	1430	remote_rwlock_rd_release( lock_xp );
[408]	1431	return vseg;
	1432	}
[406]	1433	}
	1434
[408]	1435	// return failure
[567]	1436	remote_rwlock_rd_release( lock_xp );
[408]	1437	return NULL;
[406]	1438
[595]	1439	} // end vmm_vseg_from_vaddr()
[406]	1440
[1]	1441	/////////////////////////////////////////////
	1442	error_t vmm_resize_vseg( process_t * process,
	1443	intptr_t base,
	1444	intptr_t size )
	1445	{
[406]	1446	error_t error;
	1447	vseg_t * new;
	1448	vpn_t vpn_min;
	1449	vpn_t vpn_max;
[1]	1450
[623]	1451	#if DEBUG_VMM_RESIZE_VSEG
	1452	uint32_t cycle = (uint32_t)hal_get_cycles();
	1453	thread_t * this = CURRENT_THREAD;
	1454	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1455	printk("\n[%s] thread[%x,%x] enter / process %x / base %x / size %d / cycle %d\n",
	1456	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
	1457	#endif
	1458
[1]	1459	// get pointer on process VMM
	1460	vmm_t * vmm = &process->vmm;
	1461
	1462	intptr_t addr_min = base;
	1463	intptr_t addr_max = base + size;
	1464
	1465	// get pointer on vseg
[595]	1466	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
[1]	1467
[623]	1468	if( vseg == NULL)
	1469	{
	1470	printk("\n[ERROR] in %s : vseg(%x,%d) not found\n",
	1471	__FUNCTION__, base , size );
	1472	return -1;
	1473	}
[21]	1474
[623]	1475	// resize depends on unmapped region base and size
[611]	1476	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // not included in vseg
[1]	1477	{
[623]	1478	printk("\n[ERROR] in %s : unmapped region[%x->%x[ not included in vseg[%x->%x[\n",
	1479	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1480
[611]	1481	error = -1;
[1]	1482	}
[611]	1483	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be deleted
[1]	1484	{
[623]	1485
	1486	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1487	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1488	printk("\n[%s] unmapped region[%x->%x[ equal vseg[%x->%x[\n",
	1489	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1490	#endif
[611]	1491	vmm_delete_vseg( process->pid , vseg->min );
[623]	1492
	1493	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1494	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1495	printk("\n[%s] thread[%x,%x] deleted vseg\n",
	1496	__FUNCTION__, this->process->pid, this->trdid );
	1497	#endif
[1]	1498	error = 0;
	1499	}
[611]	1500	else if( vseg->min == addr_min ) // vseg must be resized
[1]	1501	{
[623]	1502
	1503	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1504	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1505	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1506	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1507	#endif
	1508	// update vseg min address
[406]	1509	vseg->min = addr_max;
	1510
	1511	// update vpn_base and vpn_size
	1512	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1513	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1514	vseg->vpn_base = vpn_min;
	1515	vseg->vpn_size = vpn_max - vpn_min + 1;
[623]	1516
	1517	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1518	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1519	printk("\n[%s] thread[%x,%x] changed vseg_min\n",
	1520	__FUNCTION__, this->process->pid, this->trdid );
	1521	#endif
[406]	1522	error = 0;
[1]	1523	}
[611]	1524	else if( vseg->max == addr_max ) // vseg must be resized
[1]	1525	{
[623]	1526
	1527	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1528	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1529	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1530	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1531	#endif
[406]	1532	// update vseg max address
	1533	vseg->max = addr_min;
	1534
	1535	// update vpn_base and vpn_size
	1536	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1537	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1538	vseg->vpn_base = vpn_min;
	1539	vseg->vpn_size = vpn_max - vpn_min + 1;
[623]	1540
	1541	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1542	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1543	printk("\n[%s] thread[%x,%x] changed vseg_max\n",
	1544	__FUNCTION__, this->process->pid, this->trdid );
	1545	#endif
[406]	1546	error = 0;
[623]	1547
[1]	1548	}
[611]	1549	else // vseg cut in three regions
[1]	1550	{
[623]	1551
	1552	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1553	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1554	printk("\n[%s] unmapped region[%x->%x[ included in vseg[%x->%x[\n",
	1555	__FUNCTION__, addr_min, addr_max, vseg->min, vseg->max );
	1556	#endif
[406]	1557	// resize existing vseg
	1558	vseg->max = addr_min;
	1559
	1560	// update vpn_base and vpn_size
	1561	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
	1562	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
	1563	vseg->vpn_base = vpn_min;
	1564	vseg->vpn_size = vpn_max - vpn_min + 1;
	1565
	1566	// create new vseg
[407]	1567	new = vmm_create_vseg( process,
	1568	vseg->type,
	1569	addr_min,
	1570	(vseg->max - addr_max),
	1571	vseg->file_offset,
	1572	vseg->file_size,
	1573	vseg->mapper_xp,
	1574	vseg->cxy );
	1575
[623]	1576	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
	1577	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1578	printk("\n[%s] thread[%x,%x] replaced vseg by two smal vsegs\n",
	1579	__FUNCTION__, this->process->pid, this->trdid );
	1580	#endif
	1581
	1582	if( new == NULL ) error = -1;
[406]	1583	else error = 0;
[1]	1584	}
	1585
[623]	1586	#if DEBUG_VMM_RESIZE_VSEG
	1587	if( DEBUG_VMM_RESIZE_VSEG < cycle )
	1588	printk("\n[%s] thread[%x,%x] exit / process %x / base %x / size %d / cycle %d\n",
	1589	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, size, cycle );
	1590	#endif
[1]	1591
	1592	return error;
	1593
[406]	1594	} // vmm_resize_vseg()
	1595
[1]	1596	///////////////////////////////////////////
[388]	1597	error_t vmm_get_vseg( process_t * process,
[394]	1598	intptr_t vaddr,
[388]	1599	vseg_t ** found_vseg )
[1]	1600	{
[595]	1601	xptr_t vseg_xp;
	1602	vseg_t * vseg;
	1603	vmm_t * vmm;
	1604	error_t error;
[1]	1605
[440]	1606	// get pointer on local VMM
	1607	vmm = &process->vmm;
[1]	1608
[440]	1609	// try to get vseg from local VMM
[595]	1610	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
[440]	1611
[388]	1612	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
	1613	{
	1614	// get extended pointer on reference process
	1615	xptr_t ref_xp = process->ref_xp;
[1]	1616
[388]	1617	// get cluster and local pointer on reference process
	1618	cxy_t ref_cxy = GET_CXY( ref_xp );
[433]	1619	process_t * ref_ptr = GET_PTR( ref_xp );
[388]	1620
	1621	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
	1622
	1623	// get extended pointer on reference vseg
[394]	1624	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
[388]	1625
[440]	1626	if( error ) return -1; // vseg not found => illegal user vaddr
[388]	1627
	1628	// allocate a vseg in local cluster
	1629	vseg = vseg_alloc();
	1630
[440]	1631	if( vseg == NULL ) return -1; // cannot allocate a local vseg
[388]	1632
	1633	// initialise local vseg from reference
	1634	vseg_init_from_ref( vseg , vseg_xp );
	1635
[625]	1636	// build extended pointer on VSL lock
	1637	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
	1638
	1639	// take the VSL lock in write mode
	1640	remote_rwlock_wr_acquire( lock_xp );
	1641
[611]	1642	// register local vseg in local VSL
	1643	vmm_attach_vseg_to_vsl( vmm , vseg );
[625]	1644
	1645	// release the VSL lock
	1646	remote_rwlock_wr_release( lock_xp );
[388]	1647	}
[595]	1648
[388]	1649	// success
	1650	*found_vseg = vseg;
[394]	1651	return 0;
[388]	1652
	1653	} // end vmm_get_vseg()
	1654
[407]	1655	//////////////////////////////////////////////////////////////////////////////////////
	1656	// This static function compute the target cluster to allocate a physical page
[632]	1657	// for a given <vpn> in a given <vseg>, allocates the page and returns an extended
	1658	// pointer on the allocated page descriptor.
[407]	1659	// The vseg cannot have the FILE type.
	1660	//////////////////////////////////////////////////////////////////////////////////////
	1661	static xptr_t vmm_page_allocate( vseg_t * vseg,
	1662	vpn_t vpn )
	1663	{
[433]	1664
[632]	1665	#if DEBUG_VMM_PAGE_ALLOCATE
[619]	1666	uint32_t cycle = (uint32_t)hal_get_cycles();
	1667	thread_t * this = CURRENT_THREAD;
[632]	1668	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
[595]	1669	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
	1670	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
[433]	1671	#endif
	1672
[632]	1673	xptr_t page_xp;
[407]	1674	cxy_t page_cxy;
[635]	1675	page_t * page_ptr;
[577]	1676	uint32_t index;
[407]	1677
[577]	1678	uint32_t type = vseg->type;
	1679	uint32_t flags = vseg->flags;
	1680	uint32_t x_size = LOCAL_CLUSTER->x_size;
	1681	uint32_t y_size = LOCAL_CLUSTER->y_size;
[407]	1682
[567]	1683	// check vseg type
	1684	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
[407]	1685
	1686	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
	1687	{
[577]	1688	index = vpn & ((x_size * y_size) - 1);
	1689	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
[561]	1690
[577]	1691	// If the cluster selected from VPN's LSBs is empty, we select one randomly
	1692	if ( cluster_is_active( page_cxy ) == false )
	1693	{
	1694	page_cxy = cluster_random_select();
[561]	1695	}
[407]	1696	}
	1697	else // other cases => cxy specified in vseg
	1698	{
[561]	1699	page_cxy = vseg->cxy;
[407]	1700	}
	1701
[635]	1702	// allocate one small physical page from target cluster
	1703	page_ptr = ppm_remote_alloc_pages( page_cxy , 0 );
[407]	1704
[635]	1705	page_xp = XPTR( page_cxy , page_ptr );
	1706
[632]	1707	#if DEBUG_VMM_PAGE_ALLOCATE
[595]	1708	cycle = (uint32_t)hal_get_cycles();
[632]	1709	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
[635]	1710	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
	1711	__FUNCTION__ , this->process->pid, this->trdid, vpn, ppm_page2ppn(page_xp), cycle );
[433]	1712	#endif
	1713
[632]	1714	return page_xp;
[407]	1715
	1716	} // end vmm_page_allocate()
	1717
[313]	1718	////////////////////////////////////////
	1719	error_t vmm_get_one_ppn( vseg_t * vseg,
	1720	vpn_t vpn,
	1721	ppn_t * ppn )
	1722	{
	1723	error_t error;
[407]	1724	xptr_t page_xp; // extended pointer on physical page descriptor
[606]	1725	uint32_t page_id; // missing page index in vseg mapper
[406]	1726	uint32_t type; // vseg type;
[313]	1727
[406]	1728	type = vseg->type;
[606]	1729	page_id = vpn - vseg->vpn_base;
[313]	1730
[438]	1731	#if DEBUG_VMM_GET_ONE_PPN
[595]	1732	uint32_t cycle = (uint32_t)hal_get_cycles();
	1733	thread_t * this = CURRENT_THREAD;
[635]	1734	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
[606]	1735	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
	1736	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
[433]	1737	#endif
[313]	1738
[406]	1739	// FILE type : get the physical page from the file mapper
[313]	1740	if( type == VSEG_TYPE_FILE )
	1741	{
[406]	1742	// get extended pointer on mapper
[407]	1743	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1744
[567]	1745	assert( (mapper_xp != XPTR_NULL),
	1746	"mapper not defined for a FILE vseg\n" );
[406]	1747
[606]	1748	// get extended pointer on page descriptor
	1749	page_xp = mapper_remote_get_page( mapper_xp , page_id );
[406]	1750
[606]	1751	if ( page_xp == XPTR_NULL ) return EINVAL;
[313]	1752	}
	1753
[406]	1754	// Other types : allocate a physical page from target cluster,
[407]	1755	// as defined by vseg type and vpn value
[313]	1756	else
	1757	{
[433]	1758	// allocate one physical page
[407]	1759	page_xp = vmm_page_allocate( vseg , vpn );
[406]	1760
[635]	1761	if( page_xp == XPTR_NULL ) return -1;
[313]	1762
[406]	1763	// initialise missing page from .elf file mapper for DATA and CODE types
[440]	1764	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
[313]	1765	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
	1766	{
[406]	1767	// get extended pointer on mapper
	1768	xptr_t mapper_xp = vseg->mapper_xp;
[313]	1769
[567]	1770	assert( (mapper_xp != XPTR_NULL),
	1771	"mapper not defined for a CODE or DATA vseg\n" );
[406]	1772
	1773	// compute missing page offset in vseg
[606]	1774	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
[406]	1775
[313]	1776	// compute missing page offset in .elf file
[406]	1777	uint32_t elf_offset = vseg->file_offset + offset;
[313]	1778
[438]	1779	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[635]	1780	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
[595]	1781	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
	1782	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
[433]	1783	#endif
[406]	1784	// compute extended pointer on page base
[407]	1785	xptr_t base_xp = ppm_page2base( page_xp );
[313]	1786
[406]	1787	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
	1788	uint32_t file_size = vseg->file_size;
	1789
	1790	if( file_size < offset ) // missing page fully in BSS
[313]	1791	{
[406]	1792
[438]	1793	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[635]	1794	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
[595]	1795	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
	1796	__FUNCTION__, this->process->pid, this->trdid, vpn );
[433]	1797	#endif
[407]	1798	if( GET_CXY( page_xp ) == local_cxy )
[313]	1799	{
[315]	1800	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1801	}
	1802	else
	1803	{
[315]	1804	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
[313]	1805	}
	1806	}
[406]	1807	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
[315]	1808	{
[406]	1809
[438]	1810	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[635]	1811	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
[595]	1812	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
	1813	__FUNCTION__, this->process->pid, this->trdid, vpn );
[433]	1814	#endif
[606]	1815	error = mapper_move_kernel( mapper_xp,
	1816	true, // to_buffer
	1817	elf_offset,
	1818	base_xp,
	1819	CONFIG_PPM_PAGE_SIZE );
[313]	1820	if( error ) return EINVAL;
	1821	}
[406]	1822	else // both in mapper and in BSS :
	1823	// - (file_size - offset) bytes from mapper
	1824	// - (page_size + offset - file_size) bytes from BSS
[313]	1825	{
[406]	1826
[438]	1827	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
[635]	1828	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
[610]	1829	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
[433]	1830	" %d bytes from mapper / %d bytes from BSS\n",
[595]	1831	__FUNCTION__, this->process->pid, this->trdid, vpn,
[407]	1832	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
[433]	1833	#endif
[313]	1834	// initialize mapper part
[606]	1835	error = mapper_move_kernel( mapper_xp,
	1836	true, // to buffer
	1837	elf_offset,
	1838	base_xp,
	1839	file_size - offset );
[313]	1840	if( error ) return EINVAL;
	1841
	1842	// initialize BSS part
[407]	1843	if( GET_CXY( page_xp ) == local_cxy )
[313]	1844	{
[406]	1845	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
	1846	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1847	}
	1848	else
	1849	{
[406]	1850	hal_remote_memset( base_xp + file_size - offset , 0 ,
	1851	offset + CONFIG_PPM_PAGE_SIZE - file_size );
[313]	1852	}
	1853	}
	1854	} // end initialisation for CODE or DATA types
	1855	}
	1856
	1857	// return ppn
[407]	1858	*ppn = ppm_page2ppn( page_xp );
[406]	1859
[438]	1860	#if DEBUG_VMM_GET_ONE_PPN
[595]	1861	cycle = (uint32_t)hal_get_cycles();
[635]	1862	if( (DEBUG_VMM_GET_ONE_PPN < cycle) && (vpn == 0x40b) )
	1863	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
[595]	1864	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
[433]	1865	#endif
[406]	1866
[313]	1867	return 0;
	1868
	1869	} // end vmm_get_one_ppn()
	1870
[585]	1871	///////////////////////////////////////////////////
	1872	error_t vmm_handle_page_fault( process_t * process,
	1873	vpn_t vpn )
[1]	1874	{
[585]	1875	vseg_t * vseg; // vseg containing vpn
[629]	1876	uint32_t attr; // PTE_ATTR value
	1877	ppn_t ppn; // PTE_PPN value
[585]	1878	uint32_t ref_attr; // PTE_ATTR value in reference GPT
	1879	ppn_t ref_ppn; // PTE_PPN value in reference GPT
	1880	cxy_t ref_cxy; // reference cluster for missing vpn
	1881	process_t * ref_ptr; // reference process for missing vpn
	1882	xptr_t local_gpt_xp; // extended pointer on local GPT
	1883	xptr_t ref_gpt_xp; // extended pointer on reference GPT
	1884	error_t error; // value returned by called functions
[1]	1885
[629]	1886	thread_t * this = CURRENT_THREAD;
	1887
	1888	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	1889	uint32_t start_cycle = (uint32_t)hal_get_cycles();
	1890	#endif
	1891
[625]	1892	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[635]	1893	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[625]	1894	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
[629]	1895	__FUNCTION__, this->process->pid, this->trdid, vpn, start_cycle );
[625]	1896	#endif
	1897
[629]	1898	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
[635]	1899	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
	1900	hal_vmm_display( this->process , true );
[629]	1901	#endif
	1902
[585]	1903	// get local vseg (access to reference VSL can be required)
	1904	error = vmm_get_vseg( process,
	1905	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	1906	&vseg );
	1907	if( error )
	1908	{
[629]	1909	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in registered vseg\n",
	1910	__FUNCTION__ , vpn , process->pid, this->trdid );
[585]	1911
	1912	return EXCP_USER_ERROR;
	1913	}
	1914
[635]	1915	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	1916	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[634]	1917	printk("\n[%s] thread[%x,%x] found vseg %s\n",
	1918	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
[433]	1919	#endif
[406]	1920
[629]	1921	// build extended pointer on local GPT
	1922	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
	1923
[632]	1924	// lock PTE in local GPT and get current PPN and attributes
[629]	1925	error = hal_gpt_lock_pte( local_gpt_xp,
	1926	vpn,
	1927	&attr,
	1928	&ppn );
	1929	if( error )
[438]	1930	{
[629]	1931	printk("\n[PANIC] in %s : cannot lock PTE in local GPT / vpn %x / process %x\n",
	1932	__FUNCTION__ , vpn , process->pid );
	1933
	1934	return EXCP_KERNEL_PANIC;
	1935	}
[407]	1936
[635]	1937	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	1938	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
	1939	printk("\n[%s] thread[%x,%x] locked vpn %x in cluster %x\n",
[634]	1940	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy );
[632]	1941	#endif
	1942
	1943	// handle page fault only if local PTE still unmapped after lock
[629]	1944	if( (attr & GPT_MAPPED) == 0 )
	1945	{
	1946	// get reference process cluster and local pointer
	1947	ref_cxy = GET_CXY( process->ref_xp );
	1948	ref_ptr = GET_PTR( process->ref_xp );
[407]	1949
[630]	1950	/////////////// private vseg or (local == reference)
	1951	/////////////// => access only the local GPT
[629]	1952	if( (vseg->type == VSEG_TYPE_STACK) \|\|
	1953	(vseg->type == VSEG_TYPE_CODE) \|\|
	1954	(ref_cxy == local_cxy ) )
	1955	{
[632]	1956
[635]	1957	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	1958	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
	1959	printk("\n[%s] thread[%x,%x] access local gpt : cxy %x / ref_cxy %x / type %s / cycle %d\n",
	1960	__FUNCTION__, this->process->pid, this->trdid,
	1961	local_cxy, ref_cxy, vseg_type_str(vseg->type), (uint32_t)hal_get_cycles() );
[632]	1962	#endif
	1963	// allocate and initialise a physical page
[629]	1964	error = vmm_get_one_ppn( vseg , vpn , &ppn );
[407]	1965
[585]	1966	if( error )
[408]	1967	{
[629]	1968	printk("\n[ERROR] in %s : no physical page / process = %x / vpn = %x\n",
[408]	1969	__FUNCTION__ , process->pid , vpn );
[1]	1970
[629]	1971	// unlock PTE in local GPT
	1972	hal_gpt_unlock_pte( local_gpt_xp , vpn );
[406]	1973
[585]	1974	return EXCP_KERNEL_PANIC;
[407]	1975	}
	1976
[629]	1977	// define attr from vseg flags
[632]	1978	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
[629]	1979	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
	1980	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
	1981	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
	1982	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
[407]	1983
[629]	1984	// set PTE to local GPT
[632]	1985	// it unlocks this PTE
[629]	1986	hal_gpt_set_pte( local_gpt_xp,
	1987	vpn,
	1988	attr,
	1989	ppn );
[585]	1990
[629]	1991	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	1992	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	1993	#endif
[585]	1994
	1995	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[635]	1996	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	1997	printk("\n[%s] thread[%x,%x] handled local pgfault / ppn %x / attr %x / cycle %d\n",
	1998	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
[585]	1999	#endif
	2000
[629]	2001	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2002	this->info.local_pgfault_nr++;
	2003	this->info.local_pgfault_cost += (end_cycle - start_cycle);
	2004	#endif
	2005	return EXCP_NON_FATAL;
[585]	2006
[629]	2007	} // end local GPT access
[585]	2008
[630]	2009	/////////////////// public vseg and (local != reference)
	2010	/////////////////// => access ref GPT to update local GPT
[629]	2011	else
	2012	{
[632]	2013
[635]	2014	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	2015	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
	2016	printk("\n[%s] thread[%x,%x] access ref gpt : cxy %x / ref_cxy %x / type %s / cycle %d\n",
	2017	__FUNCTION__, this->process->pid, this->trdid,
	2018	local_cxy, ref_cxy, vseg_type_str(vseg->type), (uint32_t)hal_get_cycles() );
[632]	2019	#endif
[629]	2020	// build extended pointer on reference GPT
	2021	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
[585]	2022
[632]	2023	// lock PTE in reference GPT and get current PPN and attributes
	2024	error = hal_gpt_lock_pte( ref_gpt_xp,
	2025	vpn,
	2026	&ref_attr,
	2027	&ref_ppn );
	2028	if( error )
	2029	{
	2030	printk("\n[PANIC] in %s : cannot lock PTE in ref GPT / vpn %x / process %x\n",
	2031	__FUNCTION__ , vpn , process->pid );
	2032
	2033	// unlock PTE in local GPT
	2034	hal_gpt_unlock_pte( local_gpt_xp , vpn );
	2035
	2036	return EXCP_KERNEL_PANIC;
	2037	}
[1]	2038
[635]	2039	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	2040	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2041	printk("\n[%s] thread[%x,%x] get pte from ref gpt / attr %x / ppn %x\n",
	2042	__FUNCTION__, this->process->pid, this->trdid, ref_attr, ref_ppn );
	2043	#endif
	2044
	2045	if( ref_attr & GPT_MAPPED ) // false page fault
[585]	2046	{
[629]	2047	// update local GPT from reference GPT values
[632]	2048	// this unlocks the PTE in local GPT
[629]	2049	hal_gpt_set_pte( local_gpt_xp,
	2050	vpn,
	2051	ref_attr,
	2052	ref_ppn );
[585]	2053
[635]	2054	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	2055	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2056	printk("\n[%s] thread[%x,%x] updated local gpt for a false pgfault\n",
	2057	__FUNCTION__, this->process->pid, this->trdid );
	2058	#endif
	2059
	2060	// unlock the PTE in reference GPT
	2061	hal_gpt_unlock_pte( ref_gpt_xp, vpn );
	2062
[635]	2063	#if (DEBUG_VMM_HANDLE_PAGE_FAULT &1)
	2064	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2065	printk("\n[%s] thread[%x,%x] unlock the ref gpt after a false pgfault\n",
	2066	__FUNCTION__, this->process->pid, this->trdid );
	2067	#endif
	2068
[629]	2069	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2070	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2071	#endif
	2072
[585]	2073	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[635]	2074	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2075	printk("\n[%s] thread[%x,%x] handled false pgfault / ppn %x / attr %x / cycle %d\n",
	2076	__FUNCTION__, this->process->pid, this->trdid, ref_ppn, ref_attr, end_cycle );
[433]	2077	#endif
[406]	2078
[629]	2079	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2080	this->info.false_pgfault_nr++;
	2081	this->info.false_pgfault_cost += (end_cycle - start_cycle);
	2082	#endif
	2083	return EXCP_NON_FATAL;
	2084	}
[632]	2085	else // true page fault
[629]	2086	{
[585]	2087	// allocate and initialise a physical page depending on the vseg type
[629]	2088	error = vmm_get_one_ppn( vseg , vpn , &ppn );
[1]	2089
[585]	2090	if( error )
	2091	{
	2092	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
	2093	__FUNCTION__ , process->pid , vpn );
[313]	2094
[632]	2095	// unlock PTE in local GPT and in reference GPT
[629]	2096	hal_gpt_unlock_pte( local_gpt_xp , vpn );
[632]	2097	hal_gpt_unlock_pte( ref_gpt_xp , vpn );
[585]	2098
[629]	2099	return EXCP_KERNEL_PANIC;
[585]	2100	}
[1]	2101
[629]	2102	// define attr from vseg flags
[632]	2103	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
[629]	2104	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
	2105	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
	2106	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
	2107	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
[585]	2108
[635]	2109	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	2110	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2111	printk("\n[%s] thread[%x,%x] build a new PTE for a true pgfault\n",
	2112	__FUNCTION__, this->process->pid, this->trdid );
	2113	#endif
[629]	2114	// set PTE in reference GPT
[632]	2115	// this unlock the PTE
[629]	2116	hal_gpt_set_pte( ref_gpt_xp,
	2117	vpn,
	2118	attr,
	2119	ppn );
	2120
[635]	2121	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
	2122	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2123	printk("\n[%s] thread[%x,%x] set new PTE in ref gpt for a true page fault\n",
	2124	__FUNCTION__, this->process->pid, this->trdid );
	2125	#endif
	2126
[629]	2127	// set PTE in local GPT
[632]	2128	// this unlock the PTE
[629]	2129	hal_gpt_set_pte( local_gpt_xp,
	2130	vpn,
	2131	attr,
	2132	ppn );
	2133
	2134	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2135	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2136	#endif
	2137
[440]	2138	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[635]	2139	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2140	printk("\n[%s] thread[%x,%x] handled global pgfault / ppn %x / attr %x / cycle %d\n",
	2141	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
[435]	2142	#endif
[629]	2143
	2144	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2145	this->info.global_pgfault_nr++;
	2146	this->info.global_pgfault_cost += (end_cycle - start_cycle);
	2147	#endif
	2148	return EXCP_NON_FATAL;
	2149	}
[585]	2150	}
	2151	}
[629]	2152	else // page has been locally mapped by another concurrent thread
	2153	{
[632]	2154	// unlock the PTE in local GPT
[629]	2155	hal_gpt_unlock_pte( local_gpt_xp , vpn );
	2156
[632]	2157	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
	2158	uint32_t end_cycle = (uint32_t)hal_get_cycles();
	2159	#endif
	2160
	2161	#if DEBUG_VMM_HANDLE_PAGE_FAULT
[635]	2162	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
[632]	2163	printk("\n[%s] handled by another thread / vpn %x / ppn %x / attr %x / cycle %d\n",
	2164	__FUNCTION__, vpn, ppn, attr, end_cycle );
	2165	#endif
	2166
	2167	#if CONFIG_INSTRUMENTATION_PGFAULTS
	2168	this->info.false_pgfault_nr++;
	2169	this->info.false_pgfault_cost += (end_cycle - start_cycle);
	2170	#endif
[629]	2171	return EXCP_NON_FATAL;
	2172	}
	2173
[585]	2174	} // end vmm_handle_page_fault()
[435]	2175
[585]	2176	////////////////////////////////////////////
	2177	error_t vmm_handle_cow( process_t * process,
	2178	vpn_t vpn )
	2179	{
	2180	vseg_t * vseg; // vseg containing vpn
[629]	2181	xptr_t gpt_xp; // extended pointer on GPT (local or reference)
	2182	gpt_t * gpt_ptr; // local pointer on GPT (local or reference)
	2183	cxy_t gpt_cxy; // GPT cluster identifier
[585]	2184	uint32_t old_attr; // current PTE_ATTR value
	2185	ppn_t old_ppn; // current PTE_PPN value
	2186	uint32_t new_attr; // new PTE_ATTR value
	2187	ppn_t new_ppn; // new PTE_PPN value
[629]	2188	cxy_t ref_cxy; // reference process cluster
	2189	process_t * ref_ptr; // local pointer on reference process
[585]	2190	error_t error;
[1]	2191
[629]	2192	thread_t * this = CURRENT_THREAD;
[625]	2193
[585]	2194	#if DEBUG_VMM_HANDLE_COW
[629]	2195	uint32_t cycle = (uint32_t)hal_get_cycles();
[585]	2196	if( DEBUG_VMM_HANDLE_COW < cycle )
[595]	2197	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
[619]	2198	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
[629]	2199	#endif
	2200
[635]	2201	#if ((DEBUG_VMM_HANDLE_COW & 3) == 3 )
[625]	2202	hal_vmm_display( process , true );
[585]	2203	#endif
	2204
	2205	// get local vseg
	2206	error = vmm_get_vseg( process,
	2207	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
	2208	&vseg );
[440]	2209	if( error )
[1]	2210	{
[629]	2211	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in a registered vseg\n",
[625]	2212	__FUNCTION__, vpn, process->pid, this->trdid );
[585]	2213
[629]	2214	return EXCP_USER_ERROR;
[440]	2215	}
[407]	2216
[629]	2217	#if DEBUG_VMM_HANDLE_COW
[619]	2218	if( DEBUG_VMM_HANDLE_COW < cycle )
[629]	2219	printk("\n[%s] thread[%x,%x] get vseg %s\n",
	2220	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
[619]	2221	#endif
	2222
[629]	2223	// get reference process cluster and local pointer
[585]	2224	ref_cxy = GET_CXY( process->ref_xp );
	2225	ref_ptr = GET_PTR( process->ref_xp );
[407]	2226
[629]	2227	// build pointers on relevant GPT
	2228	// - access only local GPT for a private vseg
	2229	// - access reference GPT and all copies for a public vseg
[585]	2230	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[440]	2231	{
[629]	2232	gpt_cxy = local_cxy;
	2233	gpt_ptr = &process->vmm.gpt;
	2234	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
[1]	2235	}
[440]	2236	else
[1]	2237	{
[629]	2238	gpt_cxy = ref_cxy;
	2239	gpt_ptr = &ref_ptr->vmm.gpt;
	2240	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
[1]	2241	}
	2242
[629]	2243	// lock target PTE in relevant GPT (local or reference)
[632]	2244	// and get current PTE value
[629]	2245	error = hal_gpt_lock_pte( gpt_xp,
	2246	vpn,
	2247	&old_attr,
	2248	&old_ppn );
	2249	if( error )
	2250	{
	2251	printk("\n[PANIC] in %s : cannot lock PTE in GPT / cxy %x / vpn %x / process %x\n",
	2252	__FUNCTION__ , gpt_cxy, vpn , process->pid );
	2253
	2254	return EXCP_KERNEL_PANIC;
	2255	}
[441]	2256
[629]	2257	#if DEBUG_VMM_HANDLE_COW
[619]	2258	if( DEBUG_VMM_HANDLE_COW < cycle )
	2259	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
	2260	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
	2261	#endif
	2262
[629]	2263	// return user error if COW attribute not set or PTE2 unmapped
	2264	if( ((old_attr & GPT_COW) == 0) \|\| ((old_attr & GPT_MAPPED) == 0) )
[585]	2265	{
[629]	2266	hal_gpt_unlock_pte( gpt_xp , vpn );
[407]	2267
[629]	2268	return EXCP_USER_ERROR;
[407]	2269	}
	2270
[619]	2271	// get pointers on physical page descriptor
[585]	2272	xptr_t page_xp = ppm_ppn2page( old_ppn );
	2273	cxy_t page_cxy = GET_CXY( page_xp );
	2274	page_t * page_ptr = GET_PTR( page_xp );
[435]	2275
[585]	2276	// get extended pointers on forks and lock field in page descriptor
	2277	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
	2278	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
[407]	2279
[585]	2280	// take lock protecting "forks" counter
	2281	remote_busylock_acquire( forks_lock_xp );
[407]	2282
[585]	2283	// get number of pending forks from page descriptor
	2284	uint32_t forks = hal_remote_l32( forks_xp );
[441]	2285
[629]	2286	#if DEBUG_VMM_HANDLE_COW
[619]	2287	if( DEBUG_VMM_HANDLE_COW < cycle )
	2288	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
	2289	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
	2290	#endif
	2291
[585]	2292	if( forks ) // pending fork => allocate a new page, and copy old to new
	2293	{
[619]	2294	// decrement pending forks counter in page descriptor
	2295	hal_remote_atomic_add( forks_xp , -1 );
	2296
	2297	// release lock protecting "forks" counter
	2298	remote_busylock_release( forks_lock_xp );
	2299
[629]	2300	// allocate a new physical page depending on vseg type
[585]	2301	page_xp = vmm_page_allocate( vseg , vpn );
[619]	2302
[585]	2303	if( page_xp == XPTR_NULL )
	2304	{
	2305	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
	2306	__FUNCTION__ , vpn, process->pid );
[441]	2307
[629]	2308	hal_gpt_unlock_pte( gpt_xp , vpn );
[441]	2309
[585]	2310	return EXCP_KERNEL_PANIC;
	2311	}
[441]	2312
[585]	2313	// compute allocated page PPN
	2314	new_ppn = ppm_page2ppn( page_xp );
[441]	2315
[629]	2316	#if DEBUG_VMM_HANDLE_COW
[619]	2317	if( DEBUG_VMM_HANDLE_COW < cycle )
	2318	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
	2319	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
	2320	#endif
	2321
[585]	2322	// copy old page content to new page
[619]	2323	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
	2324	ppm_ppn2base( old_ppn ),
	2325	CONFIG_PPM_PAGE_SIZE );
[441]	2326
[629]	2327	#if DEBUG_VMM_HANDLE_COW
[585]	2328	if( DEBUG_VMM_HANDLE_COW < cycle )
[619]	2329	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
	2330	__FUNCTION__, this->process->pid, this->trdid );
[585]	2331	#endif
[440]	2332
[585]	2333	}
	2334	else // no pending fork => keep the existing page
	2335	{
[619]	2336	// release lock protecting "forks" counter
	2337	remote_busylock_release( forks_lock_xp );
[1]	2338
[585]	2339	#if(DEBUG_VMM_HANDLE_COW & 1)
	2340	if( DEBUG_VMM_HANDLE_COW < cycle )
[635]	2341	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
[619]	2342	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
[585]	2343	#endif
	2344	new_ppn = old_ppn;
	2345	}
[1]	2346
[629]	2347	// build new_attr : set WRITABLE, reset COW, reset LOCKED
	2348	new_attr = (((old_attr \| GPT_WRITABLE) & (~GPT_COW)) & (~GPT_LOCKED));
[585]	2349
[635]	2350	#if(DEBUG_VMM_HANDLE_COW & 1)
	2351	if( DEBUG_VMM_HANDLE_COW < cycle )
	2352	printk("\n[%s] thread[%x,%x] new_attr %x / new_ppn %x\n",
	2353	__FUNCTION__, this->process->pid, this->trdid, new_attr, new_ppn );
	2354	#endif
	2355
[629]	2356	// update the relevant GPT(s)
	2357	// - private vseg => update only the local GPT
	2358	// - public vseg => update the reference GPT AND all the GPT copies
[585]	2359	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
[1]	2360	{
[635]	2361	// set new PTE in local gpt
[585]	2362	hal_gpt_set_pte( gpt_xp,
	2363	vpn,
	2364	new_attr,
	2365	new_ppn );
[1]	2366	}
[585]	2367	else
[1]	2368	{
[585]	2369	if( ref_cxy == local_cxy ) // reference cluster is local
	2370	{
	2371	vmm_global_update_pte( process,
	2372	vpn,
	2373	new_attr,
	2374	new_ppn );
	2375	}
	2376	else // reference cluster is remote
	2377	{
	2378	rpc_vmm_global_update_pte_client( ref_cxy,
	2379	ref_ptr,
	2380	vpn,
	2381	new_attr,
	2382	new_ppn );
	2383	}
[1]	2384	}
	2385
[585]	2386	#if DEBUG_VMM_HANDLE_COW
	2387	cycle = (uint32_t)hal_get_cycles();
	2388	if( DEBUG_VMM_HANDLE_COW < cycle )
[595]	2389	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
[619]	2390	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
[585]	2391	#endif
[313]	2392
[635]	2393	#if ((DEBUG_VMM_HANDLE_COW & 3) == 3)
	2394	hal_vmm_display( process , true );
	2395	#endif
	2396
[585]	2397	return EXCP_NON_FATAL;
[1]	2398
[585]	2399	} // end vmm_handle_cow()
	2400

Note: See TracBrowser for help on using the repository browser.

Download in other formats: