Context Navigation

source: trunk/kernel/mm/vmm.c @ 659

Last change on this file since 659 was 657, checked in by alain, 5 years ago
Introduce remote_buf.c/.h & socket.c/.h files. Update dev_nic.c/.h files.
File size: 102.3 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations definition.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Alain Greiner (2016,2017,2018,2019,2020)
6	*
7	* Copyright (c) UPMC Sorbonne Universites
8	*
9	* This file is part of ALMOS-MKH.
10	*
11	* ALMOS-MKH is free software; you can redistribute it and/or modify it
12	* under the terms of the GNU General Public License as published by
13	* the Free Software Foundation; version 2.0 of the License.
14	*
15	* ALMOS-MKH is distributed in the hope that it will be useful, but
16	* WITHOUT ANY WARRANTY; without even the implied warranty of
17	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18	* General Public License for more details.
19	*
20	* You should have received a copy of the GNU General Public License
21	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
22	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23	*/
24
25	#include <kernel_config.h>
26	#include <hal_kernel_types.h>
27	#include <hal_special.h>
28	#include <hal_gpt.h>
29	#include <hal_vmm.h>
30	#include <hal_irqmask.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_queuelock.h>
35	#include <list.h>
36	#include <xlist.h>
37	#include <bits.h>
38	#include <process.h>
39	#include <thread.h>
40	#include <vseg.h>
41	#include <cluster.h>
42	#include <scheduler.h>
43	#include <vfs.h>
44	#include <mapper.h>
45	#include <page.h>
46	#include <kmem.h>
47	#include <vmm.h>
48	#include <hal_exception.h>
49
50	////////////////////////////////////////////////////////////////////////////////////////////
51	// Extern global variables
52	////////////////////////////////////////////////////////////////////////////////////////////
53
54	extern process_t process_zero; // allocated in cluster.c
55
56	////////////////////////////////////////////////////////////////////////////////////////////
57	// This static function is called by the vmm_user_init() function.
58	// It initialises the free lists of vsegs used by the VMM MMAP allocator.
59	// It makes the assumption that HEAP_BASE == 1 Gbytes and HEAP_SIZE == 2 Gbytes.
60	////////////////////////////////////////////////////////////////////////////////////////////
61	static void vmm_stack_init( vmm_t * vmm )
62	{
63
64	// check STACK zone
65	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
66	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) , "STACK zone too small\n");
67
68	// get pointer on STACK allocator
69	stack_mgr_t * mgr = &vmm->stack_mgr;
70
71	mgr->bitmap = 0;
72	mgr->vpn_base = CONFIG_VMM_STACK_BASE;
73	busylock_init( &mgr->lock , LOCK_VMM_STACK );
74
75	}
76
77	////////////////////////////////////////////////////////////////////////////////////////////
78	// This static function is called by the vmm_create_vseg() function, and implements
79	// the VMM STACK specific allocator. Depending on the local thread index <ltid>,
80	// it ckeks availability of the corresponding slot in the process STACKS region,
81	// allocates a vseg descriptor, and initializes the "vpn_base" and "vpn_size" fields.
82	////////////////////////////////////////////////////////////////////////////////////////////
83	// @ vmm : [in] pointer on VMM.
84	// @ ltid : [in] requested slot == local user thread identifier.
85	////////////////////////////////////////////////////////////////////////////////////////////
86	static vseg_t * vmm_stack_alloc( vmm_t * vmm,
87	ltid_t ltid )
88	{
89
90	// check ltid argument
91	assert( (ltid <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
92	"slot index %d too large for an user stack vseg", ltid );
93
94	// get stack allocator pointer
95	stack_mgr_t * mgr = &vmm->stack_mgr;
96
97	// get lock protecting stack allocator
98	busylock_acquire( &mgr->lock );
99
100	// check requested slot is available
101	assert( (bitmap_state( &mgr->bitmap , ltid ) == false),
102	"slot index %d already allocated", ltid );
103
104	// allocate a vseg descriptor
105	vseg_t * vseg = vseg_alloc();
106
107	if( vseg == NULL )
108	{
109	// release lock protecting free lists
110	busylock_release( &mgr->lock );
111
112	printk("\n[ERROR] %s cannot allocate memory for vseg in cluster %x\n",
113	__FUNCTION__ , local_cxy );
114
115	return NULL;
116	}
117
118	// update bitmap
119	bitmap_set( &mgr->bitmap , ltid );
120
121	// release lock on stack allocator
122	busylock_release( &mgr->lock );
123
124	// set "vpn_base" & "vpn_size" fields (first page non allocated)
125	vseg->vpn_base = mgr->vpn_base + (ltid * CONFIG_VMM_STACK_SIZE) + 1;
126	vseg->vpn_size = CONFIG_VMM_STACK_SIZE - 1;
127
128	return vseg;
129
130	} // end vmm_stack_alloc()
131
132	////////////////////////////////////////////////////////////////////////////////////////////
133	// This static function is called by the vmm_remove_vseg() function, and implements
134	// the VMM STACK specific desallocator.
135	// It updates the bitmap to release the corresponding slot in the process STACKS region,
136	// and releases memory allocated to vseg descriptor.
137	////////////////////////////////////////////////////////////////////////////////////////////
138	// @ vmm : [in] pointer on VMM.
139	// @ vseg : [in] pointer on released vseg.
140	////////////////////////////////////////////////////////////////////////////////////////////
141	static void vmm_stack_free( vmm_t * vmm,
142	vseg_t * vseg )
143	{
144	// get stack allocator pointer
145	stack_mgr_t * mgr = &vmm->stack_mgr;
146
147	// compute slot index
148	uint32_t index = (vseg->vpn_base - 1 - mgr->vpn_base) / CONFIG_VMM_STACK_SIZE;
149
150	// check index
151	assert( (index <= ((CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE) / CONFIG_VMM_STACK_SIZE)),
152	"slot index %d too large for an user stack vseg", index );
153
154	// check released slot is allocated
155	assert( (bitmap_state( &mgr->bitmap , index ) == true),
156	"released slot index %d non allocated", index );
157
158	// get lock on stack allocator
159	busylock_acquire( &mgr->lock );
160
161	// update stacks_bitmap
162	bitmap_clear( &mgr->bitmap , index );
163
164	// release lock on stack allocator
165	busylock_release( &mgr->lock );
166
167	// release memory allocated to vseg descriptor
168	vseg_free( vseg );
169
170	} // end vmm_stack_free()
171
172
173
174	////////////////////////////////////////////////////////////////////////////////////////////
175	// This function display the current state of the VMM MMAP allocator of a process VMM
176	// identified by the <vmm> argument.
177	////////////////////////////////////////////////////////////////////////////////////////////
178	void vmm_mmap_display( vmm_t * vmm )
179	{
180	uint32_t order;
181	xptr_t root_xp;
182	xptr_t iter_xp;
183
184	// get pointer on process
185	process_t * process = (process_t )(((char)vmm) - OFFSETOF( process_t , vmm ));
186
187	// get process PID
188	pid_t pid = process->pid;
189
190	// get pointer on VMM MMAP allocator
191	mmap_mgr_t * mgr = &vmm->mmap_mgr;
192
193	// display header
194	printk("*** VMM MMAP allocator / process %x ***\n", pid );
195
196	// scan the array of free lists of vsegs
197	for( order = 0 ; order <= CONFIG_VMM_HEAP_MAX_ORDER ; order++ )
198	{
199	root_xp = XPTR( local_cxy , &mgr->free_list_root[order] );
200
201	if( !xlist_is_empty( root_xp ) )
202	{
203	printk(" - %d (%x pages) : ", order , 1<<order );
204
205	XLIST_FOREACH( root_xp , iter_xp )
206	{
207	xptr_t vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
208	vseg_t * vseg = GET_PTR( vseg_xp );
209
210	printk("%x \| ", vseg->vpn_base );
211	}
212
213	printk("\n");
214	}
215	}
216	} // end vmm_mmap_display()
217
218	////////////////////////////////////////////////////////////////////////////////////////////
219	// This static function is called by the vmm_user_init() function.
220	// It initialises the free lists of vsegs used by the VMM MMAP allocator.
221	// TODO this function is only valid for 32 bits cores, and makes three assumptions:
222	// HEAP_BASE == 1 Gbytes / HEAP_SIZE == 2 Gbytes / MMAP_MAX_SIZE == 1 Gbytes
223	////////////////////////////////////////////////////////////////////////////////////////////
224	void vmm_mmap_init( vmm_t * vmm )
225	{
226
227	// check HEAP base and size
228	assert( (CONFIG_VMM_HEAP_BASE == 0x40000) & (CONFIG_VMM_STACK_BASE == 0xc0000),
229	"CONFIG_VMM_HEAP_BASE != 0x40000 or CONFIG_VMM_STACK_BASE != 0xc0000" );
230
231	// check MMAP vseg max order
232	assert( (CONFIG_VMM_HEAP_MAX_ORDER == 18), "max mmap vseg size is 256K pages" );
233
234	// get pointer on MMAP allocator
235	mmap_mgr_t * mgr = &vmm->mmap_mgr;
236
237	// initialize HEAP base and size
238	mgr->vpn_base = CONFIG_VMM_HEAP_BASE;
239	mgr->vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
240
241	// initialize lock
242	busylock_init( &mgr->lock , LOCK_VMM_MMAP );
243
244	// initialize free lists
245	uint32_t i;
246	for( i = 0 ; i <= CONFIG_VMM_HEAP_MAX_ORDER ; i++ )
247	{
248	xlist_root_init( XPTR( local_cxy , &mgr->free_list_root[i] ) );
249	}
250
251	// allocate and register first 1 Gbytes vseg
252	vseg_t * vseg0 = vseg_alloc();
253
254	assert( (vseg0 != NULL) , "cannot allocate vseg" );
255
256	vseg0->vpn_base = CONFIG_VMM_HEAP_BASE;
257	vseg0->vpn_size = CONFIG_VMM_HEAP_BASE;
258
259	xlist_add_first( XPTR( local_cxy , &mgr->free_list_root[CONFIG_VMM_HEAP_MAX_ORDER] ),
260	XPTR( local_cxy , &vseg0->xlist ) );
261
262	// allocate and register second 1 Gbytes vseg
263	vseg_t * vseg1 = vseg_alloc();
264
265	assert( (vseg1 != NULL) , "cannot allocate vseg" );
266
267	vseg1->vpn_base = CONFIG_VMM_HEAP_BASE << 1;
268	vseg1->vpn_size = CONFIG_VMM_HEAP_BASE;
269
270	xlist_add_first( XPTR( local_cxy , &mgr->free_list_root[CONFIG_VMM_HEAP_MAX_ORDER] ),
271	XPTR( local_cxy , &vseg1->xlist ) );
272
273	#if DEBUG_VMM_MMAP
274	thread_t * this = CURRENT_THREAD;
275	uint32_t cycle = (uint32_t)hal_get_cycles();
276	printk("\n[%s] thread[%x,%x] / cycle %d\n",
277	__FUNCTION__, this->process->pid, this->trdid, cycle );
278	vmm_mmap_display( vmm );
279	#endif
280
281	} // end vmm_mmap_init()
282
283	////////////////////////////////////////////////////////////////////////////////////////////
284	// This static function is called by the vmm_create_vseg() function, and implements
285	// the VMM MMAP specific allocator. Depending on the requested number of pages <npages>,
286	// it get a free vseg from the relevant free_list, and initializes the "vpn_base" and
287	// "vpn_size" fields.
288	////////////////////////////////////////////////////////////////////////////////////////////
289	// @ vmm : [in] pointer on VMM.
290	// @ npages : [in] requested number of pages.
291	// @ returns local pointer on vseg if success / returns NULL if failure.
292	////////////////////////////////////////////////////////////////////////////////////////////
293	static vseg_t * vmm_mmap_alloc( vmm_t * vmm,
294	vpn_t npages )
295	{
296
297	#if DEBUG_VMM_MMAP
298	thread_t * this = CURRENT_THREAD;
299	uint32_t cycle = (uint32_t)hal_get_cycles();
300	if( DEBUG_VMM_MMAP < cycle )
301	printk("\n[%s] thread[%x,%x] for %x pages / cycle %d\n",
302	__FUNCTION__, this->process->pid, this->trdid, npages, cycle );
303	#endif
304
305	// number of allocated pages must be power of 2
306	// compute actual size and order
307	vpn_t required_vpn_size = POW2_ROUNDUP( npages );
308	uint32_t required_order = bits_log2( required_vpn_size );
309
310	// get mmap allocator pointer
311	mmap_mgr_t * mgr = &vmm->mmap_mgr;
312
313	// take lock protecting free lists in MMAP allocator
314	busylock_acquire( &mgr->lock );
315
316	// initialises the while loop variables
317	uint32_t current_order = required_order;
318	vseg_t * current_vseg = NULL;
319
320	// search a free vseg equal or larger than requested size
321	while( current_order <= CONFIG_VMM_HEAP_MAX_ORDER )
322	{
323	// build extended pointer on free_pages_root[current_order]
324	xptr_t root_xp = XPTR( local_cxy , &mgr->free_list_root[current_order] );
325
326	if( !xlist_is_empty( root_xp ) )
327	{
328	// get extended pointer on first vseg in this free_list
329	xptr_t current_vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
330	current_vseg = GET_PTR( current_vseg_xp );
331
332	// build extended pointer on xlist field in vseg descriptor
333	xptr_t list_entry_xp = XPTR( local_cxy , &current_vseg->xlist );
334
335	// remove this vseg from the free_list
336	xlist_unlink( list_entry_xp );
337
338	break;
339	}
340
341	// increment loop index
342	current_order++;
343
344	} // end while loop
345
346	if( current_vseg == NULL ) // return failure
347	{
348	// release lock protecting free lists
349	busylock_release( &mgr->lock );
350
351	printk("\n[ERROR] %s cannot allocate ) %d page(s) in cluster %x\n",
352	__FUNCTION__, npages , local_cxy );
353
354	return NULL;
355	}
356
357	// split recursively the found vseg in smaller vsegs
358	// if required, and update the free-lists accordingly
359	while( current_order > required_order )
360	{
361	// get found vseg base and size
362	vpn_t vpn_base = current_vseg->vpn_base;
363	vpn_t vpn_size = current_vseg->vpn_size;
364
365	// allocate a new vseg for the upper half of current vseg
366	vseg_t * new_vseg = vseg_alloc();
367
368	if( new_vseg == NULL )
369	{
370	// release lock protecting free lists
371	busylock_release( &mgr->lock );
372
373	printk("\n[ERROR] %s cannot allocate memory for vseg in cluster %x\n",
374	__FUNCTION__ , local_cxy );
375
376	return NULL;
377	}
378
379	// initialise new vseg (upper half of found vseg)
380	new_vseg->vmm = vmm;
381	new_vseg->vpn_base = vpn_base + (vpn_size >> 1);
382	new_vseg->vpn_size = vpn_size >> 1;
383
384	// insert new vseg in relevant free_list
385	xlist_add_first( XPTR( local_cxy , &mgr->free_list_root[current_order-1] ),
386	XPTR( local_cxy , &new_vseg->xlist ) );
387
388	// update found vseg
389	current_vseg->vpn_size = vpn_size>>1;
390
391	// update order
392	current_order --;
393	}
394
395	// release lock protecting free lists
396	busylock_release( &mgr->lock );
397
398	#if DEBUG_VMM_MMAP
399	vmm_mmap_display( vmm );
400	#endif
401
402	return current_vseg;
403
404	} // end vmm_mmap_alloc()
405
406	////////////////////////////////////////////////////////////////////////////////////////////
407	// This static function implements the VMM MMAP specific desallocator.
408	// It is called by the vmm_remove_vseg() function.
409	// It releases the vseg to the relevant free_list, after trying (recursively) to
410	// merge it to the buddy vseg.
411	////////////////////////////////////////////////////////////////////////////////////////////
412	// @ vmm : [in] pointer on VMM.
413	// @ vseg : [in] pointer on released vseg.
414	////////////////////////////////////////////////////////////////////////////////////////////
415	static void vmm_mmap_free( vmm_t * vmm,
416	vseg_t * vseg )
417	{
418
419	#if DEBUG_VMM_MMAP
420	thread_t * this = CURRENT_THREAD;
421	uint32_t cycle = (uint32_t)hal_get_cycles();
422	if( DEBUG_VMM_MMAP < cycle )
423	printk("\n[%s] thread[%x,%x] for vpn_base %x / vpn_size %x / cycle %d\n",
424	__FUNCTION__, this->process->pid, this->trdid, vseg->vpn_base, vseg->vpn_size, cycle );
425	#endif
426
427	vseg_t * buddy_vseg;
428
429	// get mmap allocator pointer
430	mmap_mgr_t * mgr = &vmm->mmap_mgr;
431
432	// take lock protecting free lists
433	busylock_acquire( &mgr->lock );
434
435	// initialise loop variables
436	// released_vseg is the currently released vseg
437	vseg_t * released_vseg = vseg;
438	uint32_t released_order = bits_log2( vseg->vpn_size );
439
440	// iteratively merge the released vseg to the buddy vseg
441	// release the current page and exit when buddy not found
442	while( released_order <= CONFIG_VMM_HEAP_MAX_ORDER )
443	{
444	// compute buddy_vseg vpn_base
445	vpn_t buddy_vpn_base = released_vseg->vpn_base ^ (1 << released_order);
446
447	// build extended pointer on free_pages_root[current_order]
448	xptr_t root_xp = XPTR( local_cxy , &mgr->free_list_root[released_order] );
449
450	// scan this free list to find the buddy vseg
451	xptr_t iter_xp;
452	buddy_vseg = NULL;
453	XLIST_FOREACH( root_xp , iter_xp )
454	{
455	xptr_t current_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
456	vseg_t * current_vseg = GET_PTR( current_vseg_xp );
457
458	if( current_vseg->vpn_base == buddy_vpn_base )
459	{
460	buddy_vseg = current_vseg;
461	break;
462	}
463	}
464
465	if( buddy_vseg != NULL ) // buddy found => merge released & buddy
466	{
467	// update released vseg fields
468	released_vseg->vpn_size = buddy_vseg->vpn_size<<1;
469	if( released_vseg->vpn_base > buddy_vseg->vpn_base)
470	released_vseg->vpn_base = buddy_vseg->vpn_base;
471
472	// remove buddy vseg from free_list
473	xlist_unlink( XPTR( local_cxy , &buddy_vseg->xlist ) );
474
475	// release memory allocated to buddy descriptor
476	vseg_free( buddy_vseg );
477	}
478	else // buddy not found => register & exit
479	{
480	// register released vseg in free list
481	xlist_add_first( root_xp , XPTR( local_cxy , &released_vseg->xlist ) );
482
483	// exit while loop
484	break;
485	}
486
487	// increment released_order
488	released_order++;
489	}
490
491	// release lock
492	busylock_release( &mgr->lock );
493
494	#if DEBUG_VMM_MMAP
495	vmm_mmap_display( vmm );
496	#endif
497
498	} // end vmm_mmap_free()
499
500	////////////////////////////////////////////////////////////////////////////////////////////
501	// This static function registers one vseg in the VSL of a local process descriptor.
502	////////////////////////////////////////////////////////////////////////////////////////////
503	// vmm : [in] pointer on VMM.
504	// vseg : [in] pointer on vseg.
505	////////////////////////////////////////////////////////////////////////////////////////////
506	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
507	vseg_t * vseg )
508	{
509	// update vseg descriptor
510	vseg->vmm = vmm;
511
512	// increment vsegs number
513	vmm->vsegs_nr++;
514
515	// add vseg in vmm list
516	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
517	XPTR( local_cxy , &vseg->xlist ) );
518
519	} // end vmm_attach_vseg_from_vsl()
520
521	////////////////////////////////////////////////////////////////////////////////////////////
522	// This static function removes one vseg from the VSL of a local process descriptor.
523	////////////////////////////////////////////////////////////////////////////////////////////
524	// vmm : [in] pointer on VMM.
525	// vseg : [in] pointer on vseg.
526	////////////////////////////////////////////////////////////////////////////////////////////
527	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
528	vseg_t * vseg )
529	{
530	// update vseg descriptor
531	vseg->vmm = NULL;
532
533	// decrement vsegs number
534	vmm->vsegs_nr--;
535
536	// remove vseg from VSL
537	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
538
539	} // end vmm_detach_from_vsl()
540
541	////////////////////////////////////////////
542	error_t vmm_user_init( process_t * process )
543	{
544
545	#if DEBUG_VMM_USER_INIT
546	thread_t * this = CURRENT_THREAD;
547	uint32_t cycle = (uint32_t)hal_get_cycles();
548	if( DEBUG_VMM_USER_INIT )
549	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
550	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
551	#endif
552
553	// get pointer on VMM
554	vmm_t * vmm = &process->vmm;
555
556	// check UTILS zone
557	assert( ((CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE) <=
558	(CONFIG_VMM_ELF_BASE - CONFIG_VMM_UTILS_BASE)) ,
559	"UTILS zone too small\n" );
560
561	// initialize lock protecting the VSL
562	remote_queuelock_init( XPTR( local_cxy , &vmm->vsl_lock ) , LOCK_VMM_VSL );
563
564
565	// initialize STACK allocator
566	vmm_stack_init( vmm );
567
568	// initialize MMAP allocator
569	vmm_mmap_init( vmm );
570
571	// initialize instrumentation counters
572	vmm->false_pgfault_nr = 0;
573	vmm->local_pgfault_nr = 0;
574	vmm->global_pgfault_nr = 0;
575	vmm->false_pgfault_cost = 0;
576	vmm->local_pgfault_cost = 0;
577	vmm->global_pgfault_cost = 0;
578
579	/*
580	// register "args" vseg in VSL
581	base = CONFIG_VMM_UTILS_BASE << CONFIG_PPM_PAGE_SHIFT;
582	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
583
584	vseg_args = vmm_create_vseg( process,
585	VSEG_TYPE_DATA,
586	base,
587	size,
588	0, // file_offset unused
589	0, // file_size unused
590	XPTR_NULL, // mapper_xp unused
591	local_cxy );
592	if( vseg_args == NULL )
593	{
594	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
595	return -1;
596	}
597
598	vmm->args_vpn_base = base;
599
600	// register "envs" vseg in VSL
601	base = (CONFIG_VMM_UTILS_BASE + CONFIG_VMM_ARGS_SIZE) << CONFIG_PPM_PAGE_SHIFT;
602	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
603
604	vseg_envs = vmm_create_vseg( process,
605	VSEG_TYPE_DATA,
606	base,
607	size,
608	0, // file_offset unused
609	0, // file_size unused
610	XPTR_NULL, // mapper_xp unused
611	local_cxy );
612	if( vseg_envs == NULL )
613	{
614	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
615	return -1;
616	}
617
618	vmm->envs_vpn_base = base;
619	*/
620	hal_fence();
621
622	#if DEBUG_VMM_USER_INIT
623	cycle = (uint32_t)hal_get_cycles();
624	if( DEBUG_VMM_USER_INIT )
625	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
626	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
627	#endif
628
629	return 0;
630
631	} // end vmm_user_init()
632
633	//////////////////////////////////////////
634	void vmm_user_reset( process_t * process )
635	{
636	xptr_t vseg_xp;
637	vseg_t * vseg;
638	vseg_type_t vseg_type;
639
640	#if DEBUG_VMM_USER_RESET
641	uint32_t cycle;
642	thread_t * this = CURRENT_THREAD;
643	#endif
644
645	#if (DEBUG_VMM_USER_RESET & 1 )
646	cycle = (uint32_t)hal_get_cycles();
647	if( DEBUG_VMM_USER_RESET < cycle )
648	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
649	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
650	#endif
651
652	#if (DEBUG_VMM_USER_RESET & 1 )
653	if( DEBUG_VMM_USER_RESET < cycle )
654	hal_vmm_display( XPTR( local_cxy , process ) , true );
655	#endif
656
657	// get pointer on local VMM
658	vmm_t * vmm = &process->vmm;
659
660	// build extended pointer on VSL root and VSL lock
661	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
662	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
663
664	// take the VSL lock
665	remote_queuelock_acquire( lock_xp );
666
667	// scan the VSL to delete all non kernel vsegs
668	// (we don't use a FOREACH in case of item deletion)
669	xptr_t iter_xp;
670	xptr_t next_xp;
671	for( iter_xp = hal_remote_l64( root_xp ) ;
672	iter_xp != root_xp ;
673	iter_xp = next_xp )
674	{
675	// save extended pointer on next item in xlist
676	next_xp = hal_remote_l64( iter_xp );
677
678	// get pointers on current vseg in VSL
679	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
680	vseg = GET_PTR( vseg_xp );
681	vseg_type = vseg->type;
682
683	#if( DEBUG_VMM_USER_RESET & 1 )
684	if( DEBUG_VMM_USER_RESET < cycle )
685	printk("\n[%s] found %s vseg / vpn_base %x / vpn_size %d\n",
686	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
687	#endif
688	// delete non kernel vseg
689	if( (vseg_type != VSEG_TYPE_KCODE) &&
690	(vseg_type != VSEG_TYPE_KDATA) &&
691	(vseg_type != VSEG_TYPE_KDEV ) )
692	{
693	// remove vseg from VSL
694	vmm_remove_vseg( process , vseg );
695
696	#if( DEBUG_VMM_USER_RESET & 1 )
697	if( DEBUG_VMM_USER_RESET < cycle )
698	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
699	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
700	#endif
701	}
702	else
703	{
704
705	#if( DEBUG_VMM_USER_RESET & 1 )
706	if( DEBUG_VMM_USER_RESET < cycle )
707	printk("\n[%s] keep %s vseg / vpn_base %x / vpn_size %d\n",
708	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
709	#endif
710	}
711	} // end loop on vsegs in VSL
712
713	// release the VSL lock
714	remote_queuelock_release( lock_xp );
715
716	// FIXME il faut gérer les process copies...
717
718	#if DEBUG_VMM_USER_RESET
719	cycle = (uint32_t)hal_get_cycles();
720	if( DEBUG_VMM_USER_RESET < cycle )
721	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
722	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
723	#endif
724
725	#if (DEBUG_VMM_USER_RESET & 1 )
726	if( DEBUG_VMM_USER_RESET < cycle )
727	hal_vmm_display( XPTR( local_cxy , process ) , true );
728	#endif
729
730	} // end vmm_user_reset()
731
732	/////////////////////////////////////////////////
733	void vmm_global_delete_vseg( process_t * process,
734	intptr_t base )
735	{
736	cxy_t owner_cxy;
737	lpid_t owner_lpid;
738	reg_t save_sr;
739
740	xptr_t process_lock_xp;
741	xptr_t process_root_xp;
742	xptr_t process_iter_xp;
743
744	xptr_t remote_process_xp;
745	cxy_t remote_process_cxy;
746	process_t * remote_process_ptr;
747
748	xptr_t vsl_root_xp;
749	xptr_t vsl_lock_xp;
750	xptr_t vsl_iter_xp;
751
752	rpc_desc_t rpc; // shared rpc descriptor for parallel RPCs
753	uint32_t responses; // RPC responses counter
754
755	thread_t * this = CURRENT_THREAD;
756	pid_t pid = process->pid;
757	cluster_t * cluster = LOCAL_CLUSTER;
758
759	#if DEBUG_VMM_GLOBAL_DELETE_VSEG
760	uint32_t cycle = (uint32_t)hal_get_cycles();
761	#endif
762
763	#if (DEBUG_VMM_GLOBAL_DELETE_VSEG & 1)
764	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
765	printk("\n[%s] thread[%x,%x] enters / process %x / base %x / cycle %d\n",
766	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, cycle );
767	#endif
768
769	// initialize a shared RPC descriptor
770	rpc.rsp = &responses;
771	rpc.blocking = false; // non blocking behaviour for rpc_send()
772	rpc.index = RPC_VMM_REMOVE_VSEG;
773	rpc.thread = this;
774	rpc.lid = this->core->lid;
775	rpc.args[0] = this->process->pid;
776	rpc.args[1] = base;
777
778	// get owner process cluster and local index
779	owner_cxy = CXY_FROM_PID( pid );
780	owner_lpid = LPID_FROM_PID( pid );
781
782	// get extended pointer on root and lock of process copies xlist in owner cluster
783	process_root_xp = XPTR( owner_cxy , &cluster->pmgr.copies_root[owner_lpid] );
784	process_lock_xp = XPTR( owner_cxy , &cluster->pmgr.copies_lock[owner_lpid] );
785
786	// mask IRQs
787	hal_disable_irq( &save_sr );
788
789	// client thread blocks itself
790	thread_block( XPTR( local_cxy , this ) , THREAD_BLOCKED_RPC );
791
792	// take the lock protecting process copies
793	remote_queuelock_acquire( process_lock_xp );
794
795	// initialize responses counter
796	responses = 0;
797
798	// loop on process copies
799	XLIST_FOREACH( process_root_xp , process_iter_xp )
800	{
801	// get cluster and local pointer on remote process
802	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
803	remote_process_ptr = GET_PTR( remote_process_xp );
804	remote_process_cxy = GET_CXY( remote_process_xp );
805
806	// build extended pointers on remote VSL root and lock
807	vsl_root_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsegs_root );
808	vsl_lock_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsl_lock );
809
810	// get lock on remote VSL
811	remote_queuelock_acquire( vsl_lock_xp );
812
813	// loop on vsegs in remote process VSL
814	XLIST_FOREACH( vsl_root_xp , vsl_iter_xp )
815	{
816	// get pointers on current vseg
817	xptr_t vseg_xp = XLIST_ELEMENT( vsl_iter_xp , vseg_t , xlist );
818	vseg_t * vseg_ptr = GET_PTR( vseg_xp );
819
820	// get current vseg base address
821	intptr_t vseg_base = (intptr_t)hal_remote_lpt( XPTR( remote_process_cxy,
822	&vseg_ptr->min ) );
823
824	if( vseg_base == base ) // found searched vseg
825	{
826	// atomically increment responses counter
827	hal_atomic_add( &responses , 1 );
828
829	#if (DEBUG_VMM_GLOBAL_DELETE_VSEG & 1)
830	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
831	printk("\n[%s] thread[%x,%x] register RPC request in cluster %x\n",
832	__FUNCTION__, this->process->pid, this->trdid, remote_process_cxy );
833	#endif
834	// send RPC to remote cluster
835	rpc_send( remote_process_cxy , &rpc );
836
837	// exit loop on vsegs
838	break;
839	}
840	} // end of loop on vsegs
841
842	// release lock on remote VSL
843	remote_queuelock_release( vsl_lock_xp );
844
845	} // end of loop on process copies
846
847	// release the lock protecting process copies
848	remote_queuelock_release( process_lock_xp );
849
850	#if (DEBUG_VMM_GLOBAL_DELETE_VSEG & 1)
851	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
852	printk("\n[%s] thread[%x,%x] deschedule / process %x / base %x\n",
853	__FUNCTION__, this->process->pid, this->trdid, process->pid, base );
854	#endif
855
856	// client thread deschedule
857	sched_yield("blocked on rpc_vmm_delete_vseg");
858
859	// restore IRQs
860	hal_restore_irq( save_sr );
861
862	#if DEBUG_VMM_GLOBAL_DELETE_VSEG
863	cycle = (uint32_t)hal_get_cycles();
864	if( DEBUG_VMM_GLOBAL_DELETE_VSEG < cycle )
865	printk("\n[%s] thread[%x,%x] exit / process %x / base %x / cycle %d\n",
866	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, cycle );
867	#endif
868
869	} // end vmm_global_delete_vseg()
870
871	////////////////////////////////////////////////
872	void vmm_global_resize_vseg( process_t * process,
873	intptr_t base,
874	intptr_t new_base,
875	intptr_t new_size )
876	{
877	cxy_t owner_cxy;
878	lpid_t owner_lpid;
879	reg_t save_sr;
880
881	xptr_t process_lock_xp;
882	xptr_t process_root_xp;
883	xptr_t process_iter_xp;
884
885	xptr_t remote_process_xp;
886	cxy_t remote_process_cxy;
887	process_t * remote_process_ptr;
888
889	xptr_t vsl_root_xp;
890	xptr_t vsl_lock_xp;
891	xptr_t vsl_iter_xp;
892
893	rpc_desc_t rpc; // shared rpc descriptor for parallel RPCs
894	uint32_t responses; // RPC responses counter
895
896	thread_t * this = CURRENT_THREAD;
897	pid_t pid = process->pid;
898	cluster_t * cluster = LOCAL_CLUSTER;
899
900	#if DEBUG_VMM_GLOBAL_RESIZE_VSEG
901	uint32_t cycle = (uint32_t)hal_get_cycles();
902	#endif
903
904	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
905	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
906	printk("\n[%s] thread[%x,%x] : process %x / base %x / new_base %x / new_size %x / cycle %d\n",
907	__FUNCTION__, this->process->pid, this->trdid, process->pid, base, new_base, new_size, cycle );
908	#endif
909
910	// initialize a shared RPC descriptor
911	rpc.rsp = &responses;
912	rpc.blocking = false; // non blocking behaviour for rpc_send()
913	rpc.index = RPC_VMM_REMOVE_VSEG;
914	rpc.thread = this;
915	rpc.lid = this->core->lid;
916	rpc.args[0] = this->process->pid;
917	rpc.args[1] = base;
918	rpc.args[2] = new_base;
919	rpc.args[3] = new_size;
920
921	// get owner process cluster and local index
922	owner_cxy = CXY_FROM_PID( pid );
923	owner_lpid = LPID_FROM_PID( pid );
924
925	// get extended pointer on root and lock of process copies xlist in owner cluster
926	process_root_xp = XPTR( owner_cxy , &cluster->pmgr.copies_root[owner_lpid] );
927	process_lock_xp = XPTR( owner_cxy , &cluster->pmgr.copies_lock[owner_lpid] );
928
929	// mask IRQs
930	hal_disable_irq( &save_sr );
931
932	// client thread blocks itself
933	thread_block( XPTR( local_cxy , this ) , THREAD_BLOCKED_RPC );
934
935	// take the lock protecting process copies
936	remote_queuelock_acquire( process_lock_xp );
937
938	// initialize responses counter
939	responses = 0;
940
941	// loop on process copies
942	XLIST_FOREACH( process_root_xp , process_iter_xp )
943	{
944	// get cluster and local pointer on remote process
945	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
946	remote_process_ptr = GET_PTR( remote_process_xp );
947	remote_process_cxy = GET_CXY( remote_process_xp );
948
949	// build extended pointers on remote VSL root and lock
950	vsl_root_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsegs_root );
951	vsl_lock_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.vsl_lock );
952
953	// get lock on remote VSL
954	remote_queuelock_acquire( vsl_lock_xp );
955
956	// loop on vsegs in remote process VSL
957	XLIST_FOREACH( vsl_root_xp , vsl_iter_xp )
958	{
959	// get pointers on current vseg
960	xptr_t vseg_xp = XLIST_ELEMENT( vsl_iter_xp , vseg_t , xlist );
961	vseg_t * vseg_ptr = GET_PTR( vseg_xp );
962
963	// get current vseg base address
964	intptr_t vseg_base = (intptr_t)hal_remote_lpt( XPTR( remote_process_cxy,
965	&vseg_ptr->min ) );
966
967	if( vseg_base == base ) // found searched vseg
968	{
969	// atomically increment responses counter
970	hal_atomic_add( &responses , 1 );
971
972	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
973	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
974	printk("\n[%s] thread[%x,%x] register RPC request in cluster %x\n",
975	__FUNCTION__, this->process->pid, this->trdid, remote_process_cxy );
976	#endif
977	// send RPC to remote cluster
978	rpc_send( remote_process_cxy , & rpc );
979
980	// exit loop on vsegs
981	break;
982	}
983
984	} // end of loop on vsegs
985
986	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
987	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
988	hal_vmm_display( remote_process_xp , false );
989	#endif
990
991	// release lock on remote VSL
992	remote_queuelock_release( vsl_lock_xp );
993
994	} // end of loop on process copies
995
996	// release the lock protecting process copies
997	remote_queuelock_release( process_lock_xp );
998
999	#if (DEBUG_VMM_GLOBAL_RESIZE_VSEG & 1)
1000	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
1001	printk("\n[%s] thread[%x,%x] deschedule / process %x / base %x\n",
1002	__FUNCTION__, this->process->pid, this->trdid, process->pid, base );
1003	#endif
1004
1005	// client thread deschedule
1006	sched_yield("blocked on rpc_vmm_delete_vseg");
1007
1008	// restore IRQs
1009	hal_restore_irq( save_sr );
1010
1011	#if DEBUG_VMM_GLOBAL_RESIZE_VSEG
1012	cycle = (uint32_t)hal_get_cycles();
1013	if( DEBUG_VMM_GLOBAL_RESIZE_VSEG < cycle )
1014	printk("\n[%s] thread[%x,%x] exit for process %x / base %x / cycle %d\n",
1015	__FUNCTION__, this->process->pid, this->trdid, process->pid , base, cycle );
1016	#endif
1017
1018	} // end vmm_global_resize_vseg()
1019
1020	////////////////////////////////////////////////
1021	void vmm_global_update_pte( process_t * process,
1022	vpn_t vpn,
1023	uint32_t attr,
1024	ppn_t ppn )
1025	{
1026	pid_t pid;
1027	cxy_t owner_cxy;
1028	lpid_t owner_lpid;
1029
1030	xlist_entry_t * process_root_ptr;
1031	xptr_t process_root_xp;
1032	xptr_t process_iter_xp;
1033
1034	xptr_t remote_process_xp;
1035	cxy_t remote_process_cxy;
1036	process_t * remote_process_ptr;
1037	xptr_t remote_gpt_xp;
1038
1039	#if DEBUG_VMM_GLOBAL_UPDATE_PTE
1040	uint32_t cycle = (uint32_t)hal_get_cycles();
1041	thread_t * this = CURRENT_THREAD;
1042	#endif
1043
1044
1045	#if (DEBUG_VMM_GLOBAL_UPDATE_PTE & 1)
1046	if( DEBUG_VMM_GLOBAL_UPDATE_PTE < cycle )
1047	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / attr %x / ppn %x / ycle %d\n",
1048	__FUNCTION__, this->process->pid, this->trdid, process->pid, vpn, attr, ppn, cycle );
1049	#endif
1050
1051	// get owner process cluster and local index
1052	pid = process->pid;
1053	owner_cxy = CXY_FROM_PID( pid );
1054	owner_lpid = LPID_FROM_PID( pid );
1055
1056	// get extended pointer on root of process copies xlist in owner cluster
1057	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
1058	process_root_xp = XPTR( owner_cxy , process_root_ptr );
1059
1060	// loop on process copies
1061	XLIST_FOREACH( process_root_xp , process_iter_xp )
1062	{
1063	// get cluster and local pointer on remote process
1064	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
1065	remote_process_ptr = GET_PTR( remote_process_xp );
1066	remote_process_cxy = GET_CXY( remote_process_xp );
1067
1068	#if (DEBUG_VMM_GLOBAL_UPDATE_PTE & 1)
1069	if( DEBUG_VMM_GLOBAL_UPDATE_PTE < cycle )
1070	printk("\n[%s] thread[%x,%x] handling vpn %x for process %x in cluster %x\n",
1071	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
1072	#endif
1073
1074	// get extended pointer on remote gpt
1075	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
1076
1077	// update remote GPT
1078	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
1079	}
1080
1081	#if DEBUG_VMM_GLOBAL_UPDATE_PTE
1082	cycle = (uint32_t)hal_get_cycles();
1083	if( DEBUG_VMM_GLOBAL_UPDATE_PTE < cycle )
1084	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
1085	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
1086	#endif
1087
1088	#if (DEBUG_VMM_GLOBAL_UPDATE_PTE & 1)
1089	hal_vmm_display( process , true );
1090	#endif
1091
1092	} // end vmm_global_update_pte()
1093
1094	///////////////////////////////////////
1095	void vmm_set_cow( process_t * process )
1096	{
1097	vmm_t * vmm;
1098
1099	xlist_entry_t * process_root_ptr;
1100	xptr_t process_root_xp;
1101	xptr_t process_iter_xp;
1102
1103	xptr_t remote_process_xp;
1104	cxy_t remote_process_cxy;
1105	process_t * remote_process_ptr;
1106	xptr_t remote_gpt_xp;
1107
1108	xptr_t vseg_root_xp;
1109	xptr_t vseg_iter_xp;
1110
1111	xptr_t vseg_xp;
1112	vseg_t * vseg;
1113
1114	pid_t pid;
1115	cxy_t owner_cxy;
1116	lpid_t owner_lpid;
1117
1118	// get target process PID
1119	pid = process->pid;
1120
1121	#if DEBUG_VMM_SET_COW
1122	uint32_t cycle = (uint32_t)hal_get_cycles();
1123	thread_t * this = CURRENT_THREAD;
1124	if( DEBUG_VMM_SET_COW < cycle )
1125	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
1126	__FUNCTION__, this->process->pid, this->trdid, pid , cycle );
1127	#endif
1128
1129	#if (DEBUG_VMM_SET_COW & 1)
1130	if( DEBUG_VMM_SET_COW < cycle )
1131	hal_vmm_display( process , true );
1132	#endif
1133
1134	// check cluster is reference
1135	assert( (XPTR( local_cxy , process ) == process->ref_xp),
1136	"local cluster must be process reference cluster\n");
1137
1138	// get pointer on reference VMM
1139	vmm = &process->vmm;
1140
1141	// get extended pointer on root of process copies xlist in owner cluster
1142	owner_cxy = CXY_FROM_PID( pid );
1143	owner_lpid = LPID_FROM_PID( pid );
1144	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
1145	process_root_xp = XPTR( owner_cxy , process_root_ptr );
1146
1147	// get extended pointer on root of vsegs xlist from reference VMM
1148	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1149
1150	// loop on target process copies
1151	XLIST_FOREACH( process_root_xp , process_iter_xp )
1152	{
1153	// get cluster and local pointer on remote process copy
1154	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
1155	remote_process_ptr = GET_PTR( remote_process_xp );
1156	remote_process_cxy = GET_CXY( remote_process_xp );
1157
1158	#if (DEBUG_VMM_SET_COW & 1)
1159	if( DEBUG_VMM_SET_COW < cycle )
1160	printk("\n[%s] thread[%x,%x] (%x) handles process %x in cluster %x\n",
1161	__FUNCTION__, this->process->pid, this->trdid, this, pid, remote_process_cxy );
1162	#endif
1163
1164	// get extended pointer on remote gpt
1165	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
1166
1167	// loop on vsegs in (local) reference process VSL
1168	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
1169	{
1170	// get pointer on vseg
1171	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
1172	vseg = GET_PTR( vseg_xp );
1173
1174	// get vseg type, base and size
1175	uint32_t type = vseg->type;
1176	vpn_t vpn_base = vseg->vpn_base;
1177	vpn_t vpn_size = vseg->vpn_size;
1178
1179	#if (DEBUG_VMM_SET_COW & 1)
1180	if( DEBUG_VMM_SET_COW < cycle )
1181	printk("\n[%s] thread[%x,%x] found vseg %s / vpn_base = %x / vpn_size = %x\n",
1182	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
1183	#endif
1184	// only DATA, ANON and REMOTE vsegs
1185	if( (type == VSEG_TYPE_DATA) \|\|
1186	(type == VSEG_TYPE_ANON) \|\|
1187	(type == VSEG_TYPE_REMOTE) )
1188	{
1189	vpn_t vpn;
1190	uint32_t attr;
1191	ppn_t ppn;
1192	xptr_t page_xp;
1193	cxy_t page_cxy;
1194	page_t * page_ptr;
1195	xptr_t forks_xp;
1196	xptr_t lock_xp;
1197
1198	// update flags in remote GPT
1199	hal_gpt_set_cow( remote_gpt_xp,
1200	vpn_base,
1201	vpn_size );
1202
1203	// atomically increment pending forks counter in physical pages,
1204	// this is only done once, when handling the reference copy
1205	if( remote_process_cxy == local_cxy )
1206	{
1207
1208	#if (DEBUG_VMM_SET_COW & 1)
1209	if( DEBUG_VMM_SET_COW < cycle )
1210	printk("\n[%s] thread[%x,%x] handles vseg %s / vpn_base = %x / vpn_size = %x\n",
1211	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
1212	#endif
1213	// scan all pages in vseg
1214	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
1215	{
1216	// get page attributes and PPN from reference GPT
1217	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
1218
1219	// atomically update pending forks counter if page is mapped
1220	if( attr & GPT_MAPPED )
1221	{
1222	// get pointers and cluster on page descriptor
1223	page_xp = ppm_ppn2page( ppn );
1224	page_cxy = GET_CXY( page_xp );
1225	page_ptr = GET_PTR( page_xp );
1226
1227	// get extended pointers on "forks" and "lock"
1228	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1229	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1230
1231	// take lock protecting "forks" counter
1232	remote_busylock_acquire( lock_xp );
1233
1234	// increment "forks"
1235	hal_remote_atomic_add( forks_xp , 1 );
1236
1237	// release lock protecting "forks" counter
1238	remote_busylock_release( lock_xp );
1239	}
1240	} // end loop on vpn
1241
1242	#if (DEBUG_VMM_SET_COW & 1)
1243	if( DEBUG_VMM_SET_COW < cycle )
1244	printk("\n[%s] thread[%x,%x] completes vseg %s / vpn_base = %x / vpn_size = %x\n",
1245	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
1246	#endif
1247	} // end if local
1248	} // end if vseg type
1249	} // end loop on vsegs
1250	} // end loop on process copies
1251
1252	#if DEBUG_VMM_SET_COW
1253	cycle = (uint32_t)hal_get_cycles();
1254	if( DEBUG_VMM_SET_COW < cycle )
1255	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
1256	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
1257	#endif
1258
1259	} // end vmm_set-cow()
1260
1261	/////////////////////////////////////////////////
1262	error_t vmm_fork_copy( process_t * child_process,
1263	xptr_t parent_process_xp )
1264	{
1265	error_t error;
1266	cxy_t parent_cxy;
1267	process_t * parent_process;
1268	vmm_t * parent_vmm;
1269	xptr_t parent_lock_xp;
1270	vmm_t * child_vmm;
1271	xptr_t iter_xp;
1272	xptr_t parent_vseg_xp;
1273	vseg_t * parent_vseg;
1274	vseg_t * child_vseg;
1275	uint32_t type;
1276	vpn_t vpn;
1277	vpn_t vpn_base;
1278	vpn_t vpn_size;
1279	xptr_t parent_root_xp;
1280	bool_t mapped;
1281	ppn_t ppn;
1282
1283	#if DEBUG_VMM_FORK_COPY
1284	uint32_t cycle = (uint32_t)hal_get_cycles();
1285	thread_t * this = CURRENT_THREAD;
1286	if( DEBUG_VMM_FORK_COPY < cycle )
1287	printk("\n[%s] thread %x enter / cycle %d\n",
1288	__FUNCTION__ , this->process->pid, this->trdid, cycle );
1289	#endif
1290
1291	// get parent process cluster and local pointer
1292	parent_cxy = GET_CXY( parent_process_xp );
1293	parent_process = GET_PTR( parent_process_xp );
1294
1295	// get local pointers on parent and child VMM
1296	parent_vmm = &parent_process->vmm;
1297	child_vmm = &child_process->vmm;
1298
1299	// build extended pointer on parent VSL root and lock
1300	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
1301	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsl_lock );
1302
1303	// take the lock protecting the parent VSL
1304	remote_queuelock_acquire( parent_lock_xp );
1305
1306	// loop on parent VSL xlist
1307	XLIST_FOREACH( parent_root_xp , iter_xp )
1308	{
1309	// get pointers on current parent vseg
1310	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1311	parent_vseg = GET_PTR( parent_vseg_xp );
1312
1313	// get vseg type
1314	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
1315
1316	#if DEBUG_VMM_FORK_COPY
1317	cycle = (uint32_t)hal_get_cycles();
1318	if( DEBUG_VMM_FORK_COPY < cycle )
1319	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
1320	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
1321	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
1322	#endif
1323
1324	// all parent vsegs - but STACK and kernel vsegs - must be copied in child VSL
1325	if( (type != VSEG_TYPE_STACK) && (type != VSEG_TYPE_KCODE) &&
1326	(type != VSEG_TYPE_KDATA) && (type != VSEG_TYPE_KDEV) )
1327	{
1328	// allocate memory for a new child vseg
1329	child_vseg = vseg_alloc();
1330	if( child_vseg == NULL ) // release all allocated vsegs
1331	{
1332	vmm_destroy( child_process );
1333	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
1334	return -1;
1335	}
1336
1337	// copy parent vseg to child vseg
1338	vseg_init_from_ref( child_vseg , parent_vseg_xp );
1339
1340	// build extended pointer on child VSL lock
1341	xptr_t child_lock_xp = XPTR( local_cxy , &child_vmm->vsl_lock );
1342
1343	// take the child VSL lock
1344	remote_queuelock_acquire( child_lock_xp );
1345
1346	// register child vseg in child VSL
1347	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
1348
1349	// release the child VSL lock
1350	remote_queuelock_release( child_lock_xp );
1351
1352	#if DEBUG_VMM_FORK_COPY
1353	cycle = (uint32_t)hal_get_cycles();
1354	if( DEBUG_VMM_FORK_COPY < cycle )
1355	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
1356	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
1357	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
1358	#endif
1359	// copy DATA, ANON, REMOTE, FILE parent GPT entries to child GPT
1360	if( type != VSEG_TYPE_CODE )
1361	{
1362	// activate the COW for DATA, ANON, REMOTE vsegs only
1363	// cow = ( type != VSEG_TYPE_FILE );
1364
1365	vpn_base = child_vseg->vpn_base;
1366	vpn_size = child_vseg->vpn_size;
1367
1368	// scan pages in parent vseg
1369	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
1370	{
1371	error = hal_gpt_pte_copy( &child_vmm->gpt,
1372	vpn,
1373	XPTR( parent_cxy , &parent_vmm->gpt ),
1374	vpn,
1375	false, // does not handle COW flag
1376	&ppn, // unused
1377	&mapped ); // unused
1378	if( error )
1379	{
1380	vmm_destroy( child_process );
1381	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
1382	return -1;
1383	}
1384
1385	#if DEBUG_VMM_FORK_COPY
1386	cycle = (uint32_t)hal_get_cycles();
1387	if( DEBUG_VMM_FORK_COPY < cycle )
1388	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
1389	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
1390	#endif
1391	}
1392	} // end if no code & no stack
1393	} // end if no stack
1394	} // end loop on vsegs
1395
1396	// release the parent VSL lock in read mode
1397	remote_queuelock_release( parent_lock_xp );
1398
1399	/* deprecated [AG] : this is already done by the vmm_user_init() funcfion
1400
1401	// initialize the child VMM STACK allocator
1402	vmm_stack_init( child_vmm );
1403
1404	// initialize the child VMM MMAP allocator
1405	vmm_mmap_init( child_vmm );
1406
1407	// initialize instrumentation counters
1408	child_vmm->false_pgfault_nr = 0;
1409	child_vmm->local_pgfault_nr = 0;
1410	child_vmm->global_pgfault_nr = 0;
1411	child_vmm->false_pgfault_cost = 0;
1412	child_vmm->local_pgfault_cost = 0;
1413	child_vmm->global_pgfault_cost = 0;
1414	*/
1415	// copy base addresses from parent VMM to child VMM
1416	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
1417	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
1418	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
1419	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
1420	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
1421
1422	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
1423
1424	hal_fence();
1425
1426	#if DEBUG_VMM_FORK_COPY
1427	cycle = (uint32_t)hal_get_cycles();
1428	if( DEBUG_VMM_FORK_COPY < cycle )
1429	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
1430	__FUNCTION__ , this->process->pid, this->trdid , cycle );
1431	#endif
1432
1433	return 0;
1434
1435	} // vmm_fork_copy()
1436
1437	///////////////////////////////////////
1438	void vmm_destroy( process_t * process )
1439	{
1440	xptr_t vseg_xp;
1441	vseg_t * vseg;
1442
1443	#if DEBUG_VMM_DESTROY
1444	uint32_t cycle = (uint32_t)hal_get_cycles();
1445	thread_t * this = CURRENT_THREAD;
1446	if( DEBUG_VMM_DESTROY < cycle )
1447	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
1448	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
1449	#endif
1450
1451	#if (DEBUG_VMM_DESTROY & 1 )
1452	if( DEBUG_VMM_DESTROY < cycle )
1453	hal_vmm_display( XPTR( local_cxy, process ) , true );
1454	#endif
1455
1456	// get pointer on local VMM
1457	vmm_t * vmm = &process->vmm;
1458
1459	// build extended pointer on VSL root, VSL lock and GPT lock
1460	xptr_t vsl_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1461	xptr_t vsl_lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
1462
1463	// take the VSL lock
1464	remote_queuelock_acquire( vsl_lock_xp );
1465
1466	// scan the VSL to delete all registered vsegs
1467	// (we don't use a FOREACH in case of item deletion)
1468	xptr_t iter_xp;
1469	xptr_t next_xp;
1470	for( iter_xp = hal_remote_l64( vsl_root_xp ) ;
1471	iter_xp != vsl_root_xp ;
1472	iter_xp = next_xp )
1473	{
1474	// save extended pointer on next item in xlist
1475	next_xp = hal_remote_l64( iter_xp );
1476
1477	// get pointers on current vseg in VSL
1478	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1479	vseg = GET_PTR( vseg_xp );
1480
1481	// delete vseg and release physical pages
1482	vmm_remove_vseg( process , vseg );
1483
1484	#if( DEBUG_VMM_DESTROY & 1 )
1485	if( DEBUG_VMM_DESTROY < cycle )
1486	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
1487	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
1488	#endif
1489
1490	}
1491
1492	// release the VSL lock
1493	remote_queuelock_release( vsl_lock_xp );
1494
1495	// remove all registered MMAP vsegs from free_lists in MMAP allocator
1496	uint32_t i;
1497	for( i = 0 ; i <= CONFIG_VMM_HEAP_MAX_ORDER ; i++ )
1498	{
1499	// build extended pointer on free list root
1500	xptr_t root_xp = XPTR( local_cxy , &vmm->mmap_mgr.free_list_root[i] );
1501
1502	// scan zombi_list[i]
1503	while( !xlist_is_empty( root_xp ) )
1504	{
1505	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
1506	vseg = GET_PTR( vseg_xp );
1507
1508	#if( DEBUG_VMM_DESTROY & 1 )
1509	if( DEBUG_VMM_DESTROY < cycle )
1510	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
1511	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
1512	#endif
1513	// clean vseg descriptor
1514	vseg->vmm = NULL;
1515
1516	// remove vseg from zombi_list
1517	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
1518
1519	// release vseg descriptor
1520	vseg_free( vseg );
1521
1522	#if( DEBUG_VMM_DESTROY & 1 )
1523	if( DEBUG_VMM_DESTROY < cycle )
1524	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
1525	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
1526	#endif
1527	}
1528	}
1529
1530	// release memory allocated to the GPT itself
1531	hal_gpt_destroy( &vmm->gpt );
1532
1533	#if DEBUG_VMM_DESTROY
1534	cycle = (uint32_t)hal_get_cycles();
1535	if( DEBUG_VMM_DESTROY < cycle )
1536	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
1537	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
1538	#endif
1539
1540	} // end vmm_destroy()
1541
1542	/////////////////////////////////////////////////
1543	vseg_t * vmm_check_conflict( process_t * process,
1544	vpn_t vpn_base,
1545	vpn_t vpn_size )
1546	{
1547	vmm_t * vmm = &process->vmm;
1548
1549	// scan the VSL
1550	vseg_t * vseg;
1551	xptr_t iter_xp;
1552	xptr_t vseg_xp;
1553	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1554
1555	XLIST_FOREACH( root_xp , iter_xp )
1556	{
1557	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1558	vseg = GET_PTR( vseg_xp );
1559
1560	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
1561	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
1562	}
1563	return NULL;
1564
1565	} // end vmm_check_conflict()
1566
1567	////////////////////////////////////////////////
1568	vseg_t * vmm_create_vseg( process_t * process,
1569	vseg_type_t type,
1570	intptr_t base, // ltid for VSEG_TYPE_STACK
1571	uint32_t size,
1572	uint32_t file_offset,
1573	uint32_t file_size,
1574	xptr_t mapper_xp,
1575	cxy_t cxy )
1576	{
1577	vseg_t * vseg; // pointer on allocated vseg descriptor
1578
1579	#if DEBUG_VMM_CREATE_VSEG
1580	thread_t * this = CURRENT_THREAD;
1581	uint32_t cycle;
1582	#endif
1583
1584	#if (DEBUG_VMM_CREATE_VSEG & 1)
1585	cycle = (uint32_t)hal_get_cycles();
1586	if( DEBUG_VMM_CREATE_VSEG < cycle )
1587	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cxy %x / cycle %d\n",
1588	__FUNCTION__, this->process->pid, this->trdid,
1589	process->pid, vseg_type_str(type), base, cxy, cycle );
1590	#endif
1591
1592	// get pointer on VMM
1593	vmm_t * vmm = &process->vmm;
1594
1595	// allocate a vseg descriptor and initialize it, depending on type
1596	// we use specific allocators for "stack" and "mmap" types
1597
1598	/////////////////////////////
1599	if( type == VSEG_TYPE_STACK )
1600	{
1601	// get vseg from STACK allocator
1602	vseg = vmm_stack_alloc( vmm , base ); // base == ltid
1603
1604	if( vseg == NULL )
1605	{
1606	printk("\n[ERROR] %s cannot create %s vseg for process %x in cluster %x\n",
1607	__FUNCTION__ , vseg_type_str( type ) , process->pid , local_cxy );
1608	return NULL;
1609	}
1610
1611	// initialize vseg
1612	vseg->type = type;
1613	vseg->vmm = vmm;
1614	vseg->min = vseg->vpn_base << CONFIG_PPM_PAGE_SHIFT;
1615	vseg->max = vseg->min + (vseg->vpn_size << CONFIG_PPM_PAGE_SHIFT);
1616	vseg->cxy = cxy;
1617
1618	vseg_init_flags( vseg , type );
1619	}
1620	/////////////////////////////////
1621	else if( type == VSEG_TYPE_FILE )
1622	{
1623	// compute page index (in mapper) for first and last byte
1624	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
1625	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1626
1627	// compute offset in first page and number of pages
1628	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1629	vpn_t npages = vpn_max - vpn_min + 1;
1630
1631	// get vseg from MMAP allocator
1632	vseg = vmm_mmap_alloc( vmm , npages );
1633
1634	if( vseg == NULL )
1635	{
1636	printk("\n[ERROR] %s cannot create %s vseg for process %x in cluster %x\n",
1637	__FUNCTION__ , vseg_type_str( type ) , process->pid , local_cxy );
1638	return NULL;
1639	}
1640
1641	// initialize vseg
1642	vseg->type = type;
1643	vseg->vmm = vmm;
1644	vseg->min = (vseg->vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1645	vseg->max = vseg->min + size;
1646	vseg->file_offset = file_offset;
1647	vseg->file_size = file_size;
1648	vseg->mapper_xp = mapper_xp;
1649	vseg->cxy = cxy;
1650
1651	vseg_init_flags( vseg , type );
1652	}
1653	/////////////////////////////////////////////////////////////////
1654	else if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_REMOTE) )
1655	{
1656	// compute number of required pages in virtual space
1657	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1658	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1659
1660	// allocate vseg from MMAP allocator
1661	vseg = vmm_mmap_alloc( vmm , npages );
1662
1663	if( vseg == NULL )
1664	{
1665	printk("\n[ERROR] %s cannot create %s vseg for process %x in cluster %x\n",
1666	__FUNCTION__ , vseg_type_str( type ) , process->pid , local_cxy );
1667	return NULL;
1668	}
1669
1670	// initialize vseg
1671	vseg->type = type;
1672	vseg->vmm = vmm;
1673	vseg->min = vseg->vpn_base << CONFIG_PPM_PAGE_SHIFT;
1674	vseg->max = vseg->min + (vseg->vpn_size << CONFIG_PPM_PAGE_SHIFT);
1675	vseg->cxy = cxy;
1676
1677	vseg_init_flags( vseg , type );
1678	}
1679	/////////////////////////////////////////////////////////////////
1680	else // VSEG_TYPE_DATA, VSEG_TYPE_CODE or KERNEL vseg
1681	{
1682	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1683	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1684
1685	// allocate vseg descriptor
1686	vseg = vseg_alloc();
1687
1688	if( vseg == NULL )
1689	{
1690	printk("\n[ERROR] %s cannot create %s vseg for process %x in cluster %x\n",
1691	__FUNCTION__ , vseg_type_str( type ) , process->pid , local_cxy );
1692	return NULL;
1693	}
1694	// initialize vseg
1695	vseg->type = type;
1696	vseg->vmm = vmm;
1697	vseg->min = base;
1698	vseg->max = base + size;
1699	vseg->vpn_base = base >> CONFIG_PPM_PAGE_SHIFT;
1700	vseg->vpn_size = vpn_max - vpn_min + 1;
1701	vseg->file_offset = file_offset;
1702	vseg->file_size = file_size;
1703	vseg->mapper_xp = mapper_xp;
1704	vseg->cxy = cxy;
1705
1706	vseg_init_flags( vseg , type );
1707	}
1708
1709	// check collisions
1710	vseg_t * existing_vseg = vmm_check_conflict( process , vseg->vpn_base , vseg->vpn_size );
1711
1712	if( existing_vseg != NULL )
1713	{
1714	printk("\n[ERROR] in %s for process %x : new vseg %s [vpn_base %x / vpn_size %x]\n"
1715	" overlap existing vseg %s [vpn_base %x / vpn_size %x]\n",
1716	__FUNCTION__ , process->pid, vseg_type_str(vseg->type), vseg->vpn_base, vseg->vpn_size,
1717	vseg_type_str(existing_vseg->type), existing_vseg->vpn_base, existing_vseg->vpn_size );
1718	vseg_free( vseg );
1719	return NULL;
1720	}
1721
1722	// build extended pointer on VSL lock
1723	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsl_lock );
1724
1725	// take the VSL lock in write mode
1726	remote_queuelock_acquire( lock_xp );
1727
1728	// attach vseg to VSL
1729	vmm_attach_vseg_to_vsl( vmm , vseg );
1730
1731	// release the VSL lock
1732	remote_queuelock_release( lock_xp );
1733
1734	#if DEBUG_VMM_CREATE_VSEG
1735	cycle = (uint32_t)hal_get_cycles();
1736	if( DEBUG_VMM_CREATE_VSEG < cycle )
1737	printk("\n[%s] thread[%x,%x] exit / %s / vpn_base %x / vpn_size %x / cycle %d\n",
1738	__FUNCTION__, this->process->pid, this->trdid,
1739	vseg_type_str(type), vseg->vpn_base, vseg->vpn_size, cycle );
1740	#endif
1741
1742	return vseg;
1743
1744	} // vmm_create_vseg()
1745
1746	////////////////////////////////////////////////////////////////////////////////////////////
1747	// This static function is called by the vmm_remove_vseg() and vmm_resize_vseg() functions
1748	// to update the physical page descriptor identified by the <ppn> argument.
1749	// It decrements the refcount, set the dirty bit when required, and releases the physical
1750	// page to kmem depending on the vseg type.
1751	// - KERNEL : refcount decremented / not released to kmem / dirty bit not set
1752	// - FILE : refcount decremented / not released to kmem / dirty bit set when required.
1753	// - CODE : refcount decremented / released to kmem / dirty bit not set.
1754	// - STAK : refcount decremented / released to kmem / dirty bit not set.
1755	// - DATA : refcount decremented / released to kmem if ref / dirty bit not set.
1756	// - MMAP : refcount decremented / released to kmem if ref / dirty bit not set.
1757	////////////////////////////////////////////////////////////////////////////////////////////
1758	// @ process : local pointer on process.
1759	// @ vseg : local pointer on vseg.
1760	// @ ppn : released pysical page index.
1761	// @ dirty : set the dirty bit in page descriptor when non zero.
1762	////////////////////////////////////////////////////////////////////////////////////////////
1763	static void vmm_ppn_release( process_t * process,
1764	vseg_t * vseg,
1765	ppn_t ppn,
1766	uint32_t dirty )
1767	{
1768	bool_t do_kmem_release;
1769
1770	// get vseg type
1771	vseg_type_t type = vseg->type;
1772
1773	// compute is_ref <=> this vseg is the reference vseg
1774	bool_t is_ref = (GET_CXY( process->ref_xp ) == local_cxy);
1775
1776	// get pointers on physical page descriptor
1777	xptr_t page_xp = ppm_ppn2page( ppn );
1778	cxy_t page_cxy = GET_CXY( page_xp );
1779	page_t * page_ptr = GET_PTR( page_xp );
1780
1781	// decrement page refcount
1782	xptr_t count_xp = XPTR( page_cxy , &page_ptr->refcount );
1783	hal_remote_atomic_add( count_xp , -1 );
1784
1785	// compute the do_kmem_release condition depending on vseg type
1786	if( (type == VSEG_TYPE_KCODE) \|\|
1787	(type == VSEG_TYPE_KDATA) \|\|
1788	(type == VSEG_TYPE_KDEV) )
1789	{
1790	// no physical page release for KERNEL
1791	do_kmem_release = false;
1792	}
1793	else if( type == VSEG_TYPE_FILE )
1794	{
1795	// no physical page release for KERNEL
1796	do_kmem_release = false;
1797
1798	// set dirty bit if required
1799	if( dirty ) ppm_page_do_dirty( page_xp );
1800	}
1801	else if( (type == VSEG_TYPE_CODE) \|\|
1802	(type == VSEG_TYPE_STACK) )
1803	{
1804	// always release physical page for private vsegs
1805	do_kmem_release = true;
1806	}
1807	else if( (type == VSEG_TYPE_ANON) \|\|
1808	(type == VSEG_TYPE_REMOTE) )
1809	{
1810	// release physical page if reference cluster
1811	do_kmem_release = is_ref;
1812	}
1813	else if( is_ref ) // vseg_type == DATA in reference cluster
1814	{
1815	// get extended pointers on forks and lock field in page descriptor
1816	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
1817	xptr_t lock_xp = XPTR( page_cxy , &page_ptr->lock );
1818
1819	// take lock protecting "forks" counter
1820	remote_busylock_acquire( lock_xp );
1821
1822	// get number of pending forks from page descriptor
1823	uint32_t forks = hal_remote_l32( forks_xp );
1824
1825	// decrement pending forks counter if required
1826	if( forks ) hal_remote_atomic_add( forks_xp , -1 );
1827
1828	// release lock protecting "forks" counter
1829	remote_busylock_release( lock_xp );
1830
1831	// release physical page if forks == 0
1832	do_kmem_release = (forks == 0);
1833	}
1834	else // vseg_type == DATA not in reference cluster
1835	{
1836	// no physical page release if not in reference cluster
1837	do_kmem_release = false;
1838	}
1839
1840	// release physical page to relevant kmem when required
1841	if( do_kmem_release )
1842	{
1843	kmem_req_t req;
1844	req.type = KMEM_PPM;
1845	req.ptr = GET_PTR( ppm_ppn2base( ppn ) );
1846
1847	kmem_remote_free( page_cxy , &req );
1848
1849	#if DEBUG_VMM_PPN_RELEASE
1850	thread_t * this = CURRENT_THREAD;
1851	if( DEBUG_VMM_PPN_RELEASE < cycle )
1852	printk("\n[%s] thread[%x,%x] released ppn %x to kmem\n",
1853	__FUNCTION__, this->process->pid, this->trdid, ppn );
1854	#endif
1855
1856	}
1857	} // end vmm_ppn_release()
1858
1859	//////////////////////////////////////////
1860	void vmm_remove_vseg( process_t * process,
1861	vseg_t * vseg )
1862	{
1863	uint32_t vseg_type; // vseg type
1864	vpn_t vpn; // VPN of current PTE
1865	vpn_t vpn_min; // VPN of first PTE
1866	vpn_t vpn_max; // VPN of last PTE (excluded)
1867	ppn_t ppn; // current PTE ppn value
1868	uint32_t attr; // current PTE attributes
1869
1870	// check arguments
1871	assert( (process != NULL), "process argument is NULL" );
1872	assert( (vseg != NULL), "vseg argument is NULL" );
1873
1874	// get pointers on local process VMM
1875	vmm_t * vmm = &process->vmm;
1876
1877	// build extended pointer on GPT
1878	xptr_t gpt_xp = XPTR( local_cxy , &vmm->gpt );
1879
1880	// get relevant vseg infos
1881	vseg_type = vseg->type;
1882	vpn_min = vseg->vpn_base;
1883	vpn_max = vpn_min + vseg->vpn_size;
1884
1885	#if DEBUG_VMM_REMOVE_VSEG
1886	uint32_t cycle = (uint32_t)hal_get_cycles();
1887	thread_t * this = CURRENT_THREAD;
1888	#endif
1889
1890	#if (DEBUG_VMM_REMOVE_VSEG & 1 )
1891	if( DEBUG_VMM_REMOVE_VSEG < cycle )
1892	printk("\n[%s] thread[%x,%x] enters / process %x / type %s / base %x / cycle %d\n",
1893	__FUNCTION__, this->process->pid, this->trdid,
1894	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
1895	#endif
1896
1897	// loop on PTEs in GPT to unmap all mapped PTE
1898	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1899	{
1900	// get ppn and attr
1901	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
1902
1903	if( attr & GPT_MAPPED ) // PTE is mapped
1904	{
1905
1906	#if( DEBUG_VMM_REMOVE_VSEG & 1 )
1907	if( DEBUG_VMM_REMOVE_VSEG < cycle )
1908	printk("\n[%s] thread[%x,%x] unmap vpn %x / ppn %x / type %s\n",
1909	__FUNCTION__, this->process->pid, this->trdid, vpn , ppn, vseg_type_str(vseg_type) );
1910	#endif
1911	// unmap GPT entry in local GPT
1912	hal_gpt_reset_pte( gpt_xp , vpn );
1913
1914	// release physical page depending on vseg type
1915	vmm_ppn_release( process , vseg , ppn , attr & GPT_DIRTY );
1916	}
1917	}
1918
1919	// remove vseg from VSL
1920	vmm_detach_vseg_from_vsl( vmm , vseg );
1921
1922	// release vseg descriptor depending on vseg type
1923	if( vseg_type == VSEG_TYPE_STACK )
1924	{
1925	// release slot to local stack allocator
1926	vmm_stack_free( vmm , vseg );
1927	}
1928	else if( (vseg_type == VSEG_TYPE_ANON) \|\|
1929	(vseg_type == VSEG_TYPE_FILE) \|\|
1930	(vseg_type == VSEG_TYPE_REMOTE) )
1931	{
1932	// release vseg to local mmap allocator
1933	vmm_mmap_free( vmm , vseg );
1934	}
1935	else
1936	{
1937	// release vseg descriptor to local kmem
1938	vseg_free( vseg );
1939	}
1940
1941	#if DEBUG_VMM_REMOVE_VSEG
1942	cycle = (uint32_t)hal_get_cycles();
1943	if( DEBUG_VMM_REMOVE_VSEG < cycle )
1944	printk("\n[%s] thread[%x,%x] exit / process %x / type %s / base %x / cycle %d\n",
1945	__FUNCTION__, this->process->pid, this->trdid,
1946	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
1947	#endif
1948
1949	} // end vmm_remove_vseg()
1950
1951	/////////////////////////////////////////////
1952	void vmm_resize_vseg( process_t * process,
1953	vseg_t * vseg,
1954	intptr_t new_base,
1955	intptr_t new_size )
1956	{
1957	vpn_t vpn;
1958	ppn_t ppn;
1959	uint32_t attr;
1960
1961	// check arguments
1962	assert( (process != NULL), "process argument is NULL" );
1963	assert( (vseg != NULL), "vseg argument is NULL" );
1964
1965	#if DEBUG_VMM_RESIZE_VSEG
1966	uint32_t cycle = (uint32_t)hal_get_cycles();
1967	thread_t * this = CURRENT_THREAD;
1968	#endif
1969
1970	#if (DEBUG_VMM_RESIZE_VSEG & 1)
1971	if( DEBUG_VMM_RESIZE_VSEG < cycle )
1972	printk("\n[%s] thread[%x,%x] enter / process %x / %s / base %x / cycle %d\n",
1973	__FUNCTION__, this->process->pid, this->trdid,
1974	process->pid, vseg_type_str(vseg->type), old_base, cycle );
1975	#endif
1976
1977	// get existing vseg vpn_min and vpn_max
1978	vpn_t old_vpn_min = vseg->vpn_base;
1979	vpn_t old_vpn_max = old_vpn_min + vseg->vpn_size - 1;
1980
1981	// compute new vseg vpn_min & vpn_max
1982	intptr_t min = new_base;
1983	intptr_t max = new_base + new_size;
1984	vpn_t new_vpn_min = min >> CONFIG_PPM_PAGE_SHIFT;
1985	vpn_t new_vpn_max = (max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1986
1987	// build extended pointer on GPT
1988	xptr_t gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1989
1990	// loop on PTEs in GPT to unmap PTE if (old_vpn_min <= vpn < new_vpn_min)
1991	for( vpn = old_vpn_min ; vpn < new_vpn_min ; vpn++ )
1992	{
1993	// get ppn and attr
1994	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
1995
1996	if( attr & GPT_MAPPED ) // PTE is mapped
1997	{
1998
1999	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
2000	if( DEBUG_VMM_RESIZE_VSEG < cycle )
2001	printk("\n[%s] thread[%x,%x] unmap vpn %x / ppn %x / %s",
2002	__FUNCTION__, this->process->pid, this->trdid, vpn , ppn, vseg_type_str(vseg_type) );
2003	#endif
2004	// unmap GPT entry
2005	hal_gpt_reset_pte( gpt_xp , vpn );
2006
2007	// release physical page when required
2008	vmm_ppn_release( process , vseg , ppn , attr & GPT_DIRTY );
2009	}
2010	}
2011
2012	// loop on PTEs in GPT to unmap PTE if (new vpn_max <= vpn < old_vpn_max)
2013	for( vpn = new_vpn_max ; vpn < old_vpn_max ; vpn++ )
2014	{
2015	// get ppn and attr
2016	hal_gpt_get_pte( gpt_xp , vpn , &attr , &ppn );
2017
2018	if( attr & GPT_MAPPED ) // PTE is mapped
2019	{
2020
2021	#if( DEBUG_VMM_RESIZE_VSEG & 1 )
2022	if( DEBUG_VMM_RESIZE_VSEG < cycle )
2023	printk("\n[%s] thread[%x,%x] unmap vpn %x / ppn %x / %s",
2024	__FUNCTION__, this->process->pid, this->trdid, vpn , ppn, vseg_type_str(vseg_type) );
2025	#endif
2026	// unmap GPT entry in local GPT
2027	hal_gpt_reset_pte( gpt_xp , vpn );
2028
2029	// release physical page when required
2030	vmm_ppn_release( process , vseg , ppn , attr & GPT_DIRTY );
2031	}
2032	}
2033
2034	// resize vseg in VSL
2035	vseg->min = min;
2036	vseg->max = max;
2037	vseg->vpn_base = new_vpn_min;
2038	vseg->vpn_size = new_vpn_max - new_vpn_min + 1;
2039
2040	#if DEBUG_VMM_RESIZE_VSEG
2041	cycle = (uint32_t)hal_get_cycles();
2042	if( DEBUG_VMM_RESIZE_VSEG < cycle )
2043	printk("[%s] thread[%x,%x] exit / process %x / %s / base %x / cycle %d\n",
2044	__FUNCTION__, this->process->pid, this->trdid,
2045	process->pid, vseg_type_str(vseg->type), vseg->min, cycle );
2046	#endif
2047
2048	} // end vmm_resize_vseg
2049
2050	/////////////////////////////////////////////////////////////////////////////////////////////
2051	// This static function is called twice by the vmm_get_vseg() function.
2052	// It scan the - possibly remote - VSL defined by the <vmm_xp> argument to find the vseg
2053	// containing a given virtual address <vaddr>. It uses remote accesses to access the remote
2054	// VSL if required. The VSL lock protecting the VSL must be taken by the caller.
2055	/////////////////////////////////////////////////////////////////////////////////////////////
2056	// @ vmm_xp : extended pointer on the process VMM.
2057	// @ vaddr : virtual address.
2058	// @ return local pointer on remote vseg if success / return NULL if not found.
2059	/////////////////////////////////////////////////////////////////////////////////////////////
2060	static vseg_t * vmm_vseg_from_vaddr( xptr_t vmm_xp,
2061	intptr_t vaddr )
2062	{
2063	xptr_t iter_xp;
2064	xptr_t vseg_xp;
2065	vseg_t * vseg;
2066	intptr_t min;
2067	intptr_t max;
2068
2069	// get cluster and local pointer on target VMM
2070	vmm_t * vmm_ptr = GET_PTR( vmm_xp );
2071	cxy_t vmm_cxy = GET_CXY( vmm_xp );
2072
2073	// build extended pointer on VSL root
2074	xptr_t root_xp = XPTR( vmm_cxy , &vmm_ptr->vsegs_root );
2075
2076	// scan the list of vsegs in VSL
2077	XLIST_FOREACH( root_xp , iter_xp )
2078	{
2079	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
2080	vseg = GET_PTR( vseg_xp );
2081
2082	min = hal_remote_l32( XPTR( vmm_cxy , &vseg->min ) );
2083	max = hal_remote_l32( XPTR( vmm_cxy , &vseg->max ) );
2084
2085	// return success when match
2086	if( (vaddr >= min) && (vaddr < max) ) return vseg;
2087	}
2088
2089	// return failure
2090	return NULL;
2091
2092	} // end vmm_vseg_from_vaddr()
2093
2094	///////////////////////////////////////////
2095	error_t vmm_get_vseg( process_t * process,
2096	intptr_t vaddr,
2097	vseg_t ** found_vseg )
2098	{
2099	xptr_t loc_lock_xp; // extended pointer on local VSL lock
2100	xptr_t ref_lock_xp; // extended pointer on reference VSL lock
2101	vseg_t * loc_vseg; // local pointer on local vseg
2102	vseg_t * ref_vseg; // local pointer on reference vseg
2103
2104	// build extended pointer on local VSL lock
2105	loc_lock_xp = XPTR( local_cxy , &process->vmm.vsl_lock );
2106
2107	// get local VSL lock
2108	remote_queuelock_acquire( loc_lock_xp );
2109
2110	// try to get vseg from local VMM
2111	loc_vseg = vmm_vseg_from_vaddr( XPTR( local_cxy, &process->vmm ) , vaddr );
2112
2113	if (loc_vseg == NULL) // vseg not found => access reference VSL
2114	{
2115	// get extended pointer on reference process
2116	xptr_t ref_xp = process->ref_xp;
2117
2118	// get cluster and local pointer on reference process
2119	cxy_t ref_cxy = GET_CXY( ref_xp );
2120	process_t * ref_ptr = GET_PTR( ref_xp );
2121
2122	// build extended pointer on reference VSL lock
2123	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.vsl_lock );
2124
2125	// get reference VSL lock
2126	remote_queuelock_acquire( ref_lock_xp );
2127
2128	// try to get vseg from reference VMM
2129	ref_vseg = vmm_vseg_from_vaddr( XPTR( ref_cxy , &ref_ptr->vmm ) , vaddr );
2130
2131	if( ref_vseg == NULL ) // vseg not found => return error
2132	{
2133	printk("\n[ERROR] in %s : vaddr %x in process %x out of segment\n",
2134	__FUNCTION__, vaddr, process->pid );
2135
2136	// release reference VSL lock
2137	remote_queuelock_release( ref_lock_xp );
2138
2139	return -1;
2140	}
2141	else // vseg found => try to update local VSL
2142	{
2143	// allocate a local vseg descriptor
2144	loc_vseg = vseg_alloc();
2145
2146	if( loc_vseg == NULL ) // no memory => return error
2147	{
2148	printk("\n[ERROR] in %s : vaddr %x in process %x / no memory for local vseg\n",
2149	__FUNCTION__, vaddr, process->pid );
2150
2151	// release reference VSL & local VSL locks
2152	remote_queuelock_release( ref_lock_xp );
2153	remote_queuelock_release( loc_lock_xp );
2154
2155	return -1;
2156	}
2157	else // update local VSL and return success
2158	{
2159	// initialize local vseg
2160	vseg_init_from_ref( loc_vseg , XPTR( ref_cxy , ref_vseg ) );
2161
2162	// register local vseg in local VSL
2163	vmm_attach_vseg_to_vsl( &process->vmm , loc_vseg );
2164
2165	// release reference VSL & local VSL locks
2166	remote_queuelock_release( ref_lock_xp );
2167	remote_queuelock_release( loc_lock_xp );
2168
2169	*found_vseg = loc_vseg;
2170	return 0;
2171	}
2172	}
2173	}
2174	else // vseg found in local VSL => return success
2175	{
2176	// release local VSL lock
2177	remote_queuelock_release( loc_lock_xp );
2178
2179	*found_vseg = loc_vseg;
2180	return 0;
2181	}
2182	} // end vmm_get_vseg()
2183
2184	//////////////////////////////////////////////////////////////////////////////////////
2185	// This static function compute the target cluster to allocate a physical page
2186	// for a given <vpn> in a given <vseg>, allocates the page and returns an extended
2187	// pointer on the allocated page descriptor.
2188	// The vseg cannot have the FILE type.
2189	//////////////////////////////////////////////////////////////////////////////////////
2190	// @ vseg : local pointer on vseg.
2191	// @ vpn : unmapped vpn.
2192	// @ return an extended pointer on the allocated page descriptor.
2193	//////////////////////////////////////////////////////////////////////////////////////
2194	static xptr_t vmm_page_allocate( vseg_t * vseg,
2195	vpn_t vpn )
2196	{
2197
2198	#if DEBUG_VMM_PAGE_ALLOCATE
2199	uint32_t cycle = (uint32_t)hal_get_cycles();
2200	thread_t * this = CURRENT_THREAD;
2201	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
2202	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
2203	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
2204	#endif
2205
2206	xptr_t page_xp;
2207	cxy_t page_cxy;
2208	uint32_t index;
2209
2210	uint32_t type = vseg->type;
2211	uint32_t flags = vseg->flags;
2212	uint32_t x_size = LOCAL_CLUSTER->x_size;
2213	uint32_t y_size = LOCAL_CLUSTER->y_size;
2214
2215	// check vseg type
2216	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
2217
2218	// compute target cluster identifier
2219	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
2220	{
2221	index = vpn & ((x_size * y_size) - 1);
2222	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
2223
2224	// If the cluster selected from VPN's LSBs is empty, we select one randomly
2225	if ( cluster_is_active( page_cxy ) == false )
2226	{
2227	page_cxy = cluster_random_select();
2228	}
2229	}
2230	else // other cases => cxy specified in vseg
2231	{
2232	page_cxy = vseg->cxy;
2233	}
2234
2235	// allocate one small physical page from target cluster
2236	kmem_req_t req;
2237	req.type = KMEM_PPM;
2238	req.order = 0;
2239	req.flags = AF_ZERO;
2240
2241	// get local pointer on page base
2242	void * ptr = kmem_remote_alloc( page_cxy , &req );
2243
2244	// get extended pointer on page descriptor
2245	page_xp = ppm_base2page( XPTR( page_cxy , ptr ) );
2246
2247	#if DEBUG_VMM_PAGE_ALLOCATE
2248	cycle = (uint32_t)hal_get_cycles();
2249	if( DEBUG_VMM_PAGE_ALLOCATE < cycle )
2250	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
2251	__FUNCTION__ , this->process->pid, this->trdid, vpn, ppm_page2ppn(page_xp), cycle );
2252	#endif
2253
2254	return page_xp;
2255
2256	} // end vmm_page_allocate()
2257
2258	////////////////////////////////////////
2259	error_t vmm_get_one_ppn( vseg_t * vseg,
2260	vpn_t vpn,
2261	ppn_t * ppn )
2262	{
2263	error_t error;
2264	xptr_t page_xp; // extended pointer on physical page descriptor
2265	uint32_t page_id; // missing page index in vseg mapper
2266	uint32_t type; // vseg type;
2267
2268	type = vseg->type;
2269	page_id = vpn - vseg->vpn_base;
2270
2271	#if DEBUG_VMM_GET_ONE_PPN
2272	uint32_t cycle = (uint32_t)hal_get_cycles();
2273	thread_t * this = CURRENT_THREAD;
2274	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2275	printk("\n[%s] thread[%x,%x] enter for vpn %x / vseg %s / page_id %d / cycle %d\n",
2276	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
2277	#endif
2278
2279	#if (DEBUG_VMM_GET_ONE_PPN & 2)
2280	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2281	hal_vmm_display( XPTR( local_cxy , this->process ) , true );
2282	#endif
2283
2284	// FILE type : get the physical page from the file mapper
2285	if( type == VSEG_TYPE_FILE )
2286	{
2287	// get extended pointer on mapper
2288	xptr_t mapper_xp = vseg->mapper_xp;
2289
2290	assert( (mapper_xp != XPTR_NULL),
2291	"mapper not defined for a FILE vseg\n" );
2292
2293	// get extended pointer on page descriptor
2294	page_xp = mapper_get_page( mapper_xp , page_id );
2295
2296	if ( page_xp == XPTR_NULL ) return EINVAL;
2297	}
2298
2299	// Other types : allocate a physical page from target cluster,
2300	// as defined by vseg type and vpn value
2301	else
2302	{
2303	// allocate one physical page
2304	page_xp = vmm_page_allocate( vseg , vpn );
2305
2306	if( page_xp == XPTR_NULL ) return -1;
2307
2308	// initialise missing page from .elf file mapper for DATA and CODE types
2309	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
2310	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
2311	{
2312	// get extended pointer on mapper
2313	xptr_t mapper_xp = vseg->mapper_xp;
2314
2315	assert( (mapper_xp != XPTR_NULL),
2316	"mapper not defined for a CODE or DATA vseg\n" );
2317
2318	// compute missing page offset in vseg
2319	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
2320
2321	// compute missing page offset in .elf file
2322	uint32_t elf_offset = vseg->file_offset + offset;
2323
2324	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
2325	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2326	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
2327	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
2328	#endif
2329	// compute extended pointer on page base
2330	xptr_t base_xp = ppm_page2base( page_xp );
2331
2332	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
2333	uint32_t file_size = vseg->file_size;
2334
2335	if( file_size < offset ) // missing page fully in BSS
2336	{
2337
2338	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
2339	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2340	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
2341	__FUNCTION__, this->process->pid, this->trdid, vpn );
2342	#endif
2343	if( GET_CXY( page_xp ) == local_cxy )
2344	{
2345	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
2346	}
2347	else
2348	{
2349	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
2350	}
2351	}
2352	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
2353	{
2354
2355	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
2356	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2357	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
2358	__FUNCTION__, this->process->pid, this->trdid, vpn );
2359	#endif
2360	error = mapper_move_kernel( mapper_xp,
2361	true, // to_buffer
2362	elf_offset,
2363	base_xp,
2364	CONFIG_PPM_PAGE_SIZE );
2365	if( error ) return EINVAL;
2366	}
2367	else // both in mapper and in BSS :
2368	// - (file_size - offset) bytes from mapper
2369	// - (page_size + offset - file_size) bytes from BSS
2370	{
2371
2372	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
2373	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2374	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
2375	" %d bytes from mapper / %d bytes from BSS\n",
2376	__FUNCTION__, this->process->pid, this->trdid, vpn,
2377	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
2378	#endif
2379	// initialize mapper part
2380	error = mapper_move_kernel( mapper_xp,
2381	true, // to buffer
2382	elf_offset,
2383	base_xp,
2384	file_size - offset );
2385	if( error ) return EINVAL;
2386
2387	// initialize BSS part
2388	if( GET_CXY( page_xp ) == local_cxy )
2389	{
2390	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
2391	offset + CONFIG_PPM_PAGE_SIZE - file_size );
2392	}
2393	else
2394	{
2395	hal_remote_memset( base_xp + file_size - offset , 0 ,
2396	offset + CONFIG_PPM_PAGE_SIZE - file_size );
2397	}
2398	}
2399
2400	} // end if CODE or DATA types
2401	}
2402
2403	// return ppn
2404	*ppn = ppm_page2ppn( page_xp );
2405
2406	#if DEBUG_VMM_GET_ONE_PPN
2407	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2408	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
2409	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
2410	#endif
2411
2412	#if (DEBUG_VMM_GET_ONE_PPN & 2)
2413	if( DEBUG_VMM_GET_ONE_PPN < cycle )
2414	hal_vmm_display( XPTR( local_cxy , this->process ) , true );
2415	#endif
2416
2417	return 0;
2418
2419	} // end vmm_get_one_ppn()
2420
2421	///////////////////////////////////////////////////
2422	error_t vmm_handle_page_fault( process_t * process,
2423	vpn_t vpn )
2424	{
2425	vseg_t * vseg; // vseg containing vpn
2426	uint32_t attr; // PTE_ATTR value
2427	ppn_t ppn; // PTE_PPN value
2428	uint32_t ref_attr; // PTE_ATTR value in reference GPT
2429	ppn_t ref_ppn; // PTE_PPN value in reference GPT
2430	cxy_t ref_cxy; // reference cluster for missing vpn
2431	process_t * ref_ptr; // reference process for missing vpn
2432	xptr_t local_gpt_xp; // extended pointer on local GPT
2433	xptr_t ref_gpt_xp; // extended pointer on reference GPT
2434	error_t error; // value returned by called functions
2435
2436	thread_t * this = CURRENT_THREAD;
2437
2438	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2439	uint32_t start_cycle = (uint32_t)hal_get_cycles();
2440	#endif
2441
2442	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2443	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) & (vpn > 0) )
2444	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
2445	__FUNCTION__, this->process->pid, this->trdid, vpn, start_cycle );
2446	#endif
2447
2448	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 2)
2449	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2450	hal_vmm_display( XPTR( local_cxy , this->process ) , true );
2451	#endif
2452
2453	// get local vseg (access to reference VSL can be required)
2454	error = vmm_get_vseg( process,
2455	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
2456	&vseg );
2457	if( error )
2458	{
2459	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in registered vseg\n",
2460	__FUNCTION__ , vpn , process->pid, this->trdid );
2461
2462	return EXCP_USER_ERROR;
2463	}
2464
2465	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2466	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2467	printk("\n[%s] thread[%x,%x] found vseg %s\n",
2468	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
2469	#endif
2470
2471	// build extended pointer on local GPT
2472	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
2473
2474	// lock PTE in local GPT and get current PPN and attributes
2475	error = hal_gpt_lock_pte( local_gpt_xp,
2476	vpn,
2477	&attr,
2478	&ppn );
2479	if( error )
2480	{
2481	printk("\n[PANIC] in %s : cannot lock PTE in local GPT / vpn %x / process %x\n",
2482	__FUNCTION__ , vpn , process->pid );
2483
2484	return EXCP_KERNEL_PANIC;
2485	}
2486
2487	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2488	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2489	printk("\n[%s] thread[%x,%x] locked vpn %x in cluster %x\n",
2490	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy );
2491	#endif
2492
2493	// handle page fault only if local PTE still unmapped after lock
2494	if( (attr & GPT_MAPPED) == 0 )
2495	{
2496	// get reference process cluster and local pointer
2497	ref_cxy = GET_CXY( process->ref_xp );
2498	ref_ptr = GET_PTR( process->ref_xp );
2499
2500	/////////////// private vseg or (local == reference)
2501	/////////////// => access only the local GPT
2502	if( (vseg->type == VSEG_TYPE_STACK) \|\|
2503	(vseg->type == VSEG_TYPE_CODE) \|\|
2504	(ref_cxy == local_cxy ) )
2505	{
2506
2507	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2508	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2509	printk("\n[%s] thread[%x,%x] access local gpt : cxy %x / ref_cxy %x / type %s / cycle %d\n",
2510	__FUNCTION__, this->process->pid, this->trdid,
2511	local_cxy, ref_cxy, vseg_type_str(vseg->type), (uint32_t)hal_get_cycles() );
2512	#endif
2513	// allocate and initialise a physical page
2514	error = vmm_get_one_ppn( vseg , vpn , &ppn );
2515
2516	if( error )
2517	{
2518	printk("\n[ERROR] in %s : no physical page / process = %x / vpn = %x\n",
2519	__FUNCTION__ , process->pid , vpn );
2520
2521	// unlock PTE in local GPT
2522	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2523
2524	return EXCP_KERNEL_PANIC;
2525	}
2526
2527	// define attr from vseg flags
2528	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
2529	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
2530	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
2531	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
2532	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
2533
2534	// set PTE to local GPT
2535	// it unlocks this PTE
2536	hal_gpt_set_pte( local_gpt_xp,
2537	vpn,
2538	attr,
2539	ppn );
2540
2541	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2542	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2543	#endif
2544
2545	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2546	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2547	printk("\n[%s] thread[%x,%x] handled local pgfault / ppn %x / attr %x / cycle %d\n",
2548	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
2549	#endif
2550
2551	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 2)
2552	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2553	hal_vmm_display( XPTR( local_cxy , this->process ) , true );
2554	#endif
2555
2556	#if CONFIG_INSTRUMENTATION_PGFAULTS
2557	uint32_t cost = end_cycle - start_cycle;
2558	this->info.local_pgfault_nr++;
2559	this->info.local_pgfault_cost += cost;
2560	if( cost > this->info.local_pgfault_max ) this->info.local_pgfault_max = cost;
2561	#endif
2562	return EXCP_NON_FATAL;
2563
2564	} // end local GPT access
2565
2566	/////////////////// public vseg and (local != reference)
2567	/////////////////// => access ref GPT to update local GPT
2568	else
2569	{
2570
2571	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2572	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2573	printk("\n[%s] thread[%x,%x] access ref gpt : cxy %x / ref_cxy %x / type %s / cycle %d\n",
2574	__FUNCTION__, this->process->pid, this->trdid,
2575	local_cxy, ref_cxy, vseg_type_str(vseg->type), (uint32_t)hal_get_cycles() );
2576	#endif
2577	// build extended pointer on reference GPT
2578	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
2579
2580	// lock PTE in reference GPT and get current PPN and attributes
2581	error = hal_gpt_lock_pte( ref_gpt_xp,
2582	vpn,
2583	&ref_attr,
2584	&ref_ppn );
2585	if( error )
2586	{
2587	printk("\n[PANIC] in %s : cannot lock PTE in ref GPT / vpn %x / process %x\n",
2588	__FUNCTION__ , vpn , process->pid );
2589
2590	// unlock PTE in local GPT
2591	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2592
2593	return EXCP_KERNEL_PANIC;
2594	}
2595
2596	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2597	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2598	printk("\n[%s] thread[%x,%x] get pte from ref gpt / attr %x / ppn %x\n",
2599	__FUNCTION__, this->process->pid, this->trdid, ref_attr, ref_ppn );
2600	#endif
2601
2602	if( ref_attr & GPT_MAPPED ) // false page fault
2603	{
2604	// update local GPT from reference GPT values
2605	// this unlocks the PTE in local GPT
2606	hal_gpt_set_pte( local_gpt_xp,
2607	vpn,
2608	ref_attr,
2609	ref_ppn );
2610
2611	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2612	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2613	printk("\n[%s] thread[%x,%x] updated local gpt for a false pgfault\n",
2614	__FUNCTION__, this->process->pid, this->trdid );
2615	#endif
2616
2617	// unlock the PTE in reference GPT
2618	hal_gpt_unlock_pte( ref_gpt_xp, vpn );
2619
2620	#if (DEBUG_VMM_HANDLE_PAGE_FAULT &1)
2621	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2622	printk("\n[%s] thread[%x,%x] unlock the ref gpt after a false pgfault\n",
2623	__FUNCTION__, this->process->pid, this->trdid );
2624	#endif
2625
2626	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2627	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2628	#endif
2629
2630	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2631	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2632	printk("\n[%s] thread[%x,%x] handled false pgfault / ppn %x / attr %x / cycle %d\n",
2633	__FUNCTION__, this->process->pid, this->trdid, ref_ppn, ref_attr, end_cycle );
2634	#endif
2635
2636	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 2)
2637	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2638	hal_vmm_display( XPTR( local_cxy , this->process ) , true );
2639	#endif
2640
2641	#if CONFIG_INSTRUMENTATION_PGFAULTS
2642	uint32_t cost = end_cycle - start_cycle;
2643	this->info.false_pgfault_nr++;
2644	this->info.false_pgfault_cost += cost;
2645	if( cost > this->info.false_pgfault_max ) this->info.false_pgfault_max = cost;
2646	#endif
2647	return EXCP_NON_FATAL;
2648	}
2649	else // true page fault
2650	{
2651	// allocate and initialise a physical page depending on the vseg type
2652	error = vmm_get_one_ppn( vseg , vpn , &ppn );
2653
2654	if( error )
2655	{
2656	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
2657	__FUNCTION__ , process->pid , vpn );
2658
2659	// unlock PTE in local GPT and in reference GPT
2660	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2661	hal_gpt_unlock_pte( ref_gpt_xp , vpn );
2662
2663	return EXCP_KERNEL_PANIC;
2664	}
2665
2666	// define attr from vseg flags
2667	attr = GPT_MAPPED \| GPT_SMALL \| GPT_READABLE;
2668	if( vseg->flags & VSEG_USER ) attr \|= GPT_USER;
2669	if( vseg->flags & VSEG_WRITE ) attr \|= GPT_WRITABLE;
2670	if( vseg->flags & VSEG_EXEC ) attr \|= GPT_EXECUTABLE;
2671	if( vseg->flags & VSEG_CACHE ) attr \|= GPT_CACHABLE;
2672
2673	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2674	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2675	printk("\n[%s] thread[%x,%x] build a new PTE for a true pgfault\n",
2676	__FUNCTION__, this->process->pid, this->trdid );
2677	#endif
2678	// set PTE in reference GPT
2679	// this unlock the PTE
2680	hal_gpt_set_pte( ref_gpt_xp,
2681	vpn,
2682	attr,
2683	ppn );
2684
2685	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 1)
2686	if( (start_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2687	printk("\n[%s] thread[%x,%x] set new PTE in ref gpt for a true page fault\n",
2688	__FUNCTION__, this->process->pid, this->trdid );
2689	#endif
2690
2691	// set PTE in local GPT
2692	// this unlock the PTE
2693	hal_gpt_set_pte( local_gpt_xp,
2694	vpn,
2695	attr,
2696	ppn );
2697
2698	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2699	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2700	#endif
2701
2702	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2703	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2704	printk("\n[%s] thread[%x,%x] handled global pgfault / ppn %x / attr %x / cycle %d\n",
2705	__FUNCTION__, this->process->pid, this->trdid, ppn, attr, end_cycle );
2706	#endif
2707
2708	#if (DEBUG_VMM_HANDLE_PAGE_FAULT & 2)
2709	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2710	hal_vmm_display( XPTR( local_cxy , this->process ) , true );
2711	#endif
2712
2713	#if CONFIG_INSTRUMENTATION_PGFAULTS
2714	uint32_t cost = end_cycle - start_cycle;
2715	this->info.global_pgfault_nr++;
2716	this->info.global_pgfault_cost += cost;
2717	if( cost > this->info.global_pgfault_max ) this->info.global_pgfault_max = cost;
2718	#endif
2719	return EXCP_NON_FATAL;
2720	}
2721	}
2722	}
2723	else // page has been locally mapped by another concurrent thread
2724	{
2725	// unlock the PTE in local GPT
2726	hal_gpt_unlock_pte( local_gpt_xp , vpn );
2727
2728	#if (CONFIG_INSTRUMENTATION_PGFAULTS \|\| DEBUG_VMM_HANDLE_PAGE_FAULT)
2729	uint32_t end_cycle = (uint32_t)hal_get_cycles();
2730	#endif
2731
2732	#if DEBUG_VMM_HANDLE_PAGE_FAULT
2733	if( (end_cycle > DEBUG_VMM_HANDLE_PAGE_FAULT) && (vpn > 0) )
2734	printk("\n[%s] handled by another thread / vpn %x / ppn %x / attr %x / cycle %d\n",
2735	__FUNCTION__, vpn, ppn, attr, end_cycle );
2736	#endif
2737
2738	#if CONFIG_INSTRUMENTATION_PGFAULTS
2739	uint32_t cost = end_cycle - start_cycle;
2740	this->info.false_pgfault_nr++;
2741	this->info.false_pgfault_cost += cost;
2742	if( cost > this->info.false_pgfault_max ) this->info.false_pgfault_max = cost;
2743	#endif
2744	return EXCP_NON_FATAL;
2745	}
2746
2747	} // end vmm_handle_page_fault()
2748
2749	////////////////////////////////////////////
2750	error_t vmm_handle_cow( process_t * process,
2751	vpn_t vpn )
2752	{
2753	vseg_t * vseg; // vseg containing vpn
2754	xptr_t gpt_xp; // extended pointer on GPT (local or reference)
2755	gpt_t * gpt_ptr; // local pointer on GPT (local or reference)
2756	cxy_t gpt_cxy; // GPT cluster identifier
2757	uint32_t old_attr; // current PTE_ATTR value
2758	ppn_t old_ppn; // current PTE_PPN value
2759	uint32_t new_attr; // new PTE_ATTR value
2760	ppn_t new_ppn; // new PTE_PPN value
2761	cxy_t ref_cxy; // reference process cluster
2762	process_t * ref_ptr; // local pointer on reference process
2763	error_t error;
2764
2765	thread_t * this = CURRENT_THREAD;
2766
2767	#if DEBUG_VMM_HANDLE_COW
2768	uint32_t cycle = (uint32_t)hal_get_cycles();
2769	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2770	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
2771	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2772	#endif
2773
2774	#if (DEBUG_VMM_HANDLE_COW & 2)
2775	hal_vmm_display( XPTR( local_cxy , process ) , true );
2776	#endif
2777
2778	// get local vseg
2779	error = vmm_get_vseg( process,
2780	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
2781	&vseg );
2782	if( error )
2783	{
2784	printk("\n[ERROR] in %s : vpn %x in thread[%x,%x] not in a registered vseg\n",
2785	__FUNCTION__, vpn, process->pid, this->trdid );
2786
2787	return EXCP_USER_ERROR;
2788	}
2789
2790	#if DEBUG_VMM_HANDLE_COW
2791	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2792	printk("\n[%s] thread[%x,%x] get vseg %s\n",
2793	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(vseg->type) );
2794	#endif
2795
2796	// get reference process cluster and local pointer
2797	ref_cxy = GET_CXY( process->ref_xp );
2798	ref_ptr = GET_PTR( process->ref_xp );
2799
2800	// build pointers on relevant GPT
2801	// - access only local GPT for a private vseg
2802	// - access reference GPT and all copies for a public vseg
2803	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2804	{
2805	gpt_cxy = local_cxy;
2806	gpt_ptr = &process->vmm.gpt;
2807	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
2808	}
2809	else
2810	{
2811	gpt_cxy = ref_cxy;
2812	gpt_ptr = &ref_ptr->vmm.gpt;
2813	gpt_xp = XPTR( gpt_cxy , gpt_ptr );
2814	}
2815
2816	// lock target PTE in relevant GPT (local or reference)
2817	// and get current PTE value
2818	error = hal_gpt_lock_pte( gpt_xp,
2819	vpn,
2820	&old_attr,
2821	&old_ppn );
2822	if( error )
2823	{
2824	printk("\n[PANIC] in %s : cannot lock PTE in GPT / cxy %x / vpn %x / process %x\n",
2825	__FUNCTION__ , gpt_cxy, vpn , process->pid );
2826
2827	return EXCP_KERNEL_PANIC;
2828	}
2829
2830	#if DEBUG_VMM_HANDLE_COW
2831	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2832	printk("\n[%s] thread[%x,%x] get pte for vpn %x : ppn %x / attr %x\n",
2833	__FUNCTION__, this->process->pid, this->trdid, vpn, old_ppn, old_attr );
2834	#endif
2835
2836	// return user error if COW attribute not set or PTE2 unmapped
2837	if( ((old_attr & GPT_COW) == 0) \|\| ((old_attr & GPT_MAPPED) == 0) )
2838	{
2839	hal_gpt_unlock_pte( gpt_xp , vpn );
2840
2841	return EXCP_USER_ERROR;
2842	}
2843
2844	// get pointers on physical page descriptor
2845	xptr_t page_xp = ppm_ppn2page( old_ppn );
2846	cxy_t page_cxy = GET_CXY( page_xp );
2847	page_t * page_ptr = GET_PTR( page_xp );
2848
2849	// get extended pointers on forks and lock field in page descriptor
2850	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
2851	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
2852
2853	// take lock protecting "forks" counter
2854	remote_busylock_acquire( forks_lock_xp );
2855
2856	// get number of pending forks from page descriptor
2857	uint32_t forks = hal_remote_l32( forks_xp );
2858
2859	#if DEBUG_VMM_HANDLE_COW
2860	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2861	printk("\n[%s] thread[%x,%x] get forks = %d for vpn %x\n",
2862	__FUNCTION__, this->process->pid, this->trdid, forks, vpn );
2863	#endif
2864
2865	if( forks ) // pending fork => allocate a new page, and copy old to new
2866	{
2867	// decrement pending forks counter in page descriptor
2868	hal_remote_atomic_add( forks_xp , -1 );
2869
2870	// release lock protecting "forks" counter
2871	remote_busylock_release( forks_lock_xp );
2872
2873	// allocate a new physical page depending on vseg type
2874	page_xp = vmm_page_allocate( vseg , vpn );
2875
2876	if( page_xp == XPTR_NULL )
2877	{
2878	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2879	__FUNCTION__ , vpn, process->pid );
2880
2881	hal_gpt_unlock_pte( gpt_xp , vpn );
2882
2883	return EXCP_KERNEL_PANIC;
2884	}
2885
2886	// compute allocated page PPN
2887	new_ppn = ppm_page2ppn( page_xp );
2888
2889	#if DEBUG_VMM_HANDLE_COW
2890	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2891	printk("\n[%s] thread[%x,%x] get new ppn %x for vpn %x\n",
2892	__FUNCTION__, this->process->pid, this->trdid, new_ppn, vpn );
2893	#endif
2894
2895	// copy old page content to new page
2896	hal_remote_memcpy( ppm_ppn2base( new_ppn ),
2897	ppm_ppn2base( old_ppn ),
2898	CONFIG_PPM_PAGE_SIZE );
2899
2900	#if DEBUG_VMM_HANDLE_COW
2901	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2902	printk("\n[%s] thread[%x,%x] copied old page to new page\n",
2903	__FUNCTION__, this->process->pid, this->trdid );
2904	#endif
2905
2906	}
2907	else // no pending fork => keep the existing page
2908	{
2909	// release lock protecting "forks" counter
2910	remote_busylock_release( forks_lock_xp );
2911
2912	#if(DEBUG_VMM_HANDLE_COW & 1)
2913	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2914	printk("\n[%s] thread[%x,%x] no pending forks / keep existing PPN %x\n",
2915	__FUNCTION__, this->process->pid, this->trdid, old_ppn );
2916	#endif
2917	new_ppn = old_ppn;
2918	}
2919
2920	// build new_attr : set WRITABLE, reset COW, reset LOCKED
2921	new_attr = (((old_attr \| GPT_WRITABLE) & (~GPT_COW)) & (~GPT_LOCKED));
2922
2923	#if(DEBUG_VMM_HANDLE_COW & 1)
2924	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2925	printk("\n[%s] thread[%x,%x] new_attr %x / new_ppn %x\n",
2926	__FUNCTION__, this->process->pid, this->trdid, new_attr, new_ppn );
2927	#endif
2928
2929	// update the relevant GPT(s)
2930	// - private vseg => update only the local GPT
2931	// - public vseg => update the reference GPT AND all the GPT copies
2932	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2933	{
2934	// set new PTE in local gpt
2935	hal_gpt_set_pte( gpt_xp,
2936	vpn,
2937	new_attr,
2938	new_ppn );
2939	}
2940	else
2941	{
2942	// set new PTE in all GPT copies
2943	vmm_global_update_pte( process,
2944	vpn,
2945	new_attr,
2946	new_ppn );
2947	}
2948
2949	#if DEBUG_VMM_HANDLE_COW
2950	cycle = (uint32_t)hal_get_cycles();
2951	if( (DEBUG_VMM_HANDLE_COW < cycle) && (vpn > 0) )
2952	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2953	__FUNCTION__, this->process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2954	#endif
2955
2956	#if (DEBUG_VMM_HANDLE_COW & 2)
2957	hal_vmm_display( XPTR( local_cxy , process ) , true );
2958	#endif
2959
2960	return EXCP_NON_FATAL;
2961
2962	} // end vmm_handle_cow()
2963

Note: See TracBrowser for help on using the repository browser.

Download in other formats: