Context Navigation

source: trunk/kernel/mm/vmm.c @ 618

Last change on this file since 618 was 617, checked in by alain, 6 years ago
cosmetic
File size: 73.4 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations definition.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016,2017,2018)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_rwlock.h>
35	#include <remote_queuelock.h>
36	#include <list.h>
37	#include <xlist.h>
38	#include <bits.h>
39	#include <process.h>
40	#include <thread.h>
41	#include <vseg.h>
42	#include <cluster.h>
43	#include <scheduler.h>
44	#include <vfs.h>
45	#include <mapper.h>
46	#include <page.h>
47	#include <kmem.h>
48	#include <vmm.h>
49	#include <hal_exception.h>
50
51	//////////////////////////////////////////////////////////////////////////////////
52	// Extern global variables
53	//////////////////////////////////////////////////////////////////////////////////
54
55	extern process_t process_zero; // allocated in cluster.c
56
57	///////////////////////////////////////
58	error_t vmm_init( process_t * process )
59	{
60	error_t error;
61	vseg_t * vseg_kentry;
62	vseg_t * vseg_args;
63	vseg_t * vseg_envs;
64	intptr_t base;
65	intptr_t size;
66	uint32_t i;
67
68	#if DEBUG_VMM_INIT
69	thread_t * this = CURRENT_THREAD;
70	uint32_t cycle = (uint32_t)hal_get_cycles();
71	if( DEBUG_VMM_INIT )
72	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
73	__FUNCTION__ , this->process->pid, this->trdid, process->pid, local_cxy, cycle );
74	#endif
75
76	// get pointer on VMM
77	vmm_t * vmm = &process->vmm;
78
79	// initialize local list of vsegs
80	vmm->vsegs_nr = 0;
81	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
82	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
83
84	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
85	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
86
87	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
88	"no more than 32 threads per cluster for a single process\n");
89
90	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
91	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
92	"STACK zone too small\n");
93
94	// register kentry vseg in VSL
95	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
96	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
97
98	vseg_kentry = vmm_create_vseg( process,
99	VSEG_TYPE_CODE,
100	base,
101	size,
102	0, // file_offset unused
103	0, // file_size unused
104	XPTR_NULL, // mapper_xp unused
105	local_cxy );
106
107	if( vseg_kentry == NULL )
108	{
109	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
110	return -1;
111	}
112
113	vmm->kent_vpn_base = base;
114
115	// register args vseg in VSL
116	base = (CONFIG_VMM_KENTRY_BASE +
117	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
118	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
119
120	vseg_args = vmm_create_vseg( process,
121	VSEG_TYPE_DATA,
122	base,
123	size,
124	0, // file_offset unused
125	0, // file_size unused
126	XPTR_NULL, // mapper_xp unused
127	local_cxy );
128
129	if( vseg_args == NULL )
130	{
131	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
132	return -1;
133	}
134
135	vmm->args_vpn_base = base;
136
137	// register the envs vseg in VSL
138	base = (CONFIG_VMM_KENTRY_BASE +
139	CONFIG_VMM_KENTRY_SIZE +
140	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
141	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
142
143	vseg_envs = vmm_create_vseg( process,
144	VSEG_TYPE_DATA,
145	base,
146	size,
147	0, // file_offset unused
148	0, // file_size unused
149	XPTR_NULL, // mapper_xp unused
150	local_cxy );
151
152	if( vseg_envs == NULL )
153	{
154	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
155	return -1;
156	}
157
158	vmm->envs_vpn_base = base;
159
160	// create GPT (empty)
161	error = hal_gpt_create( &vmm->gpt );
162
163	if( error )
164	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
165
166	// initialize GPT lock
167	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
168
169	// architecture specic GPT initialisation
170	// (For TSAR, identity map the kentry_vseg)
171	error = hal_vmm_init( vmm );
172
173	if( error )
174	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
175
176	// initialize STACK allocator
177	vmm->stack_mgr.bitmap = 0;
178	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
179	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
180
181	// initialize MMAP allocator
182	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
183	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
184	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
185	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
186	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
187
188	// initialize instrumentation counters
189	vmm->pgfault_nr = 0;
190
191	hal_fence();
192
193	#if DEBUG_VMM_INIT
194	cycle = (uint32_t)hal_get_cycles();
195	if( DEBUG_VMM_INIT )
196	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
197	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
198	#endif
199
200	return 0;
201
202	} // end vmm_init()
203
204	//////////////////////////////////////
205	void vmm_display( process_t * process,
206	bool_t mapping )
207	{
208	vmm_t * vmm = &process->vmm;
209	gpt_t * gpt = &vmm->gpt;
210
211	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
212	process->vmm.gpt.ptr , process->pid , local_cxy );
213
214	// get lock protecting the VSL and the GPT
215	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
216	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
217
218	// scan the list of vsegs
219	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
220	xptr_t iter_xp;
221	xptr_t vseg_xp;
222	vseg_t * vseg;
223	XLIST_FOREACH( root_xp , iter_xp )
224	{
225	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
226	vseg = GET_PTR( vseg_xp );
227
228	printk(" - %s : base = %X / size = %X / npages = %d\n",
229	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
230
231	if( mapping )
232	{
233	vpn_t vpn;
234	ppn_t ppn;
235	uint32_t attr;
236	vpn_t base = vseg->vpn_base;
237	vpn_t size = vseg->vpn_size;
238	for( vpn = base ; vpn < (base+size) ; vpn++ )
239	{
240	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
241	if( attr & GPT_MAPPED )
242	{
243	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
244	}
245	}
246	}
247	}
248
249	// release the locks
250	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
251	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
252
253	} // vmm_display()
254
255	//////////////////////////////////////////
256	void vmm_attach_vseg_to_vsl( vmm_t * vmm,
257	vseg_t * vseg )
258	{
259	// build extended pointer on rwlock protecting VSL
260	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
261
262	// get rwlock in write mode
263	remote_rwlock_wr_acquire( lock_xp );
264
265	// update vseg descriptor
266	vseg->vmm = vmm;
267
268	// add vseg in vmm list
269	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
270	XPTR( local_cxy , &vseg->xlist ) );
271
272	// release rwlock in write mode
273	remote_rwlock_wr_release( lock_xp );
274	}
275
276	////////////////////////////////////////////
277	void vmm_detach_vseg_from_vsl( vmm_t * vmm,
278	vseg_t * vseg )
279	{
280	// get vseg type
281	uint32_t type = vseg->type;
282
283	// build extended pointer on rwlock protecting VSL
284	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
285
286	// get rwlock in write mode
287	remote_rwlock_wr_acquire( lock_xp );
288
289	// update vseg descriptor
290	vseg->vmm = NULL;
291
292	// remove vseg from VSL
293	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
294
295	// release rwlock in write mode
296	remote_rwlock_wr_release( lock_xp );
297
298	// release the stack slot to VMM stack allocator if STACK type
299	if( type == VSEG_TYPE_STACK )
300	{
301	// get pointer on stack allocator
302	stack_mgr_t * mgr = &vmm->stack_mgr;
303
304	// compute slot index
305	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
306
307	// update stacks_bitmap
308	busylock_acquire( &mgr->lock );
309	bitmap_clear( &mgr->bitmap , index );
310	busylock_release( &mgr->lock );
311	}
312
313	// release the vseg to VMM mmap allocator if MMAP type
314	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
315	{
316	// get pointer on mmap allocator
317	mmap_mgr_t * mgr = &vmm->mmap_mgr;
318
319	// compute zombi_list index
320	uint32_t index = bits_log2( vseg->vpn_size );
321
322	// update zombi_list
323	busylock_acquire( &mgr->lock );
324	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
325	busylock_release( &mgr->lock );
326	}
327
328	// release physical memory allocated for vseg descriptor if no MMAP type
329	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
330	{
331	vseg_free( vseg );
332	}
333
334	} // end vmm_remove_vseg_from_vsl()
335
336	////////////////////////////////////////////////
337	void vmm_global_update_pte( process_t * process,
338	vpn_t vpn,
339	uint32_t attr,
340	ppn_t ppn )
341	{
342	xlist_entry_t * process_root_ptr;
343	xptr_t process_root_xp;
344	xptr_t process_iter_xp;
345
346	xptr_t remote_process_xp;
347	cxy_t remote_process_cxy;
348	process_t * remote_process_ptr;
349	xptr_t remote_gpt_xp;
350
351	pid_t pid;
352	cxy_t owner_cxy;
353	lpid_t owner_lpid;
354
355	#if DEBUG_VMM_UPDATE_PTE
356	uint32_t cycle = (uint32_t)hal_get_cycles();
357	thread_t * this = CURRENT_THREAD;
358	if( DEBUG_VMM_UPDATE_PTE < cycle )
359	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
360	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
361	#endif
362
363	// check cluster is reference
364	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
365
366	// get extended pointer on root of process copies xlist in owner cluster
367	pid = process->pid;
368	owner_cxy = CXY_FROM_PID( pid );
369	owner_lpid = LPID_FROM_PID( pid );
370	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
371	process_root_xp = XPTR( owner_cxy , process_root_ptr );
372
373	// loop on destination process copies
374	XLIST_FOREACH( process_root_xp , process_iter_xp )
375	{
376	// get cluster and local pointer on remote process
377	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
378	remote_process_ptr = GET_PTR( remote_process_xp );
379	remote_process_cxy = GET_CXY( remote_process_xp );
380
381	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
382	if( DEBUG_VMM_UPDATE_PTE < cycle )
383	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
384	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
385	#endif
386
387	// get extended pointer on remote gpt
388	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
389
390	// update remote GPT
391	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
392	}
393
394	#if DEBUG_VMM_UPDATE_PTE
395	cycle = (uint32_t)hal_get_cycles();
396	if( DEBUG_VMM_UPDATE_PTE < cycle )
397	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
398	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
399	#endif
400
401	} // end vmm_global_update_pte()
402
403	///////////////////////////////////////
404	void vmm_set_cow( process_t * process )
405	{
406	vmm_t * vmm;
407
408	xlist_entry_t * process_root_ptr;
409	xptr_t process_root_xp;
410	xptr_t process_iter_xp;
411
412	xptr_t remote_process_xp;
413	cxy_t remote_process_cxy;
414	process_t * remote_process_ptr;
415	xptr_t remote_gpt_xp;
416
417	xptr_t vseg_root_xp;
418	xptr_t vseg_iter_xp;
419
420	xptr_t vseg_xp;
421	vseg_t * vseg;
422
423	pid_t pid;
424	cxy_t owner_cxy;
425	lpid_t owner_lpid;
426
427	#if DEBUG_VMM_SET_COW
428	uint32_t cycle = (uint32_t)hal_get_cycles();
429	thread_t * this = CURRENT_THREAD;
430	if( DEBUG_VMM_SET_COW < cycle )
431	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
432	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
433	#endif
434
435	// check cluster is reference
436	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
437	"local cluster is not process reference cluster\n");
438
439	// get pointer on reference VMM
440	vmm = &process->vmm;
441
442	// get extended pointer on root of process copies xlist in owner cluster
443	pid = process->pid;
444	owner_cxy = CXY_FROM_PID( pid );
445	owner_lpid = LPID_FROM_PID( pid );
446	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
447	process_root_xp = XPTR( owner_cxy , process_root_ptr );
448
449	// get extended pointer on root of vsegs xlist from reference VMM
450	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
451
452	// loop on destination process copies
453	XLIST_FOREACH( process_root_xp , process_iter_xp )
454	{
455	// get cluster and local pointer on remote process
456	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
457	remote_process_ptr = GET_PTR( remote_process_xp );
458	remote_process_cxy = GET_CXY( remote_process_xp );
459
460	#if (DEBUG_VMM_SET_COW & 1)
461	if( DEBUG_VMM_SET_COW < cycle )
462	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
463	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
464	#endif
465
466	// get extended pointer on remote gpt
467	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
468
469	// loop on vsegs in (local) reference process VSL
470	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
471	{
472	// get pointer on vseg
473	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
474	vseg = GET_PTR( vseg_xp );
475
476	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
477	"all vsegs in reference VSL must be local\n" );
478
479	// get vseg type, base and size
480	uint32_t type = vseg->type;
481	vpn_t vpn_base = vseg->vpn_base;
482	vpn_t vpn_size = vseg->vpn_size;
483
484	#if (DEBUG_VMM_SET_COW & 1)
485	if( DEBUG_VMM_SET_COW < cycle )
486	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
487	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
488	#endif
489	// only DATA, ANON and REMOTE vsegs
490	if( (type == VSEG_TYPE_DATA) \|\|
491	(type == VSEG_TYPE_ANON) \|\|
492	(type == VSEG_TYPE_REMOTE) )
493	{
494	vpn_t vpn;
495	uint32_t attr;
496	ppn_t ppn;
497	xptr_t page_xp;
498	cxy_t page_cxy;
499	page_t * page_ptr;
500	xptr_t forks_xp;
501	xptr_t lock_xp;
502
503	// update flags in remote GPT
504	hal_gpt_set_cow( remote_gpt_xp,
505	vpn_base,
506	vpn_size );
507
508	// atomically increment pending forks counter in physical pages,
509	// for all vseg pages that are mapped in reference cluster
510	if( remote_process_cxy == local_cxy )
511	{
512	// scan all pages in vseg
513	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
514	{
515	// get page attributes and PPN from reference GPT
516	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
517
518	// atomically update pending forks counter if page is mapped
519	if( attr & GPT_MAPPED )
520	{
521	// get pointers and cluster on page descriptor
522	page_xp = ppm_ppn2page( ppn );
523	page_cxy = GET_CXY( page_xp );
524	page_ptr = GET_PTR( page_xp );
525
526	// get extended pointers on "forks" and "lock"
527	forks_xp = XPTR( page_cxy , &page_ptr->forks );
528	lock_xp = XPTR( page_cxy , &page_ptr->lock );
529
530	// take lock protecting "forks" counter
531	remote_busylock_acquire( lock_xp );
532
533	// increment "forks"
534	hal_remote_atomic_add( forks_xp , 1 );
535
536	// release lock protecting "forks" counter
537	remote_busylock_release( lock_xp );
538	}
539	} // end loop on vpn
540	} // end if local
541	} // end if vseg type
542	} // end loop on vsegs
543	} // end loop on process copies
544
545	#if DEBUG_VMM_SET_COW
546	cycle = (uint32_t)hal_get_cycles();
547	if( DEBUG_VMM_SET_COW < cycle )
548	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
549	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
550	#endif
551
552	} // end vmm_set-cow()
553
554	/////////////////////////////////////////////////
555	error_t vmm_fork_copy( process_t * child_process,
556	xptr_t parent_process_xp )
557	{
558	error_t error;
559	cxy_t parent_cxy;
560	process_t * parent_process;
561	vmm_t * parent_vmm;
562	xptr_t parent_lock_xp;
563	vmm_t * child_vmm;
564	xptr_t iter_xp;
565	xptr_t parent_vseg_xp;
566	vseg_t * parent_vseg;
567	vseg_t * child_vseg;
568	uint32_t type;
569	bool_t cow;
570	vpn_t vpn;
571	vpn_t vpn_base;
572	vpn_t vpn_size;
573	xptr_t page_xp; // extended pointer on page descriptor
574	page_t * page_ptr;
575	cxy_t page_cxy;
576	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
577	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
578	xptr_t parent_root_xp;
579	bool_t mapped;
580	ppn_t ppn;
581
582	#if DEBUG_VMM_FORK_COPY
583	uint32_t cycle = (uint32_t)hal_get_cycles();
584	thread_t * this = CURRENT_THREAD;
585	if( DEBUG_VMM_FORK_COPY < cycle )
586	printk("\n[%s] thread %x enter / cycle %d\n",
587	__FUNCTION__ , this->process->pid, this->trdid, cycle );
588	#endif
589
590	// get parent process cluster and local pointer
591	parent_cxy = GET_CXY( parent_process_xp );
592	parent_process = GET_PTR( parent_process_xp );
593
594	// get local pointers on parent and child VMM
595	parent_vmm = &parent_process->vmm;
596	child_vmm = &child_process->vmm;
597
598	// get extended pointer on lock protecting the parent VSL
599	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
600
601	// initialize the lock protecting the child VSL
602	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
603
604	// initialize the child VSL as empty
605	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
606	child_vmm->vsegs_nr = 0;
607
608	// create child GPT
609	error = hal_gpt_create( &child_vmm->gpt );
610
611	if( error )
612	{
613	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
614	return -1;
615	}
616
617	// build extended pointer on parent VSL
618	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
619
620	// take the lock protecting the parent VSL in read mode
621	remote_rwlock_rd_acquire( parent_lock_xp );
622
623	// loop on parent VSL xlist
624	XLIST_FOREACH( parent_root_xp , iter_xp )
625	{
626	// get local and extended pointers on current parent vseg
627	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
628	parent_vseg = GET_PTR( parent_vseg_xp );
629
630	// get vseg type
631	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
632
633	#if DEBUG_VMM_FORK_COPY
634	cycle = (uint32_t)hal_get_cycles();
635	if( DEBUG_VMM_FORK_COPY < cycle )
636	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
637	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
638	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
639	#endif
640
641	// all parent vsegs - but STACK - must be copied in child VSL
642	if( type != VSEG_TYPE_STACK )
643	{
644	// allocate memory for a new child vseg
645	child_vseg = vseg_alloc();
646	if( child_vseg == NULL ) // release all allocated vsegs
647	{
648	vmm_destroy( child_process );
649	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
650	return -1;
651	}
652
653	// copy parent vseg to child vseg
654	vseg_init_from_ref( child_vseg , parent_vseg_xp );
655
656	// register child vseg in child VSL
657	vmm_attach_vseg_to_vsl( child_vmm , child_vseg );
658
659	#if DEBUG_VMM_FORK_COPY
660	cycle = (uint32_t)hal_get_cycles();
661	if( DEBUG_VMM_FORK_COPY < cycle )
662	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
663	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
664	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
665	#endif
666
667	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
668	if( type != VSEG_TYPE_CODE )
669	{
670	// activate the COW for DATA, MMAP, REMOTE vsegs only
671	cow = ( type != VSEG_TYPE_FILE );
672
673	vpn_base = child_vseg->vpn_base;
674	vpn_size = child_vseg->vpn_size;
675
676	// scan pages in parent vseg
677	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
678	{
679	error = hal_gpt_pte_copy( &child_vmm->gpt,
680	XPTR( parent_cxy , &parent_vmm->gpt ),
681	vpn,
682	cow,
683	&ppn,
684	&mapped );
685	if( error )
686	{
687	vmm_destroy( child_process );
688	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
689	return -1;
690	}
691
692	// increment pending forks counter in page if mapped
693	if( mapped )
694	{
695	// get pointers and cluster on page descriptor
696	page_xp = ppm_ppn2page( ppn );
697	page_cxy = GET_CXY( page_xp );
698	page_ptr = GET_PTR( page_xp );
699
700	// get extended pointers on "forks" and "lock"
701	forks_xp = XPTR( page_cxy , &page_ptr->forks );
702	lock_xp = XPTR( page_cxy , &page_ptr->lock );
703
704	// get lock protecting "forks" counter
705	remote_busylock_acquire( lock_xp );
706
707	// increment "forks"
708	hal_remote_atomic_add( forks_xp , 1 );
709
710	// release lock protecting "forks" counter
711	remote_busylock_release( lock_xp );
712
713	#if DEBUG_VMM_FORK_COPY
714	cycle = (uint32_t)hal_get_cycles();
715	if( DEBUG_VMM_FORK_COPY < cycle )
716	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
717	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
718	#endif
719	}
720	}
721	} // end if no code & no stack
722	} // end if no stack
723	} // end loop on vsegs
724
725	// release the parent VSL lock in read mode
726	remote_rwlock_rd_release( parent_lock_xp );
727
728	// initialize child GPT (architecture specic)
729	// => For TSAR, identity map the kentry_vseg
730	error = hal_vmm_init( child_vmm );
731
732	if( error )
733	{
734	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
735	return -1;
736	}
737
738	// initialize the child VMM STACK allocator
739	child_vmm->stack_mgr.bitmap = 0;
740	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
741
742	// initialize the child VMM MMAP allocator
743	uint32_t i;
744	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
745	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
746	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
747	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
748
749	// initialize instrumentation counters
750	child_vmm->pgfault_nr = 0;
751
752	// copy base addresses from parent VMM to child VMM
753	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
754	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
755	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
756	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
757	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
758	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
759
760	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
761
762	hal_fence();
763
764	#if DEBUG_VMM_FORK_COPY
765	cycle = (uint32_t)hal_get_cycles();
766	if( DEBUG_VMM_FORK_COPY < cycle )
767	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
768	__FUNCTION__ , this->process->pid, this->trdid , cycle );
769	#endif
770
771	return 0;
772
773	} // vmm_fork_copy()
774
775	///////////////////////////////////////
776	void vmm_destroy( process_t * process )
777	{
778	xptr_t vseg_xp;
779	vseg_t * vseg;
780
781	#if DEBUG_VMM_DESTROY
782	uint32_t cycle = (uint32_t)hal_get_cycles();
783	thread_t * this = CURRENT_THREAD;
784	if( DEBUG_VMM_DESTROY < cycle )
785	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
786	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
787	#endif
788
789	#if (DEBUG_VMM_DESTROY & 1 )
790	if( DEBUG_VMM_DESTROY < cycle )
791	vmm_display( process , true );
792	#endif
793
794	// get pointer on local VMM
795	vmm_t * vmm = &process->vmm;
796
797	// get extended pointer on VSL root and VSL lock
798	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
799
800	// scan the VSL to delete all registered vsegs
801	// (don't use a FOREACH for item deletion in xlist)
802	while( !xlist_is_empty( root_xp ) )
803	{
804	// get pointer on first vseg in VSL
805	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
806	vseg = GET_PTR( vseg_xp );
807
808	// delete vseg and release physical pages
809	vmm_delete_vseg( process->pid , vseg->min );
810
811	#if( DEBUG_VMM_DESTROY & 1 )
812	if( DEBUG_VMM_DESTROY < cycle )
813	printk("\n[%s] %s vseg deleted / vpn_base %x / vpn_size %d\n",
814	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
815	#endif
816
817	}
818
819	// remove all vsegs from zombi_lists in MMAP allocator
820	uint32_t i;
821	for( i = 0 ; i<32 ; i++ )
822	{
823	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
824	{
825	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
826
827	#if( DEBUG_VMM_DESTROY & 1 )
828	if( DEBUG_VMM_DESTROY < cycle )
829	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
830	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
831	#endif
832	// clean vseg descriptor
833	vseg->vmm = NULL;
834
835	// remove vseg from xlist
836	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
837
838	// release vseg descriptor
839	vseg_free( vseg );
840
841	#if( DEBUG_VMM_DESTROY & 1 )
842	if( DEBUG_VMM_DESTROY < cycle )
843	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
844	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
845	#endif
846	}
847	}
848
849	// release memory allocated to the GPT itself
850	hal_gpt_destroy( &vmm->gpt );
851
852	#if DEBUG_VMM_DESTROY
853	cycle = (uint32_t)hal_get_cycles();
854	if( DEBUG_VMM_DESTROY < cycle )
855	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
856	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
857	#endif
858
859	} // end vmm_destroy()
860
861	/////////////////////////////////////////////////
862	vseg_t * vmm_check_conflict( process_t * process,
863	vpn_t vpn_base,
864	vpn_t vpn_size )
865	{
866	vmm_t * vmm = &process->vmm;
867
868	// scan the VSL
869	vseg_t * vseg;
870	xptr_t iter_xp;
871	xptr_t vseg_xp;
872	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
873
874	XLIST_FOREACH( root_xp , iter_xp )
875	{
876	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
877	vseg = GET_PTR( vseg_xp );
878
879	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
880	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
881	}
882	return NULL;
883
884	} // end vmm_check_conflict()
885
886	////////////////////////////////////////////////////////////////////////////////////////////
887	// This static function is called by the vmm_create_vseg() function, and implements
888	// the VMM stack_vseg specific allocator.
889	////////////////////////////////////////////////////////////////////////////////////////////
890	// @ vmm : pointer on VMM.
891	// @ vpn_base : (return value) first allocated page
892	// @ vpn_size : (return value) number of allocated pages
893	////////////////////////////////////////////////////////////////////////////////////////////
894	static error_t vmm_stack_alloc( vmm_t * vmm,
895	vpn_t * vpn_base,
896	vpn_t * vpn_size )
897	{
898	// get stack allocator pointer
899	stack_mgr_t * mgr = &vmm->stack_mgr;
900
901	// get lock on stack allocator
902	busylock_acquire( &mgr->lock );
903
904	// get first free slot index in bitmap
905	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
906	if( (index < 0) \|\| (index > 31) )
907	{
908	busylock_release( &mgr->lock );
909	return 0xFFFFFFFF;
910	}
911
912	// update bitmap
913	bitmap_set( &mgr->bitmap , index );
914
915	// release lock on stack allocator
916	busylock_release( &mgr->lock );
917
918	// returns vpn_base, vpn_size (one page non allocated)
919	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
920	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
921	return 0;
922
923	} // end vmm_stack_alloc()
924
925	////////////////////////////////////////////////////////////////////////////////////////////
926	// This static function is called by the vmm_create_vseg() function, and implements
927	// the VMM MMAP specific allocator.
928	////////////////////////////////////////////////////////////////////////////////////////////
929	// @ vmm : [in] pointer on VMM.
930	// @ npages : [in] requested number of pages.
931	// @ vpn_base : [out] first allocated page.
932	// @ vpn_size : [out] actual number of allocated pages.
933	////////////////////////////////////////////////////////////////////////////////////////////
934	static error_t vmm_mmap_alloc( vmm_t * vmm,
935	vpn_t npages,
936	vpn_t * vpn_base,
937	vpn_t * vpn_size )
938	{
939	uint32_t index;
940	vseg_t * vseg;
941	vpn_t base;
942	vpn_t size;
943	vpn_t free;
944
945	#if DEBUG_VMM_MMAP_ALLOC
946	thread_t * this = CURRENT_THREAD;
947	uint32_t cycle = (uint32_t)hal_get_cycles();
948	if( DEBUG_VMM_MMAP_ALLOC < cycle )
949	printk("\n[%s] thread[%x,%x] enter / cycle %d\n",
950	__FUNCTION__, this->process->pid, this->trdid, cycle );
951	#endif
952
953	// vseg size must be power of 2
954	// compute actual size and index in zombi_list array
955	size = POW2_ROUNDUP( npages );
956	index = bits_log2( size );
957
958	// get mmap allocator pointer
959	mmap_mgr_t * mgr = &vmm->mmap_mgr;
960
961	// get lock on mmap allocator
962	busylock_acquire( &mgr->lock );
963
964	// get vseg from zombi_list or from mmap zone
965	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
966	{
967	// check overflow
968	free = mgr->first_free_vpn;
969	if( (free + size) > mgr->vpn_size ) return -1;
970
971	// update MMAP allocator
972	mgr->first_free_vpn += size;
973
974	// compute base
975	base = free;
976	}
977	else // from zombi_list
978	{
979	// get pointer on zombi vseg from zombi_list
980	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
981
982	// remove vseg from free-list
983	list_unlink( &vseg->zlist );
984
985	// compute base
986	base = vseg->vpn_base;
987	}
988
989	// release lock on mmap allocator
990	busylock_release( &mgr->lock );
991
992	#if DEBUG_VMM_MMAP_ALLOC
993	cycle = (uint32_t)hal_get_cycles();
994	if( DEBUG_VMM_DESTROY < cycle )
995	printk("\n[%s] thread[%x,%x] exit / vpn_base %x / vpn_size %x / cycle %d\n",
996	__FUNCTION__, this->process->pid, this->trdid, base, size, cycle );
997	#endif
998
999	// returns vpn_base, vpn_size
1000	*vpn_base = base;
1001	*vpn_size = size;
1002	return 0;
1003
1004	} // end vmm_mmap_alloc()
1005
1006	////////////////////////////////////////////////
1007	vseg_t * vmm_create_vseg( process_t * process,
1008	vseg_type_t type,
1009	intptr_t base,
1010	uint32_t size,
1011	uint32_t file_offset,
1012	uint32_t file_size,
1013	xptr_t mapper_xp,
1014	cxy_t cxy )
1015	{
1016	vseg_t * vseg; // created vseg pointer
1017	vpn_t vpn_base; // first page index
1018	vpn_t vpn_size; // number of pages covered by vseg
1019	error_t error;
1020
1021	#if DEBUG_VMM_CREATE_VSEG
1022	thread_t * this = CURRENT_THREAD;
1023	uint32_t cycle = (uint32_t)hal_get_cycles();
1024	if( DEBUG_VMM_CREATE_VSEG < cycle )
1025	printk("\n[%s] thread[%x,%x] enter for process %x / %s / cxy %x / cycle %d\n",
1026	__FUNCTION__, this->process->pid, this->trdid, process->pid, vseg_type_str(type), cxy, cycle );
1027	#endif
1028
1029	// get pointer on VMM
1030	vmm_t * vmm = &process->vmm;
1031
1032	// compute base, size, vpn_base, vpn_size, depending on vseg type
1033	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
1034
1035	if( type == VSEG_TYPE_STACK )
1036	{
1037	// get vpn_base and vpn_size from STACK allocator
1038	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
1039	if( error )
1040	{
1041	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
1042	__FUNCTION__ , process->pid , local_cxy );
1043	return NULL;
1044	}
1045
1046	// compute vseg base and size from vpn_base and vpn_size
1047	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1048	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
1049	}
1050	else if( type == VSEG_TYPE_FILE )
1051	{
1052	// compute page index (in mapper) for first byte
1053	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
1054
1055	// compute page index (in mapper) for last byte
1056	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1057
1058	// compute offset in first page
1059	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1060
1061	// compute number of pages required in virtual space
1062	vpn_t npages = vpn_max - vpn_min + 1;
1063
1064	// get vpn_base and vpn_size from MMAP allocator
1065	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1066	if( error )
1067	{
1068	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1069	__FUNCTION__ , process->pid , local_cxy );
1070	return NULL;
1071	}
1072
1073	// set the vseg base (not always aligned for FILE)
1074	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1075	}
1076	else if( (type == VSEG_TYPE_ANON) \|\|
1077	(type == VSEG_TYPE_REMOTE) )
1078	{
1079	// compute number of required pages in virtual space
1080	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1081	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1082
1083	// get vpn_base and vpn_size from MMAP allocator
1084	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1085	if( error )
1086	{
1087	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1088	__FUNCTION__ , process->pid , local_cxy );
1089	return NULL;
1090	}
1091
1092	// set vseg base (always aligned for ANON or REMOTE)
1093	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1094	}
1095	else // VSEG_TYPE_DATA or VSEG_TYPE_CODE
1096	{
1097	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1098	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1099
1100	vpn_base = vpn_min;
1101	vpn_size = vpn_max - vpn_min + 1;
1102	}
1103
1104	// check collisions
1105	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1106	if( vseg != NULL )
1107	{
1108	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base %x / vpn_size %x]\n"
1109	" overlap existing vseg [vpn_base %x / vpn_size %x]\n",
1110	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1111	return NULL;
1112	}
1113
1114	// allocate physical memory for vseg descriptor
1115	vseg = vseg_alloc();
1116	if( vseg == NULL )
1117	{
1118	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1119	__FUNCTION__ , process->pid );
1120	return NULL;
1121	}
1122
1123	#if DEBUG_VMM_CREATE_VSEG
1124	if( DEBUG_VMM_CREATE_VSEG < cycle )
1125	printk("\n[%s] thread[%x,%x] : base %x / size %x / vpn_base %x / vpn_size %x\n",
1126	__FUNCTION__, this->process->pid, this->trdid, base, size, vpn_base, vpn_size );
1127	#endif
1128
1129	// initialize vseg descriptor
1130	vseg_init( vseg,
1131	type,
1132	base,
1133	size,
1134	vpn_base,
1135	vpn_size,
1136	file_offset,
1137	file_size,
1138	mapper_xp,
1139	cxy );
1140
1141	// attach vseg to VSL
1142	vmm_attach_vseg_to_vsl( vmm , vseg );
1143
1144	#if DEBUG_VMM_CREATE_VSEG
1145	cycle = (uint32_t)hal_get_cycles();
1146	if( DEBUG_VMM_CREATE_VSEG < cycle )
1147	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
1148	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
1149	#endif
1150
1151	return vseg;
1152
1153	} // vmm_create_vseg()
1154
1155	///////////////////////////////////
1156	void vmm_delete_vseg( pid_t pid,
1157	intptr_t vaddr )
1158	{
1159	process_t * process; // local pointer on local process
1160	vmm_t * vmm; // local pointer on local process VMM
1161	vseg_t * vseg; // local pointer on local vseg containing vaddr
1162	gpt_t * gpt; // local pointer on local process GPT
1163	vpn_t vpn; // VPN of current PTE
1164	vpn_t vpn_min; // VPN of first PTE
1165	vpn_t vpn_max; // VPN of last PTE (excluded)
1166	ppn_t ppn; // current PTE ppn value
1167	uint32_t attr; // current PTE attributes
1168	kmem_req_t req; // request to release memory
1169	xptr_t page_xp; // extended pointer on page descriptor
1170	cxy_t page_cxy; // page descriptor cluster
1171	page_t * page_ptr; // page descriptor pointer
1172	xptr_t forks_xp; // extended pointer on pending forks counter
1173	xptr_t lock_xp; // extended pointer on lock protecting forks counter
1174	uint32_t forks; // actual number of pendinf forks
1175
1176	#if DEBUG_VMM_DELETE_VSEG
1177	uint32_t cycle = (uint32_t)hal_get_cycles();
1178	thread_t * this = CURRENT_THREAD;
1179	if( DEBUG_VMM_DELETE_VSEG < cycle )
1180	printk("\n[%s] thread[%x,%x] enter / process %x / vaddr %x / cycle %d\n",
1181	__FUNCTION__, this->process->pid, this->trdid, pid, vaddr, cycle );
1182	#endif
1183
1184	// get local pointer on local process descriptor
1185	process = cluster_get_local_process_from_pid( pid );
1186
1187	if( process == NULL ) return;
1188
1189	// get pointers on local process VMM an GPT
1190	vmm = &process->vmm;
1191	gpt = &process->vmm.gpt;
1192
1193	// get local pointer on vseg containing vaddr
1194	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1195
1196	if( vseg == NULL ) return;
1197
1198	// loop to invalidate all vseg PTEs in GPT
1199	vpn_min = vseg->vpn_base;
1200	vpn_max = vpn_min + vseg->vpn_size;
1201	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1202	{
1203	// get GPT entry
1204	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
1205
1206	if( attr & GPT_MAPPED ) // entry is mapped
1207	{
1208
1209	#if( DEBUG_VMM_DELETE_VSEG & 1 )
1210	if( DEBUG_VMM_DELETE_VSEG < cycle )
1211	printk("- unmap vpn %x / ppn %x / vseg %s \n" , vpn , ppn, vseg_type_str(vseg->type) );
1212	#endif
1213
1214	// check small page
1215	assert( (attr & GPT_SMALL) , "an user vseg must use small pages" );
1216
1217	// unmap GPT entry in local GPT
1218	hal_gpt_reset_pte( gpt , vpn );
1219
1220	// handle pending forks counter if
1221	// 1) not identity mapped
1222	// 2) reference cluster
1223	if( ((vseg->flags & VSEG_IDENT) == 0) &&
1224	(GET_CXY( process->ref_xp ) == local_cxy) )
1225	{
1226	// get extended pointer on physical page descriptor
1227	page_xp = ppm_ppn2page( ppn );
1228	page_cxy = GET_CXY( page_xp );
1229	page_ptr = GET_PTR( page_xp );
1230
1231	// get extended pointers on forks and lock fields
1232	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1233	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1234
1235	// get pending forks counter
1236	forks = hal_remote_l32( forks_xp );
1237
1238	if( forks ) // decrement pending forks counter
1239	{
1240	hal_remote_atomic_add( forks_xp , -1 );
1241	}
1242	else // release physical page to relevant cluster
1243	{
1244	if( page_cxy == local_cxy ) // local cluster
1245	{
1246	req.type = KMEM_PAGE;
1247	req.ptr = page_ptr;
1248	kmem_free( &req );
1249	}
1250	else // remote cluster
1251	{
1252	rpc_pmem_release_pages_client( page_cxy , page_ptr );
1253	}
1254
1255	#if( DEBUG_VMM_DELETE_VSEG & 1 )
1256	if( DEBUG_VMM_DELETE_VSEG < cycle )
1257	printk("- release ppn %x\n", ppn );
1258	#endif
1259	}
1260	}
1261	}
1262	}
1263
1264	// remove vseg from VSL and release vseg descriptor (if not MMAP)
1265	vmm_detach_vseg_from_vsl( vmm , vseg );
1266
1267	#if DEBUG_VMM_DELETE_VSEG
1268	cycle = (uint32_t)hal_get_cycles();
1269	if( DEBUG_VMM_DELETE_VSEG < cycle )
1270	printk("\n[%s] thread[%x,%x] exit / process %x / vseg %s / base %x / cycle %d\n",
1271	__FUNCTION__, this->process->pid, this->trdid, pid, vseg_type_str(vseg->type), vaddr, cycle );
1272	#endif
1273
1274	} // end vmm_delete_vseg()
1275
1276	/////////////////////////////////////////////
1277	vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
1278	intptr_t vaddr )
1279	{
1280	xptr_t iter_xp;
1281	xptr_t vseg_xp;
1282	vseg_t * vseg;
1283
1284	// get extended pointers on VSL lock and root
1285	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1286	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1287
1288	// get lock protecting the VSL
1289	remote_rwlock_rd_acquire( lock_xp );
1290
1291	// scan the list of vsegs in VSL
1292	XLIST_FOREACH( root_xp , iter_xp )
1293	{
1294	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1295	vseg = GET_PTR( vseg_xp );
1296
1297	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
1298	{
1299	// return success
1300	remote_rwlock_rd_release( lock_xp );
1301	return vseg;
1302	}
1303	}
1304
1305	// return failure
1306	remote_rwlock_rd_release( lock_xp );
1307	return NULL;
1308
1309	} // end vmm_vseg_from_vaddr()
1310
1311	/////////////////////////////////////////////
1312	error_t vmm_resize_vseg( process_t * process,
1313	intptr_t base,
1314	intptr_t size )
1315	{
1316	error_t error;
1317	vseg_t * new;
1318	vpn_t vpn_min;
1319	vpn_t vpn_max;
1320
1321	// get pointer on process VMM
1322	vmm_t * vmm = &process->vmm;
1323
1324	intptr_t addr_min = base;
1325	intptr_t addr_max = base + size;
1326
1327	// get pointer on vseg
1328	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
1329
1330	if( vseg == NULL) return EINVAL;
1331
1332	// get extended pointer on VSL lock
1333	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1334
1335	// get lock protecting VSL
1336	remote_rwlock_wr_acquire( lock_xp );
1337
1338	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // not included in vseg
1339	{
1340	error = -1;
1341	}
1342	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be deleted
1343	{
1344	vmm_delete_vseg( process->pid , vseg->min );
1345	error = 0;
1346	}
1347	else if( vseg->min == addr_min ) // vseg must be resized
1348	{
1349	// update vseg base address
1350	vseg->min = addr_max;
1351
1352	// update vpn_base and vpn_size
1353	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1354	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1355	vseg->vpn_base = vpn_min;
1356	vseg->vpn_size = vpn_max - vpn_min + 1;
1357	error = 0;
1358	}
1359	else if( vseg->max == addr_max ) // vseg must be resized
1360	{
1361	// update vseg max address
1362	vseg->max = addr_min;
1363
1364	// update vpn_base and vpn_size
1365	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1366	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1367	vseg->vpn_base = vpn_min;
1368	vseg->vpn_size = vpn_max - vpn_min + 1;
1369	error = 0;
1370	}
1371	else // vseg cut in three regions
1372	{
1373	// resize existing vseg
1374	vseg->max = addr_min;
1375
1376	// update vpn_base and vpn_size
1377	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1378	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1379	vseg->vpn_base = vpn_min;
1380	vseg->vpn_size = vpn_max - vpn_min + 1;
1381
1382	// create new vseg
1383	new = vmm_create_vseg( process,
1384	vseg->type,
1385	addr_min,
1386	(vseg->max - addr_max),
1387	vseg->file_offset,
1388	vseg->file_size,
1389	vseg->mapper_xp,
1390	vseg->cxy );
1391
1392	if( new == NULL ) error = EINVAL;
1393	else error = 0;
1394	}
1395
1396	// release VMM lock
1397	remote_rwlock_wr_release( lock_xp );
1398
1399	return error;
1400
1401	} // vmm_resize_vseg()
1402
1403	///////////////////////////////////////////
1404	error_t vmm_get_vseg( process_t * process,
1405	intptr_t vaddr,
1406	vseg_t ** found_vseg )
1407	{
1408	xptr_t vseg_xp;
1409	vseg_t * vseg;
1410	vmm_t * vmm;
1411	error_t error;
1412
1413	// get pointer on local VMM
1414	vmm = &process->vmm;
1415
1416	// try to get vseg from local VMM
1417	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1418
1419	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
1420	{
1421	// get extended pointer on reference process
1422	xptr_t ref_xp = process->ref_xp;
1423
1424	// get cluster and local pointer on reference process
1425	cxy_t ref_cxy = GET_CXY( ref_xp );
1426	process_t * ref_ptr = GET_PTR( ref_xp );
1427
1428	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
1429
1430	// get extended pointer on reference vseg
1431	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
1432
1433	if( error ) return -1; // vseg not found => illegal user vaddr
1434
1435	// allocate a vseg in local cluster
1436	vseg = vseg_alloc();
1437
1438	if( vseg == NULL ) return -1; // cannot allocate a local vseg
1439
1440	// initialise local vseg from reference
1441	vseg_init_from_ref( vseg , vseg_xp );
1442
1443	// register local vseg in local VSL
1444	vmm_attach_vseg_to_vsl( vmm , vseg );
1445	}
1446
1447	// success
1448	*found_vseg = vseg;
1449	return 0;
1450
1451	} // end vmm_get_vseg()
1452
1453	//////////////////////////////////////////////////////////////////////////////////////
1454	// This static function compute the target cluster to allocate a physical page
1455	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
1456	// and returns an extended pointer on the allocated page descriptor.
1457	// It can be called by a thread running in any cluster.
1458	// The vseg cannot have the FILE type.
1459	//////////////////////////////////////////////////////////////////////////////////////
1460	static xptr_t vmm_page_allocate( vseg_t * vseg,
1461	vpn_t vpn )
1462	{
1463
1464	#if DEBUG_VMM_ALLOCATE_PAGE
1465	uint32_t cycle = (uint32_t)hal_get_cycles();
1466	thread_t * this = CURRENT_THREAD;
1467	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1468	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
1469	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
1470	#endif
1471
1472	page_t * page_ptr;
1473	cxy_t page_cxy;
1474	kmem_req_t req;
1475	uint32_t index;
1476
1477	uint32_t type = vseg->type;
1478	uint32_t flags = vseg->flags;
1479	uint32_t x_size = LOCAL_CLUSTER->x_size;
1480	uint32_t y_size = LOCAL_CLUSTER->y_size;
1481
1482	// check vseg type
1483	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1484
1485	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1486	{
1487	index = vpn & ((x_size * y_size) - 1);
1488	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1489
1490	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1491	if ( cluster_is_active( page_cxy ) == false )
1492	{
1493	page_cxy = cluster_random_select();
1494	}
1495	}
1496	else // other cases => cxy specified in vseg
1497	{
1498	page_cxy = vseg->cxy;
1499	}
1500
1501	// allocate a physical page from target cluster
1502	if( page_cxy == local_cxy ) // target cluster is the local cluster
1503	{
1504	req.type = KMEM_PAGE;
1505	req.size = 0;
1506	req.flags = AF_NONE;
1507	page_ptr = (page_t *)kmem_alloc( &req );
1508	}
1509	else // target cluster is not the local cluster
1510	{
1511	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
1512	}
1513
1514	#if DEBUG_VMM_ALLOCATE_PAGE
1515	cycle = (uint32_t)hal_get_cycles();
1516	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1517	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
1518	__FUNCTION__ , this->process->pid, this->trdid, vpn,
1519	ppm_page2ppn( XPTR( page_cxy , page_ptr ) , cycle );
1520	#endif
1521
1522	if( page_ptr == NULL ) return XPTR_NULL;
1523	else return XPTR( page_cxy , page_ptr );
1524
1525	} // end vmm_page_allocate()
1526
1527	////////////////////////////////////////
1528	error_t vmm_get_one_ppn( vseg_t * vseg,
1529	vpn_t vpn,
1530	ppn_t * ppn )
1531	{
1532	error_t error;
1533	xptr_t page_xp; // extended pointer on physical page descriptor
1534	uint32_t page_id; // missing page index in vseg mapper
1535	uint32_t type; // vseg type;
1536
1537	type = vseg->type;
1538	page_id = vpn - vseg->vpn_base;
1539
1540	#if DEBUG_VMM_GET_ONE_PPN
1541	uint32_t cycle = (uint32_t)hal_get_cycles();
1542	thread_t * this = CURRENT_THREAD;
1543	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1544	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
1545	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
1546	#endif
1547
1548	// FILE type : get the physical page from the file mapper
1549	if( type == VSEG_TYPE_FILE )
1550	{
1551	// get extended pointer on mapper
1552	xptr_t mapper_xp = vseg->mapper_xp;
1553
1554	assert( (mapper_xp != XPTR_NULL),
1555	"mapper not defined for a FILE vseg\n" );
1556
1557	// get extended pointer on page descriptor
1558	page_xp = mapper_remote_get_page( mapper_xp , page_id );
1559
1560	if ( page_xp == XPTR_NULL ) return EINVAL;
1561	}
1562
1563	// Other types : allocate a physical page from target cluster,
1564	// as defined by vseg type and vpn value
1565	else
1566	{
1567	// allocate one physical page
1568	page_xp = vmm_page_allocate( vseg , vpn );
1569
1570	if( page_xp == XPTR_NULL ) return ENOMEM;
1571
1572	// initialise missing page from .elf file mapper for DATA and CODE types
1573	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1574	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1575	{
1576	// get extended pointer on mapper
1577	xptr_t mapper_xp = vseg->mapper_xp;
1578
1579	assert( (mapper_xp != XPTR_NULL),
1580	"mapper not defined for a CODE or DATA vseg\n" );
1581
1582	// compute missing page offset in vseg
1583	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
1584
1585	// compute missing page offset in .elf file
1586	uint32_t elf_offset = vseg->file_offset + offset;
1587
1588	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1589	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1590	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
1591	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
1592	#endif
1593	// compute extended pointer on page base
1594	xptr_t base_xp = ppm_page2base( page_xp );
1595
1596	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1597	uint32_t file_size = vseg->file_size;
1598
1599	if( file_size < offset ) // missing page fully in BSS
1600	{
1601
1602	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1603	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1604	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
1605	__FUNCTION__, this->process->pid, this->trdid, vpn );
1606	#endif
1607	if( GET_CXY( page_xp ) == local_cxy )
1608	{
1609	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1610	}
1611	else
1612	{
1613	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
1614	}
1615	}
1616	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
1617	{
1618
1619	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1620	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1621	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
1622	__FUNCTION__, this->process->pid, this->trdid, vpn );
1623	#endif
1624	error = mapper_move_kernel( mapper_xp,
1625	true, // to_buffer
1626	elf_offset,
1627	base_xp,
1628	CONFIG_PPM_PAGE_SIZE );
1629	if( error ) return EINVAL;
1630	}
1631	else // both in mapper and in BSS :
1632	// - (file_size - offset) bytes from mapper
1633	// - (page_size + offset - file_size) bytes from BSS
1634	{
1635
1636	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1637	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1638	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
1639	" %d bytes from mapper / %d bytes from BSS\n",
1640	__FUNCTION__, this->process->pid, this->trdid, vpn,
1641	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
1642	#endif
1643	// initialize mapper part
1644	error = mapper_move_kernel( mapper_xp,
1645	true, // to buffer
1646	elf_offset,
1647	base_xp,
1648	file_size - offset );
1649	if( error ) return EINVAL;
1650
1651	// initialize BSS part
1652	if( GET_CXY( page_xp ) == local_cxy )
1653	{
1654	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
1655	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1656	}
1657	else
1658	{
1659	hal_remote_memset( base_xp + file_size - offset , 0 ,
1660	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1661	}
1662	}
1663	} // end initialisation for CODE or DATA types
1664	}
1665
1666	// return ppn
1667	*ppn = ppm_page2ppn( page_xp );
1668
1669	#if DEBUG_VMM_GET_ONE_PPN
1670	cycle = (uint32_t)hal_get_cycles();
1671	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1672	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
1673	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
1674	#endif
1675
1676	return 0;
1677
1678	} // end vmm_get_one_ppn()
1679
1680	///////////////////////////////////////////////////
1681	error_t vmm_handle_page_fault( process_t * process,
1682	vpn_t vpn )
1683	{
1684	vseg_t * vseg; // vseg containing vpn
1685	uint32_t new_attr; // new PTE_ATTR value
1686	ppn_t new_ppn; // new PTE_PPN value
1687	uint32_t ref_attr; // PTE_ATTR value in reference GPT
1688	ppn_t ref_ppn; // PTE_PPN value in reference GPT
1689	cxy_t ref_cxy; // reference cluster for missing vpn
1690	process_t * ref_ptr; // reference process for missing vpn
1691	xptr_t local_gpt_xp; // extended pointer on local GPT
1692	xptr_t local_lock_xp; // extended pointer on local GPT lock
1693	xptr_t ref_gpt_xp; // extended pointer on reference GPT
1694	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
1695	error_t error; // value returned by called functions
1696
1697	// get local vseg (access to reference VSL can be required)
1698	error = vmm_get_vseg( process,
1699	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1700	&vseg );
1701	if( error )
1702	{
1703	printk("\n[ERROR] in %s : vpn %x in process %x not in a registered vseg\n",
1704	__FUNCTION__ , vpn , process->pid );
1705
1706	return EXCP_USER_ERROR;
1707	}
1708
1709	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1710	uint32_t cycle = (uint32_t)hal_get_cycles();
1711	thread_t * this = CURRENT_THREAD;
1712	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1713	printk("\n[%s] threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
1714	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
1715	#endif
1716
1717	//////////////// private vseg => access only the local GPT
1718	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1719	{
1720	// build extended pointer on local GPT and local GPT lock
1721	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1722	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1723
1724	// take local GPT lock in write mode
1725	remote_rwlock_wr_acquire( local_lock_xp );
1726
1727	// check VPN still unmapped in local GPT
1728
1729	// do nothing if VPN has been mapped by a a concurrent page_fault
1730	hal_gpt_get_pte( local_gpt_xp,
1731	vpn,
1732	&new_attr,
1733	&new_ppn );
1734
1735	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1736	{
1737	// allocate and initialise a physical page depending on the vseg type
1738	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1739
1740	if( error )
1741	{
1742	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1743	__FUNCTION__ , process->pid , vpn );
1744
1745	// release local GPT lock in write mode
1746	remote_rwlock_wr_release( local_lock_xp );
1747
1748	return EXCP_KERNEL_PANIC;
1749	}
1750
1751	// define new_attr from vseg flags
1752	new_attr = GPT_MAPPED \| GPT_SMALL;
1753	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1754	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1755	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1756	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1757
1758	// set PTE (PPN & attribute) to local GPT
1759	error = hal_gpt_set_pte( local_gpt_xp,
1760	vpn,
1761	new_attr,
1762	new_ppn );
1763	if ( error )
1764	{
1765	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1766	__FUNCTION__ , process->pid , vpn );
1767
1768	// release local GPT lock in write mode
1769	remote_rwlock_wr_release( local_lock_xp );
1770
1771	return EXCP_KERNEL_PANIC;
1772	}
1773	}
1774
1775	// release local GPT lock in write mode
1776	remote_rwlock_wr_release( local_lock_xp );
1777
1778	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1779	cycle = (uint32_t)hal_get_cycles();
1780	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1781	printk("\n[%s] private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1782	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1783	#endif
1784	return EXCP_NON_FATAL;
1785
1786	} // end local GPT access
1787
1788	//////////// public vseg => access reference GPT
1789	else
1790	{
1791	// get reference process cluster and local pointer
1792	ref_cxy = GET_CXY( process->ref_xp );
1793	ref_ptr = GET_PTR( process->ref_xp );
1794
1795	// build extended pointer on reference GPT and reference GPT lock
1796	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1797	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1798
1799	// build extended pointer on local GPT and local GPT lock
1800	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1801	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1802
1803	// take reference GPT lock in read mode
1804	remote_rwlock_rd_acquire( ref_lock_xp );
1805
1806	// get directly PPN & attributes from reference GPT
1807	// this can avoids a costly RPC for a false page fault
1808	hal_gpt_get_pte( ref_gpt_xp,
1809	vpn,
1810	&ref_attr,
1811	&ref_ppn );
1812
1813	// release reference GPT lock in read mode
1814	remote_rwlock_rd_release( ref_lock_xp );
1815
1816	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
1817	{
1818	// take local GPT lock in write mode
1819	remote_rwlock_wr_acquire( local_lock_xp );
1820
1821	// check VPN still unmapped in local GPT
1822	hal_gpt_get_pte( local_gpt_xp,
1823	vpn,
1824	&new_attr,
1825	&new_ppn );
1826
1827	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1828	{
1829	// update local GPT from reference GPT
1830	error = hal_gpt_set_pte( local_gpt_xp,
1831	vpn,
1832	ref_attr,
1833	ref_ppn );
1834	if( error )
1835	{
1836	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn %x\n",
1837	__FUNCTION__ , process->pid , vpn );
1838
1839	// release local GPT lock in write mode
1840	remote_rwlock_wr_release( local_lock_xp );
1841
1842	return EXCP_KERNEL_PANIC;
1843	}
1844	}
1845	else // VPN has been mapped by a a concurrent page_fault
1846	{
1847	// keep PTE from local GPT
1848	ref_attr = new_attr;
1849	ref_ppn = new_ppn;
1850	}
1851
1852	// release local GPT lock in write mode
1853	remote_rwlock_wr_release( local_lock_xp );
1854
1855	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1856	cycle = (uint32_t)hal_get_cycles();
1857	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1858	printk("\n[%s] false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1859	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
1860	#endif
1861	return EXCP_NON_FATAL;
1862	}
1863	else // true page fault => update reference GPT
1864	{
1865	// take reference GPT lock in write mode
1866	remote_rwlock_wr_acquire( ref_lock_xp );
1867
1868	// check VPN still unmapped in reference GPT
1869	// do nothing if VPN has been mapped by a a concurrent page_fault
1870	hal_gpt_get_pte( ref_gpt_xp,
1871	vpn,
1872	&ref_attr,
1873	&ref_ppn );
1874
1875	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
1876	{
1877	// allocate and initialise a physical page depending on the vseg type
1878	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1879
1880	if( error )
1881	{
1882	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1883	__FUNCTION__ , process->pid , vpn );
1884
1885	// release reference GPT lock in write mode
1886	remote_rwlock_wr_release( ref_lock_xp );
1887
1888	return EXCP_KERNEL_PANIC;
1889	}
1890
1891	// define new_attr from vseg flags
1892	new_attr = GPT_MAPPED \| GPT_SMALL;
1893	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1894	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1895	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1896	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1897
1898	// update reference GPT
1899	error = hal_gpt_set_pte( ref_gpt_xp,
1900	vpn,
1901	new_attr,
1902	new_ppn );
1903
1904	// update local GPT (protected by reference GPT lock)
1905	error \|= hal_gpt_set_pte( local_gpt_xp,
1906	vpn,
1907	new_attr,
1908	new_ppn );
1909
1910	if( error )
1911	{
1912	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
1913	__FUNCTION__ , process->pid , vpn );
1914
1915	// release reference GPT lock in write mode
1916	remote_rwlock_wr_release( ref_lock_xp );
1917
1918	return EXCP_KERNEL_PANIC;
1919	}
1920	}
1921
1922	// release reference GPT lock in write mode
1923	remote_rwlock_wr_release( ref_lock_xp );
1924
1925	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1926	cycle = (uint32_t)hal_get_cycles();
1927	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1928	printk("\n[%s] true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1929	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1930	#endif
1931	return EXCP_NON_FATAL;
1932	}
1933	}
1934	} // end vmm_handle_page_fault()
1935
1936	////////////////////////////////////////////
1937	error_t vmm_handle_cow( process_t * process,
1938	vpn_t vpn )
1939	{
1940	vseg_t * vseg; // vseg containing vpn
1941	cxy_t ref_cxy; // reference cluster for missing vpn
1942	process_t * ref_ptr; // reference process for missing vpn
1943	xptr_t gpt_xp; // extended pointer on GPT
1944	xptr_t gpt_lock_xp; // extended pointer on GPT lock
1945	uint32_t old_attr; // current PTE_ATTR value
1946	ppn_t old_ppn; // current PTE_PPN value
1947	uint32_t new_attr; // new PTE_ATTR value
1948	ppn_t new_ppn; // new PTE_PPN value
1949	error_t error;
1950
1951	#if DEBUG_VMM_HANDLE_COW
1952	uint32_t cycle = (uint32_t)hal_get_cycles();
1953	thread_t * this = CURRENT_THREAD;
1954	if( DEBUG_VMM_HANDLE_COW < cycle )
1955	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
1956	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
1957	#endif
1958
1959	// access local GPT to get GPT_COW flag
1960	bool_t cow = hal_gpt_pte_is_cow( &(process->vmm.gpt), vpn );
1961
1962	if( cow == false ) return EXCP_USER_ERROR;
1963
1964	// get local vseg
1965	error = vmm_get_vseg( process,
1966	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1967	&vseg );
1968	if( error )
1969	{
1970	printk("\n[PANIC] in %s : vpn %x in process %x not in a registered vseg\n",
1971	__FUNCTION__, vpn, process->pid );
1972
1973	return EXCP_KERNEL_PANIC;
1974	}
1975
1976	// get reference GPT cluster and local pointer
1977	ref_cxy = GET_CXY( process->ref_xp );
1978	ref_ptr = GET_PTR( process->ref_xp );
1979
1980	// build relevant extended pointers on relevant GPT and GPT lock
1981	// - access local GPT for a private vseg
1982	// - access reference GPT for a public vseg
1983	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1984	{
1985	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1986	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1987	}
1988	else
1989	{
1990	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1991	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1992	}
1993
1994	// take GPT lock in write mode
1995	remote_rwlock_wr_acquire( gpt_lock_xp );
1996
1997	// get current PTE from reference GPT
1998	hal_gpt_get_pte( gpt_xp,
1999	vpn,
2000	&old_attr,
2001	&old_ppn );
2002
2003	// the PTE must be mapped for a COW
2004	if( (old_attr & GPT_MAPPED) == 0 )
2005	{
2006	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
2007	__FUNCTION__, vpn, process->pid );
2008
2009	// release GPT lock in write mode
2010	remote_rwlock_wr_acquire( gpt_lock_xp );
2011
2012	return EXCP_KERNEL_PANIC;
2013	}
2014
2015	// get extended pointer, cluster and local pointer on physical page descriptor
2016	xptr_t page_xp = ppm_ppn2page( old_ppn );
2017	cxy_t page_cxy = GET_CXY( page_xp );
2018	page_t * page_ptr = GET_PTR( page_xp );
2019
2020	// get extended pointers on forks and lock field in page descriptor
2021	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
2022	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
2023
2024	// take lock protecting "forks" counter
2025	remote_busylock_acquire( forks_lock_xp );
2026
2027	// get number of pending forks from page descriptor
2028	uint32_t forks = hal_remote_l32( forks_xp );
2029
2030	if( forks ) // pending fork => allocate a new page, and copy old to new
2031	{
2032	// allocate a new physical page
2033	page_xp = vmm_page_allocate( vseg , vpn );
2034	if( page_xp == XPTR_NULL )
2035	{
2036	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2037	__FUNCTION__ , vpn, process->pid );
2038
2039	// release GPT lock in write mode
2040	remote_rwlock_wr_acquire( gpt_lock_xp );
2041
2042	// release lock protecting "forks" counter
2043	remote_busylock_release( forks_lock_xp );
2044
2045	return EXCP_KERNEL_PANIC;
2046	}
2047
2048	// compute allocated page PPN
2049	new_ppn = ppm_page2ppn( page_xp );
2050
2051	// copy old page content to new page
2052	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
2053	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
2054	memcpy( GET_PTR( new_base_xp ),
2055	GET_PTR( old_base_xp ),
2056	CONFIG_PPM_PAGE_SIZE );
2057
2058	// decrement pending forks counter in page descriptor
2059	hal_remote_atomic_add( forks_xp , -1 );
2060
2061	#if(DEBUG_VMM_HANDLE_COW & 1)
2062	if( DEBUG_VMM_HANDLE_COW < cycle )
2063	printk("\n[%s] thread[%x,%x] : pending forks => allocate a new PPN %x\n",
2064	__FUNCTION__, process->pid, this->trdid, new_ppn );
2065	#endif
2066
2067	}
2068	else // no pending fork => keep the existing page
2069	{
2070
2071	#if(DEBUG_VMM_HANDLE_COW & 1)
2072	if( DEBUG_VMM_HANDLE_COW < cycle )
2073	printk("\n[%s] thread[%x,%x] no pending forks => keep existing PPN %x\n",
2074	__FUNCTION__, process->pid, this->trdid, new_ppn );
2075	#endif
2076	new_ppn = old_ppn;
2077	}
2078
2079	// release lock protecting "forks" counter
2080	remote_busylock_release( forks_lock_xp );
2081
2082	// build new_attr : reset COW and set WRITABLE,
2083	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
2084
2085	// update the relevan GPT
2086	// - private vseg => update local GPT
2087	// - public vseg => update all GPT copies
2088	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2089	{
2090	hal_gpt_set_pte( gpt_xp,
2091	vpn,
2092	new_attr,
2093	new_ppn );
2094	}
2095	else
2096	{
2097	if( ref_cxy == local_cxy ) // reference cluster is local
2098	{
2099	vmm_global_update_pte( process,
2100	vpn,
2101	new_attr,
2102	new_ppn );
2103	}
2104	else // reference cluster is remote
2105	{
2106	rpc_vmm_global_update_pte_client( ref_cxy,
2107	ref_ptr,
2108	vpn,
2109	new_attr,
2110	new_ppn );
2111	}
2112	}
2113
2114	// release GPT lock in write mode
2115	remote_rwlock_wr_release( gpt_lock_xp );
2116
2117	#if DEBUG_VMM_HANDLE_COW
2118	cycle = (uint32_t)hal_get_cycles();
2119	if( DEBUG_VMM_HANDLE_COW < cycle )
2120	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2121	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2122	#endif
2123
2124	return EXCP_NON_FATAL;
2125
2126	} // end vmm_handle_cow()
2127

Note: See TracBrowser for help on using the repository browser.

Download in other formats: