Context Navigation

source: trunk/kernel/mm/vmm.c @ 610

Last change on this file since 610 was 610, checked in by alain, 6 years ago
Fix several bugs in VFS to support the following ksh commandis : cp, mv, rm, mkdir, cd, pwd
File size: 72.6 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations interface.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016,2017,2018)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_rwlock.h>
35	#include <remote_queuelock.h>
36	#include <list.h>
37	#include <xlist.h>
38	#include <bits.h>
39	#include <process.h>
40	#include <thread.h>
41	#include <vseg.h>
42	#include <cluster.h>
43	#include <scheduler.h>
44	#include <vfs.h>
45	#include <mapper.h>
46	#include <page.h>
47	#include <kmem.h>
48	#include <vmm.h>
49	#include <hal_exception.h>
50
51	//////////////////////////////////////////////////////////////////////////////////
52	// Extern global variables
53	//////////////////////////////////////////////////////////////////////////////////
54
55	extern process_t process_zero; // allocated in cluster.c
56
57	///////////////////////////////////////
58	error_t vmm_init( process_t * process )
59	{
60	error_t error;
61	vseg_t * vseg_kentry;
62	vseg_t * vseg_args;
63	vseg_t * vseg_envs;
64	intptr_t base;
65	intptr_t size;
66
67	#if DEBUG_VMM_INIT
68	thread_t * this = CURRENT_THREAD;
69	uint32_t cycle = (uint32_t)hal_get_cycles();
70	if( DEBUG_VMM_INIT )
71	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
72	__FUNCTION__ , this->process->pid, this->trdid, process->pid , cycle );
73	#endif
74
75	// get pointer on VMM
76	vmm_t * vmm = &process->vmm;
77
78	// initialize local list of vsegs
79	vmm->vsegs_nr = 0;
80	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
81	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
82
83	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
84	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
85
86	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
87	"no more than 32 threads per cluster for a single process\n");
88
89	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
90	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
91	"STACK zone too small\n");
92
93	// register kentry vseg in VSL
94	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
95	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
96
97	vseg_kentry = vmm_create_vseg( process,
98	VSEG_TYPE_CODE,
99	base,
100	size,
101	0, // file_offset unused
102	0, // file_size unused
103	XPTR_NULL, // mapper_xp unused
104	local_cxy );
105
106	if( vseg_kentry == NULL )
107	{
108	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
109	return -1;
110	}
111
112	vmm->kent_vpn_base = base;
113
114	// register args vseg in VSL
115	base = (CONFIG_VMM_KENTRY_BASE +
116	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
117	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
118
119	vseg_args = vmm_create_vseg( process,
120	VSEG_TYPE_DATA,
121	base,
122	size,
123	0, // file_offset unused
124	0, // file_size unused
125	XPTR_NULL, // mapper_xp unused
126	local_cxy );
127
128	if( vseg_args == NULL )
129	{
130	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
131	return -1;
132	}
133
134	vmm->args_vpn_base = base;
135
136	// register the envs vseg in VSL
137	base = (CONFIG_VMM_KENTRY_BASE +
138	CONFIG_VMM_KENTRY_SIZE +
139	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
140	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
141
142	vseg_envs = vmm_create_vseg( process,
143	VSEG_TYPE_DATA,
144	base,
145	size,
146	0, // file_offset unused
147	0, // file_size unused
148	XPTR_NULL, // mapper_xp unused
149	local_cxy );
150
151	if( vseg_envs == NULL )
152	{
153	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
154	return -1;
155	}
156
157	vmm->envs_vpn_base = base;
158
159	// create GPT (empty)
160	error = hal_gpt_create( &vmm->gpt );
161
162	if( error )
163	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
164
165	// initialize GPT lock
166	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
167
168	// architecture specic GPT initialisation
169	// (For TSAR, identity map the kentry_vseg)
170	error = hal_vmm_init( vmm );
171
172	if( error )
173	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
174
175	// initialize STACK allocator
176	vmm->stack_mgr.bitmap = 0;
177	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
178	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
179
180	// initialize MMAP allocator
181	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
182	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
183	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
184	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
185
186	uint32_t i;
187	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
188
189	// initialize instrumentation counters
190	vmm->pgfault_nr = 0;
191
192	hal_fence();
193
194	#if DEBUG_VMM_INIT
195	cycle = (uint32_t)hal_get_cycles();
196	if( DEBUG_VMM_INIT )
197	printk("\n[%s] thread[%x,%x] exit / process %x / entry_point %x / cycle %d\n",
198	__FUNCTION__, this->process->pid, this->trdid, process->pid, process->vmm.entry_point, cycle );
199	#endif
200
201	return 0;
202
203	} // end vmm_init()
204
205	//////////////////////////////////////
206	void vmm_display( process_t * process,
207	bool_t mapping )
208	{
209	vmm_t * vmm = &process->vmm;
210	gpt_t * gpt = &vmm->gpt;
211
212	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
213	process->vmm.gpt.ptr , process->pid , local_cxy );
214
215	// get lock protecting the VSL and the GPT
216	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
217	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
218
219	// scan the list of vsegs
220	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
221	xptr_t iter_xp;
222	xptr_t vseg_xp;
223	vseg_t * vseg;
224	XLIST_FOREACH( root_xp , iter_xp )
225	{
226	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
227	vseg = GET_PTR( vseg_xp );
228
229	printk(" - %s : base = %X / size = %X / npages = %d\n",
230	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
231
232	if( mapping )
233	{
234	vpn_t vpn;
235	ppn_t ppn;
236	uint32_t attr;
237	vpn_t base = vseg->vpn_base;
238	vpn_t size = vseg->vpn_size;
239	for( vpn = base ; vpn < (base+size) ; vpn++ )
240	{
241	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
242	if( attr & GPT_MAPPED )
243	{
244	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
245	}
246	}
247	}
248	}
249
250	// release the locks
251	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
252	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
253
254	} // vmm_display()
255
256	///////////////////////////////////
257	void vmm_vseg_attach( vmm_t * vmm,
258	vseg_t * vseg )
259	{
260	// build extended pointer on rwlock protecting VSL
261	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
262
263	// get rwlock in write mode
264	remote_rwlock_wr_acquire( lock_xp );
265
266	// update vseg descriptor
267	vseg->vmm = vmm;
268
269	// add vseg in vmm list
270	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
271	XPTR( local_cxy , &vseg->xlist ) );
272
273	// release rwlock in write mode
274	remote_rwlock_wr_release( lock_xp );
275	}
276
277	///////////////////////////////////
278	void vmm_vseg_detach( vmm_t * vmm,
279	vseg_t * vseg )
280	{
281	// build extended pointer on rwlock protecting VSL
282	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
283
284	// get rwlock in write mode
285	remote_rwlock_wr_acquire( lock_xp );
286
287	// update vseg descriptor
288	vseg->vmm = NULL;
289
290	// remove vseg from vmm list
291	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
292
293	// release rwlock in write mode
294	remote_rwlock_wr_release( lock_xp );
295	}
296
297	////////////////////////////////////////////////
298	void vmm_global_update_pte( process_t * process,
299	vpn_t vpn,
300	uint32_t attr,
301	ppn_t ppn )
302	{
303	xlist_entry_t * process_root_ptr;
304	xptr_t process_root_xp;
305	xptr_t process_iter_xp;
306
307	xptr_t remote_process_xp;
308	cxy_t remote_process_cxy;
309	process_t * remote_process_ptr;
310	xptr_t remote_gpt_xp;
311
312	pid_t pid;
313	cxy_t owner_cxy;
314	lpid_t owner_lpid;
315
316	#if DEBUG_VMM_UPDATE_PTE
317	uint32_t cycle = (uint32_t)hal_get_cycles();
318	thread_t * this = CURRENT_THREAD;
319	if( DEBUG_VMM_UPDATE_PTE < cycle )
320	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
321	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
322	#endif
323
324	// check cluster is reference
325	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
326
327	// get extended pointer on root of process copies xlist in owner cluster
328	pid = process->pid;
329	owner_cxy = CXY_FROM_PID( pid );
330	owner_lpid = LPID_FROM_PID( pid );
331	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
332	process_root_xp = XPTR( owner_cxy , process_root_ptr );
333
334	// loop on destination process copies
335	XLIST_FOREACH( process_root_xp , process_iter_xp )
336	{
337	// get cluster and local pointer on remote process
338	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
339	remote_process_ptr = GET_PTR( remote_process_xp );
340	remote_process_cxy = GET_CXY( remote_process_xp );
341
342	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
343	if( DEBUG_VMM_UPDATE_PTE < cycle )
344	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
345	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
346	#endif
347
348	// get extended pointer on remote gpt
349	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
350
351	// update remote GPT
352	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
353	}
354
355	#if DEBUG_VMM_UPDATE_PTE
356	cycle = (uint32_t)hal_get_cycles();
357	if( DEBUG_VMM_UPDATE_PTE < cycle )
358	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
359	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
360	#endif
361
362	} // end vmm_global_update_pte()
363
364	///////////////////////////////////////
365	void vmm_set_cow( process_t * process )
366	{
367	vmm_t * vmm;
368
369	xlist_entry_t * process_root_ptr;
370	xptr_t process_root_xp;
371	xptr_t process_iter_xp;
372
373	xptr_t remote_process_xp;
374	cxy_t remote_process_cxy;
375	process_t * remote_process_ptr;
376	xptr_t remote_gpt_xp;
377
378	xptr_t vseg_root_xp;
379	xptr_t vseg_iter_xp;
380
381	xptr_t vseg_xp;
382	vseg_t * vseg;
383
384	pid_t pid;
385	cxy_t owner_cxy;
386	lpid_t owner_lpid;
387
388	#if DEBUG_VMM_SET_COW
389	uint32_t cycle = (uint32_t)hal_get_cycles();
390	thread_t * this = CURRENT_THREAD;
391	if( DEBUG_VMM_SET_COW < cycle )
392	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
393	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
394	#endif
395
396	// check cluster is reference
397	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
398	"local cluster is not process reference cluster\n");
399
400	// get pointer on reference VMM
401	vmm = &process->vmm;
402
403	// get extended pointer on root of process copies xlist in owner cluster
404	pid = process->pid;
405	owner_cxy = CXY_FROM_PID( pid );
406	owner_lpid = LPID_FROM_PID( pid );
407	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
408	process_root_xp = XPTR( owner_cxy , process_root_ptr );
409
410	// get extended pointer on root of vsegs xlist from reference VMM
411	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
412
413	// loop on destination process copies
414	XLIST_FOREACH( process_root_xp , process_iter_xp )
415	{
416	// get cluster and local pointer on remote process
417	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
418	remote_process_ptr = GET_PTR( remote_process_xp );
419	remote_process_cxy = GET_CXY( remote_process_xp );
420
421	#if (DEBUG_VMM_SET_COW & 1)
422	if( DEBUG_VMM_SET_COW < cycle )
423	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
424	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
425	#endif
426
427	// get extended pointer on remote gpt
428	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
429
430	// loop on vsegs in (local) reference process VSL
431	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
432	{
433	// get pointer on vseg
434	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
435	vseg = GET_PTR( vseg_xp );
436
437	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
438	"all vsegs in reference VSL must be local\n" );
439
440	// get vseg type, base and size
441	uint32_t type = vseg->type;
442	vpn_t vpn_base = vseg->vpn_base;
443	vpn_t vpn_size = vseg->vpn_size;
444
445	#if (DEBUG_VMM_SET_COW & 1)
446	if( DEBUG_VMM_SET_COW < cycle )
447	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
448	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
449	#endif
450	// only DATA, ANON and REMOTE vsegs
451	if( (type == VSEG_TYPE_DATA) \|\|
452	(type == VSEG_TYPE_ANON) \|\|
453	(type == VSEG_TYPE_REMOTE) )
454	{
455	vpn_t vpn;
456	uint32_t attr;
457	ppn_t ppn;
458	xptr_t page_xp;
459	cxy_t page_cxy;
460	page_t * page_ptr;
461	xptr_t forks_xp;
462	xptr_t lock_xp;
463
464	// update flags in remote GPT
465	hal_gpt_set_cow( remote_gpt_xp,
466	vpn_base,
467	vpn_size );
468
469	// atomically increment pending forks counter in physical pages,
470	// for all vseg pages that are mapped in reference cluster
471	if( remote_process_cxy == local_cxy )
472	{
473	// scan all pages in vseg
474	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
475	{
476	// get page attributes and PPN from reference GPT
477	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
478
479	// atomically update pending forks counter if page is mapped
480	if( attr & GPT_MAPPED )
481	{
482	// get pointers and cluster on page descriptor
483	page_xp = ppm_ppn2page( ppn );
484	page_cxy = GET_CXY( page_xp );
485	page_ptr = GET_PTR( page_xp );
486
487	// get extended pointers on "forks" and "lock"
488	forks_xp = XPTR( page_cxy , &page_ptr->forks );
489	lock_xp = XPTR( page_cxy , &page_ptr->lock );
490
491	// take lock protecting "forks" counter
492	remote_busylock_acquire( lock_xp );
493
494	// increment "forks"
495	hal_remote_atomic_add( forks_xp , 1 );
496
497	// release lock protecting "forks" counter
498	remote_busylock_release( lock_xp );
499	}
500	} // end loop on vpn
501	} // end if local
502	} // end if vseg type
503	} // end loop on vsegs
504	} // end loop on process copies
505
506	#if DEBUG_VMM_SET_COW
507	cycle = (uint32_t)hal_get_cycles();
508	if( DEBUG_VMM_SET_COW < cycle )
509	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
510	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
511	#endif
512
513	} // end vmm_set-cow()
514
515	/////////////////////////////////////////////////
516	error_t vmm_fork_copy( process_t * child_process,
517	xptr_t parent_process_xp )
518	{
519	error_t error;
520	cxy_t parent_cxy;
521	process_t * parent_process;
522	vmm_t * parent_vmm;
523	xptr_t parent_lock_xp;
524	vmm_t * child_vmm;
525	xptr_t iter_xp;
526	xptr_t parent_vseg_xp;
527	vseg_t * parent_vseg;
528	vseg_t * child_vseg;
529	uint32_t type;
530	bool_t cow;
531	vpn_t vpn;
532	vpn_t vpn_base;
533	vpn_t vpn_size;
534	xptr_t page_xp; // extended pointer on page descriptor
535	page_t * page_ptr;
536	cxy_t page_cxy;
537	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
538	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
539	xptr_t parent_root_xp;
540	bool_t mapped;
541	ppn_t ppn;
542
543	#if DEBUG_VMM_FORK_COPY
544	uint32_t cycle = (uint32_t)hal_get_cycles();
545	thread_t * this = CURRENT_THREAD;
546	if( DEBUG_VMM_FORK_COPY < cycle )
547	printk("\n[%s] thread %x enter / cycle %d\n",
548	__FUNCTION__ , this->process->pid, this->trdid, cycle );
549	#endif
550
551	// get parent process cluster and local pointer
552	parent_cxy = GET_CXY( parent_process_xp );
553	parent_process = GET_PTR( parent_process_xp );
554
555	// get local pointers on parent and child VMM
556	parent_vmm = &parent_process->vmm;
557	child_vmm = &child_process->vmm;
558
559	// get extended pointer on lock protecting the parent VSL
560	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
561
562	// initialize the lock protecting the child VSL
563	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
564
565	// initialize the child VSL as empty
566	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
567	child_vmm->vsegs_nr = 0;
568
569	// create child GPT
570	error = hal_gpt_create( &child_vmm->gpt );
571
572	if( error )
573	{
574	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
575	return -1;
576	}
577
578	// build extended pointer on parent VSL
579	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
580
581	// take the lock protecting the parent VSL in read mode
582	remote_rwlock_rd_acquire( parent_lock_xp );
583
584	// loop on parent VSL xlist
585	XLIST_FOREACH( parent_root_xp , iter_xp )
586	{
587	// get local and extended pointers on current parent vseg
588	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
589	parent_vseg = GET_PTR( parent_vseg_xp );
590
591	// get vseg type
592	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
593
594	#if DEBUG_VMM_FORK_COPY
595	cycle = (uint32_t)hal_get_cycles();
596	if( DEBUG_VMM_FORK_COPY < cycle )
597	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
598	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
599	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
600	#endif
601
602	// all parent vsegs - but STACK - must be copied in child VSL
603	if( type != VSEG_TYPE_STACK )
604	{
605	// allocate memory for a new child vseg
606	child_vseg = vseg_alloc();
607	if( child_vseg == NULL ) // release all allocated vsegs
608	{
609	vmm_destroy( child_process );
610	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
611	return -1;
612	}
613
614	// copy parent vseg to child vseg
615	vseg_init_from_ref( child_vseg , parent_vseg_xp );
616
617	// register child vseg in child VSL
618	vmm_vseg_attach( child_vmm , child_vseg );
619
620	#if DEBUG_VMM_FORK_COPY
621	cycle = (uint32_t)hal_get_cycles();
622	if( DEBUG_VMM_FORK_COPY < cycle )
623	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
624	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
625	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
626	#endif
627
628	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
629	if( type != VSEG_TYPE_CODE )
630	{
631	// activate the COW for DATA, MMAP, REMOTE vsegs only
632	cow = ( type != VSEG_TYPE_FILE );
633
634	vpn_base = child_vseg->vpn_base;
635	vpn_size = child_vseg->vpn_size;
636
637	// scan pages in parent vseg
638	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
639	{
640	error = hal_gpt_pte_copy( &child_vmm->gpt,
641	XPTR( parent_cxy , &parent_vmm->gpt ),
642	vpn,
643	cow,
644	&ppn,
645	&mapped );
646	if( error )
647	{
648	vmm_destroy( child_process );
649	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
650	return -1;
651	}
652
653	// increment pending forks counter in page if mapped
654	if( mapped )
655	{
656	// get pointers and cluster on page descriptor
657	page_xp = ppm_ppn2page( ppn );
658	page_cxy = GET_CXY( page_xp );
659	page_ptr = GET_PTR( page_xp );
660
661	// get extended pointers on "forks" and "lock"
662	forks_xp = XPTR( page_cxy , &page_ptr->forks );
663	lock_xp = XPTR( page_cxy , &page_ptr->lock );
664
665	// get lock protecting "forks" counter
666	remote_busylock_acquire( lock_xp );
667
668	// increment "forks"
669	hal_remote_atomic_add( forks_xp , 1 );
670
671	// release lock protecting "forks" counter
672	remote_busylock_release( lock_xp );
673
674	#if DEBUG_VMM_FORK_COPY
675	cycle = (uint32_t)hal_get_cycles();
676	if( DEBUG_VMM_FORK_COPY < cycle )
677	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
678	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
679	#endif
680	}
681	}
682	} // end if no code & no stack
683	} // end if no stack
684	} // end loop on vsegs
685
686	// release the parent VSL lock in read mode
687	remote_rwlock_rd_release( parent_lock_xp );
688
689	// initialize child GPT (architecture specic)
690	// => For TSAR, identity map the kentry_vseg
691	error = hal_vmm_init( child_vmm );
692
693	if( error )
694	{
695	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
696	return -1;
697	}
698
699	// initialize the child VMM STACK allocator
700	child_vmm->stack_mgr.bitmap = 0;
701	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
702
703	// initialize the child VMM MMAP allocator
704	uint32_t i;
705	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
706	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
707	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
708	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
709
710	// initialize instrumentation counters
711	child_vmm->pgfault_nr = 0;
712
713	// copy base addresses from parent VMM to child VMM
714	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
715	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
716	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
717	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
718	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
719	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
720
721	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
722
723	hal_fence();
724
725	#if DEBUG_VMM_FORK_COPY
726	cycle = (uint32_t)hal_get_cycles();
727	if( DEBUG_VMM_FORK_COPY < cycle )
728	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
729	__FUNCTION__ , this->process->pid, this->trdid , cycle );
730	#endif
731
732	return 0;
733
734	} // vmm_fork_copy()
735
736	///////////////////////////////////////
737	void vmm_destroy( process_t * process )
738	{
739	xptr_t vseg_xp;
740	vseg_t * vseg;
741
742	#if DEBUG_VMM_DESTROY
743	uint32_t cycle = (uint32_t)hal_get_cycles();
744	thread_t * this = CURRENT_THREAD;
745	if( DEBUG_VMM_DESTROY < cycle )
746	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
747	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
748	#endif
749
750	#if (DEBUG_VMM_DESTROY & 1 )
751	if( DEBUG_VMM_DESTROY < cycle )
752	vmm_display( process , true );
753	#endif
754
755	// get pointer on local VMM
756	vmm_t * vmm = &process->vmm;
757
758	// get extended pointer on VSL root and VSL lock
759	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
760
761	// remove all user vsegs registered in VSL
762	while( !xlist_is_empty( root_xp ) )
763	{
764	// get pointer on first vseg in VSL
765	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
766	vseg = GET_PTR( vseg_xp );
767
768	// unmap and release physical pages
769	vmm_unmap_vseg( process , vseg );
770
771	// remove vseg from VSL
772	vmm_vseg_detach( vmm , vseg );
773
774	// release memory allocated to vseg descriptor
775	vseg_free( vseg );
776
777	#if( DEBUG_VMM_DESTROY & 1 )
778	if( DEBUG_VMM_DESTROY < cycle )
779	printk("\n[%s] %s vseg released / vpn_base %x / vpn_size %d\n",
780	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
781	#endif
782
783	}
784
785	// remove all vsegs from zombi_lists in MMAP allocator
786	uint32_t i;
787	for( i = 0 ; i<32 ; i++ )
788	{
789	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
790	{
791	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
792
793	#if( DEBUG_VMM_DESTROY & 1 )
794	if( DEBUG_VMM_DESTROY < cycle )
795	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
796	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
797	#endif
798	vmm_vseg_detach( vmm , vseg );
799	vseg_free( vseg );
800
801	#if( DEBUG_VMM_DESTROY & 1 )
802	if( DEBUG_VMM_DESTROY < cycle )
803	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
804	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
805	#endif
806	}
807	}
808
809	// release memory allocated to the GPT itself
810	hal_gpt_destroy( &vmm->gpt );
811
812	#if DEBUG_VMM_DESTROY
813	cycle = (uint32_t)hal_get_cycles();
814	if( DEBUG_VMM_DESTROY < cycle )
815	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
816	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
817	#endif
818
819	} // end vmm_destroy()
820
821	/////////////////////////////////////////////////
822	vseg_t * vmm_check_conflict( process_t * process,
823	vpn_t vpn_base,
824	vpn_t vpn_size )
825	{
826	vmm_t * vmm = &process->vmm;
827
828	// scan the VSL
829	vseg_t * vseg;
830	xptr_t iter_xp;
831	xptr_t vseg_xp;
832	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
833
834	XLIST_FOREACH( root_xp , iter_xp )
835	{
836	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
837	vseg = GET_PTR( vseg_xp );
838
839	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
840	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
841	}
842	return NULL;
843
844	} // end vmm_check_conflict()
845
846	////////////////////////////////////////////////////////////////////////////////////////////
847	// This static function is called by the vmm_create_vseg() function, and implements
848	// the VMM stack_vseg specific allocator.
849	////////////////////////////////////////////////////////////////////////////////////////////
850	// @ vmm : pointer on VMM.
851	// @ vpn_base : (return value) first allocated page
852	// @ vpn_size : (return value) number of allocated pages
853	////////////////////////////////////////////////////////////////////////////////////////////
854	static error_t vmm_stack_alloc( vmm_t * vmm,
855	vpn_t * vpn_base,
856	vpn_t * vpn_size )
857	{
858	// get stack allocator pointer
859	stack_mgr_t * mgr = &vmm->stack_mgr;
860
861	// get lock on stack allocator
862	busylock_acquire( &mgr->lock );
863
864	// get first free slot index in bitmap
865	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
866	if( (index < 0) \|\| (index > 31) )
867	{
868	busylock_release( &mgr->lock );
869	return 0xFFFFFFFF;
870	}
871
872	// update bitmap
873	bitmap_set( &mgr->bitmap , index );
874
875	// release lock on stack allocator
876	busylock_release( &mgr->lock );
877
878	// returns vpn_base, vpn_size (one page non allocated)
879	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
880	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
881	return 0;
882
883	} // end vmm_stack_alloc()
884
885	////////////////////////////////////////////////////////////////////////////////////////////
886	// This static function is called by the vmm_create_vseg() function, and implements
887	// the VMM MMAP specific allocator.
888	////////////////////////////////////////////////////////////////////////////////////////////
889	// @ vmm : [in] pointer on VMM.
890	// @ npages : [in] requested number of pages.
891	// @ vpn_base : [out] first allocated page.
892	// @ vpn_size : [out] actual number of allocated pages.
893	////////////////////////////////////////////////////////////////////////////////////////////
894	static error_t vmm_mmap_alloc( vmm_t * vmm,
895	vpn_t npages,
896	vpn_t * vpn_base,
897	vpn_t * vpn_size )
898	{
899	uint32_t index;
900	vseg_t * vseg;
901	vpn_t base;
902	vpn_t size;
903	vpn_t free;
904
905	// mmap vseg size must be power of 2
906	// compute actual size and index in zombi_list array
907	size = POW2_ROUNDUP( npages );
908	index = bits_log2( size );
909
910	// get mmap allocator pointer
911	mmap_mgr_t * mgr = &vmm->mmap_mgr;
912
913	// get lock on mmap allocator
914	busylock_acquire( &mgr->lock );
915
916	// get vseg from zombi_list or from mmap zone
917	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
918	{
919	// check overflow
920	free = mgr->first_free_vpn;
921	if( (free + size) > mgr->vpn_size ) return ENOMEM;
922
923	// update STACK allocator
924	mgr->first_free_vpn += size;
925
926	// compute base
927	base = free;
928	}
929	else // from zombi_list
930	{
931	// get pointer on zombi vseg from zombi_list
932	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
933
934	// remove vseg from free-list
935	list_unlink( &vseg->zlist );
936
937	// compute base
938	base = vseg->vpn_base;
939	}
940
941	// release lock on mmap allocator
942	busylock_release( &mgr->lock );
943
944	// returns vpn_base, vpn_size
945	*vpn_base = base;
946	*vpn_size = size;
947	return 0;
948
949	} // end vmm_mmap_alloc()
950
951	////////////////////////////////////////////////
952	vseg_t * vmm_create_vseg( process_t * process,
953	vseg_type_t type,
954	intptr_t base,
955	uint32_t size,
956	uint32_t file_offset,
957	uint32_t file_size,
958	xptr_t mapper_xp,
959	cxy_t cxy )
960	{
961	vseg_t * vseg; // created vseg pointer
962	vpn_t vpn_base; // first page index
963	vpn_t vpn_size; // number of pages covered by vseg
964	error_t error;
965
966	#if DEBUG_VMM_CREATE_VSEG
967	thread_t * this = CURRENT_THREAD;
968	uint32_t cycle = (uint32_t)hal_get_cycles();
969	if( DEBUG_VMM_CREATE_VSEG < cycle )
970	printk("\n[%s] thread[%x,%x] enter / %s / cxy %x / cycle %d\n",
971	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
972	#endif
973
974	// get pointer on VMM
975	vmm_t * vmm = &process->vmm;
976
977	// compute base, size, vpn_base, vpn_size, depending on vseg type
978	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
979
980	if( type == VSEG_TYPE_STACK )
981	{
982	// get vpn_base and vpn_size from STACK allocator
983	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
984	if( error )
985	{
986	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
987	__FUNCTION__ , process->pid , local_cxy );
988	return NULL;
989	}
990
991	// compute vseg base and size from vpn_base and vpn_size
992	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
993	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
994	}
995	else if( type == VSEG_TYPE_FILE )
996	{
997	// compute page index (in mapper) for first byte
998	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
999
1000	// compute page index (in mapper) for last byte
1001	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1002
1003	// compute offset in first page
1004	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1005
1006	// compute number of pages required in virtual space
1007	vpn_t npages = vpn_max - vpn_min + 1;
1008
1009	// get vpn_base and vpn_size from MMAP allocator
1010	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1011	if( error )
1012	{
1013	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1014	__FUNCTION__ , process->pid , local_cxy );
1015	return NULL;
1016	}
1017
1018	// set the vseg base (not always aligned for FILE)
1019	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1020	}
1021	else if( (type == VSEG_TYPE_ANON) \|\|
1022	(type == VSEG_TYPE_REMOTE) )
1023	{
1024	// compute number of required pages in virtual space
1025	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1026	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1027
1028	// get vpn_base and vpn_size from MMAP allocator
1029	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1030	if( error )
1031	{
1032	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1033	__FUNCTION__ , process->pid , local_cxy );
1034	return NULL;
1035	}
1036
1037	// set vseg base (always aligned for ANON or REMOTE)
1038	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1039	}
1040	else // VSEG_TYPE_DATA or VSEG_TYPE_CODE
1041	{
1042	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1043	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1044
1045	vpn_base = vpn_min;
1046	vpn_size = vpn_max - vpn_min + 1;
1047	}
1048
1049	// check collisions
1050	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1051	if( vseg != NULL )
1052	{
1053	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
1054	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
1055	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1056	return NULL;
1057	}
1058
1059	// allocate physical memory for vseg descriptor
1060	vseg = vseg_alloc();
1061	if( vseg == NULL )
1062	{
1063	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1064	__FUNCTION__ , process->pid );
1065	return NULL;
1066	}
1067
1068	// initialize vseg descriptor
1069	vseg_init( vseg,
1070	type,
1071	base,
1072	size,
1073	vpn_base,
1074	vpn_size,
1075	file_offset,
1076	file_size,
1077	mapper_xp,
1078	cxy );
1079
1080	// attach vseg to VSL
1081	vmm_vseg_attach( vmm , vseg );
1082
1083	#if DEBUG_VMM_CREATE_VSEG
1084	cycle = (uint32_t)hal_get_cycles();
1085	if( DEBUG_VMM_CREATE_VSEG < cycle )
1086	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
1087	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
1088	#endif
1089
1090	return vseg;
1091
1092	} // vmm_create_vseg()
1093
1094	/////////////////////////////////////
1095	void vmm_remove_vseg( vseg_t * vseg )
1096	{
1097	// get pointers on calling process and VMM
1098	thread_t * this = CURRENT_THREAD;
1099	vmm_t * vmm = &this->process->vmm;
1100	uint32_t type = vseg->type;
1101
1102	// detach vseg from VSL
1103	vmm_vseg_detach( vmm , vseg );
1104
1105	// release the stack slot to VMM stack allocator if STACK type
1106	if( type == VSEG_TYPE_STACK )
1107	{
1108	// get pointer on stack allocator
1109	stack_mgr_t * mgr = &vmm->stack_mgr;
1110
1111	// compute slot index
1112	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
1113
1114	// update stacks_bitmap
1115	busylock_acquire( &mgr->lock );
1116	bitmap_clear( &mgr->bitmap , index );
1117	busylock_release( &mgr->lock );
1118	}
1119
1120	// release the vseg to VMM mmap allocator if MMAP type
1121	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
1122	{
1123	// get pointer on mmap allocator
1124	mmap_mgr_t * mgr = &vmm->mmap_mgr;
1125
1126	// compute zombi_list index
1127	uint32_t index = bits_log2( vseg->vpn_size );
1128
1129	// update zombi_list
1130	busylock_acquire( &mgr->lock );
1131	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
1132	busylock_release( &mgr->lock );
1133	}
1134
1135	// release physical memory allocated for vseg descriptor if no MMAP type
1136	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
1137	{
1138	vseg_free( vseg );
1139	}
1140	} // end vmm_remove_vseg()
1141
1142	/////////////////////////////////////////
1143	void vmm_unmap_vseg( process_t * process,
1144	vseg_t * vseg )
1145	{
1146	vpn_t vpn; // VPN of current PTE
1147	vpn_t vpn_min; // VPN of first PTE
1148	vpn_t vpn_max; // VPN of last PTE (excluded)
1149	ppn_t ppn; // current PTE ppn value
1150	uint32_t attr; // current PTE attributes
1151	kmem_req_t req; // request to release memory
1152	xptr_t page_xp; // extended pointer on page descriptor
1153	cxy_t page_cxy; // page descriptor cluster
1154	page_t * page_ptr; // page descriptor pointer
1155	xptr_t forks_xp; // extended pointer on pending forks counter
1156	xptr_t lock_xp; // extended pointer on lock protecting forks counter
1157	uint32_t forks; // actual number of pendinf forks
1158
1159	#if DEBUG_VMM_UNMAP_VSEG
1160	uint32_t cycle = (uint32_t)hal_get_cycles();
1161	thread_t * this = CURRENT_THREAD;
1162	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1163	printk("\n[%s] thread[%x,%x] enter / process %x / vseg %s / base %x / cycle %d\n",
1164	__FUNCTION__, this->process->pid, this->trdid, process->pid,
1165	vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
1166	#endif
1167
1168	// get pointer on local GPT
1169	gpt_t * gpt = &process->vmm.gpt;
1170
1171	// loop on pages in vseg
1172	vpn_min = vseg->vpn_base;
1173	vpn_max = vpn_min + vseg->vpn_size;
1174	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1175	{
1176	// get GPT entry
1177	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
1178
1179	if( attr & GPT_MAPPED ) // entry is mapped
1180	{
1181
1182	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
1183	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1184	printk("- vpn %x / ppn %x\n" , vpn , ppn );
1185	#endif
1186
1187	// check small page
1188	assert( (attr & GPT_SMALL) , "an user vseg must use small pages" );
1189
1190	// unmap GPT entry in local GPT
1191	hal_gpt_reset_pte( gpt , vpn );
1192
1193	// handle pending forks counter if
1194	// 1) not identity mapped
1195	// 2) reference cluster
1196	if( ((vseg->flags & VSEG_IDENT) == 0) &&
1197	(GET_CXY( process->ref_xp ) == local_cxy) )
1198	{
1199	// get extended pointer on physical page descriptor
1200	page_xp = ppm_ppn2page( ppn );
1201	page_cxy = GET_CXY( page_xp );
1202	page_ptr = GET_PTR( page_xp );
1203
1204	// get extended pointers on forks and lock fields
1205	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1206	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1207
1208	// get pending forks counter
1209	forks = hal_remote_l32( forks_xp );
1210
1211	if( forks ) // decrement pending forks counter
1212	{
1213	hal_remote_atomic_add( forks_xp , -1 );
1214	}
1215	else // release physical page to relevant cluster
1216	{
1217	if( page_cxy == local_cxy ) // local cluster
1218	{
1219	req.type = KMEM_PAGE;
1220	req.ptr = page_ptr;
1221	kmem_free( &req );
1222	}
1223	else // remote cluster
1224	{
1225	rpc_pmem_release_pages_client( page_cxy , page_ptr );
1226	}
1227	}
1228	}
1229	}
1230	}
1231
1232	#if DEBUG_VMM_UNMAP_VSEG
1233	cycle = (uint32_t)hal_get_cycles();
1234	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1235	printk("\n[%s] thread[%x,%x] exit / process %x / vseg %s / base %x / cycle %d\n",
1236	__FUNCTION__, this->process->pid, this->trdid, process->pid,
1237	vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
1238	#endif
1239
1240	} // end vmm_unmap_vseg()
1241
1242	//////////////////////////////////////////////////////////////////////////////////////////
1243	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
1244	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
1245	// containing a given virtual address.
1246	//////////////////////////////////////////////////////////////////////////////////////////
1247	// @ vmm : pointer on the process VMM.
1248	// @ vaddr : virtual address.
1249	// @ return vseg pointer if success / return NULL if not found.
1250	//////////////////////////////////////////////////////////////////////////////////////////
1251	static vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
1252	intptr_t vaddr )
1253	{
1254	xptr_t iter_xp;
1255	xptr_t vseg_xp;
1256	vseg_t * vseg;
1257
1258	// get extended pointers on VSL lock and root
1259	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1260	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1261
1262	// get lock protecting the VSL
1263	remote_rwlock_rd_acquire( lock_xp );
1264
1265	// scan the list of vsegs in VSL
1266	XLIST_FOREACH( root_xp , iter_xp )
1267	{
1268	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1269	vseg = GET_PTR( vseg_xp );
1270
1271	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
1272	{
1273	// return success
1274	remote_rwlock_rd_release( lock_xp );
1275	return vseg;
1276	}
1277	}
1278
1279	// return failure
1280	remote_rwlock_rd_release( lock_xp );
1281	return NULL;
1282
1283	} // end vmm_vseg_from_vaddr()
1284
1285	/////////////////////////////////////////////
1286	error_t vmm_resize_vseg( process_t * process,
1287	intptr_t base,
1288	intptr_t size )
1289	{
1290	error_t error;
1291	vseg_t * new;
1292	vpn_t vpn_min;
1293	vpn_t vpn_max;
1294
1295	// get pointer on process VMM
1296	vmm_t * vmm = &process->vmm;
1297
1298	intptr_t addr_min = base;
1299	intptr_t addr_max = base + size;
1300
1301	// get pointer on vseg
1302	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
1303
1304	if( vseg == NULL) return EINVAL;
1305
1306	// get extended pointer on VSL lock
1307	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1308
1309	// get lock protecting VSL
1310	remote_rwlock_wr_acquire( lock_xp );
1311
1312	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
1313	{
1314	error = EINVAL;
1315	}
1316	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
1317	{
1318	vmm_remove_vseg( vseg );
1319	error = 0;
1320	}
1321	else if( vseg->min == addr_min ) // vseg must be resized
1322	{
1323	// update vseg base address
1324	vseg->min = addr_max;
1325
1326	// update vpn_base and vpn_size
1327	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1328	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1329	vseg->vpn_base = vpn_min;
1330	vseg->vpn_size = vpn_max - vpn_min + 1;
1331	error = 0;
1332	}
1333	else if( vseg->max == addr_max ) // vseg must be resized
1334	{
1335	// update vseg max address
1336	vseg->max = addr_min;
1337
1338	// update vpn_base and vpn_size
1339	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1340	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1341	vseg->vpn_base = vpn_min;
1342	vseg->vpn_size = vpn_max - vpn_min + 1;
1343	error = 0;
1344	}
1345	else // vseg cut in three regions
1346	{
1347	// resize existing vseg
1348	vseg->max = addr_min;
1349
1350	// update vpn_base and vpn_size
1351	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1352	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1353	vseg->vpn_base = vpn_min;
1354	vseg->vpn_size = vpn_max - vpn_min + 1;
1355
1356	// create new vseg
1357	new = vmm_create_vseg( process,
1358	vseg->type,
1359	addr_min,
1360	(vseg->max - addr_max),
1361	vseg->file_offset,
1362	vseg->file_size,
1363	vseg->mapper_xp,
1364	vseg->cxy );
1365
1366	if( new == NULL ) error = EINVAL;
1367	else error = 0;
1368	}
1369
1370	// release VMM lock
1371	remote_rwlock_wr_release( lock_xp );
1372
1373	return error;
1374
1375	} // vmm_resize_vseg()
1376
1377	///////////////////////////////////////////
1378	error_t vmm_get_vseg( process_t * process,
1379	intptr_t vaddr,
1380	vseg_t ** found_vseg )
1381	{
1382	xptr_t vseg_xp;
1383	vseg_t * vseg;
1384	vmm_t * vmm;
1385	error_t error;
1386
1387	// get pointer on local VMM
1388	vmm = &process->vmm;
1389
1390	// try to get vseg from local VMM
1391	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1392
1393	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
1394	{
1395	// get extended pointer on reference process
1396	xptr_t ref_xp = process->ref_xp;
1397
1398	// get cluster and local pointer on reference process
1399	cxy_t ref_cxy = GET_CXY( ref_xp );
1400	process_t * ref_ptr = GET_PTR( ref_xp );
1401
1402	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
1403
1404	// get extended pointer on reference vseg
1405	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
1406
1407	if( error ) return -1; // vseg not found => illegal user vaddr
1408
1409	// allocate a vseg in local cluster
1410	vseg = vseg_alloc();
1411
1412	if( vseg == NULL ) return -1; // cannot allocate a local vseg
1413
1414	// initialise local vseg from reference
1415	vseg_init_from_ref( vseg , vseg_xp );
1416
1417	// register local vseg in local VMM
1418	vmm_vseg_attach( vmm , vseg );
1419	}
1420
1421	// success
1422	*found_vseg = vseg;
1423	return 0;
1424
1425	} // end vmm_get_vseg()
1426
1427	//////////////////////////////////////////////////////////////////////////////////////
1428	// This static function compute the target cluster to allocate a physical page
1429	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
1430	// and returns an extended pointer on the allocated page descriptor.
1431	// It can be called by a thread running in any cluster.
1432	// The vseg cannot have the FILE type.
1433	//////////////////////////////////////////////////////////////////////////////////////
1434	static xptr_t vmm_page_allocate( vseg_t * vseg,
1435	vpn_t vpn )
1436	{
1437
1438	#if DEBUG_VMM_ALLOCATE_PAGE
1439	uint32_t cycle = (uint32_t)hal_get_cycles();
1440	thread_t * this = CURRENT_THREAD;
1441	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1442	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
1443	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
1444	#endif
1445
1446	page_t * page_ptr;
1447	cxy_t page_cxy;
1448	kmem_req_t req;
1449	uint32_t index;
1450
1451	uint32_t type = vseg->type;
1452	uint32_t flags = vseg->flags;
1453	uint32_t x_size = LOCAL_CLUSTER->x_size;
1454	uint32_t y_size = LOCAL_CLUSTER->y_size;
1455
1456	// check vseg type
1457	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1458
1459	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1460	{
1461	index = vpn & ((x_size * y_size) - 1);
1462	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1463
1464	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1465	if ( cluster_is_active( page_cxy ) == false )
1466	{
1467	page_cxy = cluster_random_select();
1468	}
1469	}
1470	else // other cases => cxy specified in vseg
1471	{
1472	page_cxy = vseg->cxy;
1473	}
1474
1475	// allocate a physical page from target cluster
1476	if( page_cxy == local_cxy ) // target cluster is the local cluster
1477	{
1478	req.type = KMEM_PAGE;
1479	req.size = 0;
1480	req.flags = AF_NONE;
1481	page_ptr = (page_t *)kmem_alloc( &req );
1482	}
1483	else // target cluster is not the local cluster
1484	{
1485	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
1486	}
1487
1488	#if DEBUG_VMM_ALLOCATE_PAGE
1489	cycle = (uint32_t)hal_get_cycles();
1490	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1491	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
1492	__FUNCTION__ , this->process->pid, this->trdid, vpn,
1493	ppm_page2ppn( XPTR( page_cxy , page_ptr ) , cycle );
1494	#endif
1495
1496	if( page_ptr == NULL ) return XPTR_NULL;
1497	else return XPTR( page_cxy , page_ptr );
1498
1499	} // end vmm_page_allocate()
1500
1501	////////////////////////////////////////
1502	error_t vmm_get_one_ppn( vseg_t * vseg,
1503	vpn_t vpn,
1504	ppn_t * ppn )
1505	{
1506	error_t error;
1507	xptr_t page_xp; // extended pointer on physical page descriptor
1508	uint32_t page_id; // missing page index in vseg mapper
1509	uint32_t type; // vseg type;
1510
1511	type = vseg->type;
1512	page_id = vpn - vseg->vpn_base;
1513
1514	#if DEBUG_VMM_GET_ONE_PPN
1515	uint32_t cycle = (uint32_t)hal_get_cycles();
1516	thread_t * this = CURRENT_THREAD;
1517	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1518	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
1519	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
1520	#endif
1521
1522	// FILE type : get the physical page from the file mapper
1523	if( type == VSEG_TYPE_FILE )
1524	{
1525	// get extended pointer on mapper
1526	xptr_t mapper_xp = vseg->mapper_xp;
1527
1528	assert( (mapper_xp != XPTR_NULL),
1529	"mapper not defined for a FILE vseg\n" );
1530
1531	// get extended pointer on page descriptor
1532	page_xp = mapper_remote_get_page( mapper_xp , page_id );
1533
1534	if ( page_xp == XPTR_NULL ) return EINVAL;
1535	}
1536
1537	// Other types : allocate a physical page from target cluster,
1538	// as defined by vseg type and vpn value
1539	else
1540	{
1541	// allocate one physical page
1542	page_xp = vmm_page_allocate( vseg , vpn );
1543
1544	if( page_xp == XPTR_NULL ) return ENOMEM;
1545
1546	// initialise missing page from .elf file mapper for DATA and CODE types
1547	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1548	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1549	{
1550	// get extended pointer on mapper
1551	xptr_t mapper_xp = vseg->mapper_xp;
1552
1553	assert( (mapper_xp != XPTR_NULL),
1554	"mapper not defined for a CODE or DATA vseg\n" );
1555
1556	// compute missing page offset in vseg
1557	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
1558
1559	// compute missing page offset in .elf file
1560	uint32_t elf_offset = vseg->file_offset + offset;
1561
1562	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1563	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1564	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
1565	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
1566	#endif
1567	// compute extended pointer on page base
1568	xptr_t base_xp = ppm_page2base( page_xp );
1569
1570	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1571	uint32_t file_size = vseg->file_size;
1572
1573	if( file_size < offset ) // missing page fully in BSS
1574	{
1575
1576	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1577	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1578	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
1579	__FUNCTION__, this->process->pid, this->trdid, vpn );
1580	#endif
1581	if( GET_CXY( page_xp ) == local_cxy )
1582	{
1583	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1584	}
1585	else
1586	{
1587	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
1588	}
1589	}
1590	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
1591	{
1592
1593	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1594	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1595	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
1596	__FUNCTION__, this->process->pid, this->trdid, vpn );
1597	#endif
1598	error = mapper_move_kernel( mapper_xp,
1599	true, // to_buffer
1600	elf_offset,
1601	base_xp,
1602	CONFIG_PPM_PAGE_SIZE );
1603	if( error ) return EINVAL;
1604	}
1605	else // both in mapper and in BSS :
1606	// - (file_size - offset) bytes from mapper
1607	// - (page_size + offset - file_size) bytes from BSS
1608	{
1609
1610	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1611	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1612	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n"
1613	" %d bytes from mapper / %d bytes from BSS\n",
1614	__FUNCTION__, this->process->pid, this->trdid, vpn,
1615	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
1616	#endif
1617	// initialize mapper part
1618	error = mapper_move_kernel( mapper_xp,
1619	true, // to buffer
1620	elf_offset,
1621	base_xp,
1622	file_size - offset );
1623	if( error ) return EINVAL;
1624
1625	// initialize BSS part
1626	if( GET_CXY( page_xp ) == local_cxy )
1627	{
1628	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
1629	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1630	}
1631	else
1632	{
1633	hal_remote_memset( base_xp + file_size - offset , 0 ,
1634	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1635	}
1636	}
1637	} // end initialisation for CODE or DATA types
1638	}
1639
1640	// return ppn
1641	*ppn = ppm_page2ppn( page_xp );
1642
1643	#if DEBUG_VMM_GET_ONE_PPN
1644	cycle = (uint32_t)hal_get_cycles();
1645	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1646	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
1647	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
1648	#endif
1649
1650	return 0;
1651
1652	} // end vmm_get_one_ppn()
1653
1654	///////////////////////////////////////////////////
1655	error_t vmm_handle_page_fault( process_t * process,
1656	vpn_t vpn )
1657	{
1658	vseg_t * vseg; // vseg containing vpn
1659	uint32_t new_attr; // new PTE_ATTR value
1660	ppn_t new_ppn; // new PTE_PPN value
1661	uint32_t ref_attr; // PTE_ATTR value in reference GPT
1662	ppn_t ref_ppn; // PTE_PPN value in reference GPT
1663	cxy_t ref_cxy; // reference cluster for missing vpn
1664	process_t * ref_ptr; // reference process for missing vpn
1665	xptr_t local_gpt_xp; // extended pointer on local GPT
1666	xptr_t local_lock_xp; // extended pointer on local GPT lock
1667	xptr_t ref_gpt_xp; // extended pointer on reference GPT
1668	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
1669	error_t error; // value returned by called functions
1670
1671	// get local vseg (access to reference VSL can be required)
1672	error = vmm_get_vseg( process,
1673	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1674	&vseg );
1675	if( error )
1676	{
1677	printk("\n[ERROR] in %s : vpn %x in process %x not in a registered vseg\n",
1678	__FUNCTION__ , vpn , process->pid );
1679
1680	return EXCP_USER_ERROR;
1681	}
1682
1683	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1684	uint32_t cycle = (uint32_t)hal_get_cycles();
1685	thread_t * this = CURRENT_THREAD;
1686	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1687	printk("\n[%s] threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
1688	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
1689	#endif
1690
1691	//////////////// private vseg => access only the local GPT
1692	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1693	{
1694	// build extended pointer on local GPT and local GPT lock
1695	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1696	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1697
1698	// take local GPT lock in write mode
1699	remote_rwlock_wr_acquire( local_lock_xp );
1700
1701	// check VPN still unmapped in local GPT
1702
1703	// do nothing if VPN has been mapped by a a concurrent page_fault
1704	hal_gpt_get_pte( local_gpt_xp,
1705	vpn,
1706	&new_attr,
1707	&new_ppn );
1708
1709	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1710	{
1711	// allocate and initialise a physical page depending on the vseg type
1712	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1713
1714	if( error )
1715	{
1716	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1717	__FUNCTION__ , process->pid , vpn );
1718
1719	// release local GPT lock in write mode
1720	remote_rwlock_wr_release( local_lock_xp );
1721
1722	return EXCP_KERNEL_PANIC;
1723	}
1724
1725	// define new_attr from vseg flags
1726	new_attr = GPT_MAPPED \| GPT_SMALL;
1727	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1728	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1729	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1730	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1731
1732	// set PTE (PPN & attribute) to local GPT
1733	error = hal_gpt_set_pte( local_gpt_xp,
1734	vpn,
1735	new_attr,
1736	new_ppn );
1737	if ( error )
1738	{
1739	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1740	__FUNCTION__ , process->pid , vpn );
1741
1742	// release local GPT lock in write mode
1743	remote_rwlock_wr_release( local_lock_xp );
1744
1745	return EXCP_KERNEL_PANIC;
1746	}
1747	}
1748
1749	// release local GPT lock in write mode
1750	remote_rwlock_wr_release( local_lock_xp );
1751
1752	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1753	cycle = (uint32_t)hal_get_cycles();
1754	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1755	printk("\n[%s] private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1756	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1757	#endif
1758	return EXCP_NON_FATAL;
1759
1760	} // end local GPT access
1761
1762	//////////// public vseg => access reference GPT
1763	else
1764	{
1765	// get reference process cluster and local pointer
1766	ref_cxy = GET_CXY( process->ref_xp );
1767	ref_ptr = GET_PTR( process->ref_xp );
1768
1769	// build extended pointer on reference GPT and reference GPT lock
1770	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1771	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1772
1773	// build extended pointer on local GPT and local GPT lock
1774	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1775	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1776
1777	// take reference GPT lock in read mode
1778	remote_rwlock_rd_acquire( ref_lock_xp );
1779
1780	// get directly PPN & attributes from reference GPT
1781	// this can avoids a costly RPC for a false page fault
1782	hal_gpt_get_pte( ref_gpt_xp,
1783	vpn,
1784	&ref_attr,
1785	&ref_ppn );
1786
1787	// release reference GPT lock in read mode
1788	remote_rwlock_rd_release( ref_lock_xp );
1789
1790	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
1791	{
1792	// take local GPT lock in write mode
1793	remote_rwlock_wr_acquire( local_lock_xp );
1794
1795	// check VPN still unmapped in local GPT
1796	hal_gpt_get_pte( local_gpt_xp,
1797	vpn,
1798	&new_attr,
1799	&new_ppn );
1800
1801	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1802	{
1803	// update local GPT from reference GPT
1804	error = hal_gpt_set_pte( local_gpt_xp,
1805	vpn,
1806	ref_attr,
1807	ref_ppn );
1808	if( error )
1809	{
1810	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn %x\n",
1811	__FUNCTION__ , process->pid , vpn );
1812
1813	// release local GPT lock in write mode
1814	remote_rwlock_wr_release( local_lock_xp );
1815
1816	return EXCP_KERNEL_PANIC;
1817	}
1818	}
1819	else // VPN has been mapped by a a concurrent page_fault
1820	{
1821	// keep PTE from local GPT
1822	ref_attr = new_attr;
1823	ref_ppn = new_ppn;
1824	}
1825
1826	// release local GPT lock in write mode
1827	remote_rwlock_wr_release( local_lock_xp );
1828
1829	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1830	cycle = (uint32_t)hal_get_cycles();
1831	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1832	printk("\n[%s] false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1833	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
1834	#endif
1835	return EXCP_NON_FATAL;
1836	}
1837	else // true page fault => update reference GPT
1838	{
1839	// take reference GPT lock in write mode
1840	remote_rwlock_wr_acquire( ref_lock_xp );
1841
1842	// check VPN still unmapped in reference GPT
1843	// do nothing if VPN has been mapped by a a concurrent page_fault
1844	hal_gpt_get_pte( ref_gpt_xp,
1845	vpn,
1846	&ref_attr,
1847	&ref_ppn );
1848
1849	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
1850	{
1851	// allocate and initialise a physical page depending on the vseg type
1852	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1853
1854	if( error )
1855	{
1856	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1857	__FUNCTION__ , process->pid , vpn );
1858
1859	// release reference GPT lock in write mode
1860	remote_rwlock_wr_release( ref_lock_xp );
1861
1862	return EXCP_KERNEL_PANIC;
1863	}
1864
1865	// define new_attr from vseg flags
1866	new_attr = GPT_MAPPED \| GPT_SMALL;
1867	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1868	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1869	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1870	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1871
1872	// update reference GPT
1873	error = hal_gpt_set_pte( ref_gpt_xp,
1874	vpn,
1875	new_attr,
1876	new_ppn );
1877
1878	// update local GPT (protected by reference GPT lock)
1879	error \|= hal_gpt_set_pte( local_gpt_xp,
1880	vpn,
1881	new_attr,
1882	new_ppn );
1883
1884	if( error )
1885	{
1886	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
1887	__FUNCTION__ , process->pid , vpn );
1888
1889	// release reference GPT lock in write mode
1890	remote_rwlock_wr_release( ref_lock_xp );
1891
1892	return EXCP_KERNEL_PANIC;
1893	}
1894	}
1895
1896	// release reference GPT lock in write mode
1897	remote_rwlock_wr_release( ref_lock_xp );
1898
1899	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1900	cycle = (uint32_t)hal_get_cycles();
1901	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1902	printk("\n[%s] true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1903	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1904	#endif
1905	return EXCP_NON_FATAL;
1906	}
1907	}
1908	} // end vmm_handle_page_fault()
1909
1910	////////////////////////////////////////////
1911	error_t vmm_handle_cow( process_t * process,
1912	vpn_t vpn )
1913	{
1914	vseg_t * vseg; // vseg containing vpn
1915	cxy_t ref_cxy; // reference cluster for missing vpn
1916	process_t * ref_ptr; // reference process for missing vpn
1917	xptr_t gpt_xp; // extended pointer on GPT
1918	xptr_t gpt_lock_xp; // extended pointer on GPT lock
1919	uint32_t old_attr; // current PTE_ATTR value
1920	ppn_t old_ppn; // current PTE_PPN value
1921	uint32_t new_attr; // new PTE_ATTR value
1922	ppn_t new_ppn; // new PTE_PPN value
1923	error_t error;
1924
1925	#if DEBUG_VMM_HANDLE_COW
1926	uint32_t cycle = (uint32_t)hal_get_cycles();
1927	thread_t * this = CURRENT_THREAD;
1928	if( DEBUG_VMM_HANDLE_COW < cycle )
1929	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
1930	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
1931	#endif
1932
1933	// access local GPT to get GPT_COW flag
1934	bool_t cow = hal_gpt_pte_is_cow( &(process->vmm.gpt), vpn );
1935
1936	if( cow == false ) return EXCP_USER_ERROR;
1937
1938	// get local vseg
1939	error = vmm_get_vseg( process,
1940	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1941	&vseg );
1942	if( error )
1943	{
1944	printk("\n[PANIC] in %s : vpn %x in process %x not in a registered vseg\n",
1945	__FUNCTION__, vpn, process->pid );
1946
1947	return EXCP_KERNEL_PANIC;
1948	}
1949
1950	// get reference GPT cluster and local pointer
1951	ref_cxy = GET_CXY( process->ref_xp );
1952	ref_ptr = GET_PTR( process->ref_xp );
1953
1954	// build relevant extended pointers on relevant GPT and GPT lock
1955	// - access local GPT for a private vseg
1956	// - access reference GPT for a public vseg
1957	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1958	{
1959	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1960	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1961	}
1962	else
1963	{
1964	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1965	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1966	}
1967
1968	// take GPT lock in write mode
1969	remote_rwlock_wr_acquire( gpt_lock_xp );
1970
1971	// get current PTE from reference GPT
1972	hal_gpt_get_pte( gpt_xp,
1973	vpn,
1974	&old_attr,
1975	&old_ppn );
1976
1977	// the PTE must be mapped for a COW
1978	if( (old_attr & GPT_MAPPED) == 0 )
1979	{
1980	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
1981	__FUNCTION__, vpn, process->pid );
1982
1983	// release GPT lock in write mode
1984	remote_rwlock_wr_acquire( gpt_lock_xp );
1985
1986	return EXCP_KERNEL_PANIC;
1987	}
1988
1989	// get extended pointer, cluster and local pointer on physical page descriptor
1990	xptr_t page_xp = ppm_ppn2page( old_ppn );
1991	cxy_t page_cxy = GET_CXY( page_xp );
1992	page_t * page_ptr = GET_PTR( page_xp );
1993
1994	// get extended pointers on forks and lock field in page descriptor
1995	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
1996	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
1997
1998	// take lock protecting "forks" counter
1999	remote_busylock_acquire( forks_lock_xp );
2000
2001	// get number of pending forks from page descriptor
2002	uint32_t forks = hal_remote_l32( forks_xp );
2003
2004	if( forks ) // pending fork => allocate a new page, and copy old to new
2005	{
2006	// allocate a new physical page
2007	page_xp = vmm_page_allocate( vseg , vpn );
2008	if( page_xp == XPTR_NULL )
2009	{
2010	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2011	__FUNCTION__ , vpn, process->pid );
2012
2013	// release GPT lock in write mode
2014	remote_rwlock_wr_acquire( gpt_lock_xp );
2015
2016	// release lock protecting "forks" counter
2017	remote_busylock_release( forks_lock_xp );
2018
2019	return EXCP_KERNEL_PANIC;
2020	}
2021
2022	// compute allocated page PPN
2023	new_ppn = ppm_page2ppn( page_xp );
2024
2025	// copy old page content to new page
2026	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
2027	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
2028	memcpy( GET_PTR( new_base_xp ),
2029	GET_PTR( old_base_xp ),
2030	CONFIG_PPM_PAGE_SIZE );
2031
2032	// decrement pending forks counter in page descriptor
2033	hal_remote_atomic_add( forks_xp , -1 );
2034
2035	#if(DEBUG_VMM_HANDLE_COW & 1)
2036	if( DEBUG_VMM_HANDLE_COW < cycle )
2037	printk("\n[%s] thread[%x,%x] : pending forks => allocate a new PPN %x\n",
2038	__FUNCTION__, process->pid, this->trdid, new_ppn );
2039	#endif
2040
2041	}
2042	else // no pending fork => keep the existing page
2043	{
2044
2045	#if(DEBUG_VMM_HANDLE_COW & 1)
2046	if( DEBUG_VMM_HANDLE_COW < cycle )
2047	printk("\n[%s] thread[%x,%x] no pending forks => keep existing PPN %x\n",
2048	__FUNCTION__, process->pid, this->trdid, new_ppn );
2049	#endif
2050	new_ppn = old_ppn;
2051	}
2052
2053	// release lock protecting "forks" counter
2054	remote_busylock_release( forks_lock_xp );
2055
2056	// build new_attr : reset COW and set WRITABLE,
2057	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
2058
2059	// update the relevan GPT
2060	// - private vseg => update local GPT
2061	// - public vseg => update all GPT copies
2062	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2063	{
2064	hal_gpt_set_pte( gpt_xp,
2065	vpn,
2066	new_attr,
2067	new_ppn );
2068	}
2069	else
2070	{
2071	if( ref_cxy == local_cxy ) // reference cluster is local
2072	{
2073	vmm_global_update_pte( process,
2074	vpn,
2075	new_attr,
2076	new_ppn );
2077	}
2078	else // reference cluster is remote
2079	{
2080	rpc_vmm_global_update_pte_client( ref_cxy,
2081	ref_ptr,
2082	vpn,
2083	new_attr,
2084	new_ppn );
2085	}
2086	}
2087
2088	// release GPT lock in write mode
2089	remote_rwlock_wr_release( gpt_lock_xp );
2090
2091	#if DEBUG_VMM_HANDLE_COW
2092	cycle = (uint32_t)hal_get_cycles();
2093	if( DEBUG_VMM_HANDLE_COW < cycle )
2094	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2095	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2096	#endif
2097
2098	return EXCP_NON_FATAL;
2099
2100	} // end vmm_handle_cow()
2101

Note: See TracBrowser for help on using the repository browser.

Download in other formats: