Context Navigation

source: trunk/kernel/mm/vmm.c @ 608

Last change on this file since 608 was 606, checked in by alain, 6 years ago
Improve the FAT32 file system to support cat, rm, cp commands.
File size: 72.4 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations interface.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016,2017,2018)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_rwlock.h>
35	#include <remote_queuelock.h>
36	#include <list.h>
37	#include <xlist.h>
38	#include <bits.h>
39	#include <process.h>
40	#include <thread.h>
41	#include <vseg.h>
42	#include <cluster.h>
43	#include <scheduler.h>
44	#include <vfs.h>
45	#include <mapper.h>
46	#include <page.h>
47	#include <kmem.h>
48	#include <vmm.h>
49	#include <hal_exception.h>
50
51	//////////////////////////////////////////////////////////////////////////////////
52	// Extern global variables
53	//////////////////////////////////////////////////////////////////////////////////
54
55	extern process_t process_zero; // allocated in cluster.c
56
57	///////////////////////////////////////
58	error_t vmm_init( process_t * process )
59	{
60	error_t error;
61	vseg_t * vseg_kentry;
62	vseg_t * vseg_args;
63	vseg_t * vseg_envs;
64	intptr_t base;
65	intptr_t size;
66
67	#if DEBUG_VMM_INIT
68	thread_t * this = CURRENT_THREAD;
69	uint32_t cycle = (uint32_t)hal_get_cycles();
70	if( DEBUG_VMM_INIT )
71	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
72	__FUNCTION__ , this->process->pid, this->trdid, process->pid , cycle );
73	#endif
74
75	// get pointer on VMM
76	vmm_t * vmm = &process->vmm;
77
78	// initialize local list of vsegs
79	vmm->vsegs_nr = 0;
80	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
81	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
82
83	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
84	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
85
86	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
87	"no more than 32 threads per cluster for a single process\n");
88
89	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
90	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
91	"STACK zone too small\n");
92
93	// register kentry vseg in VSL
94	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
95	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
96
97	vseg_kentry = vmm_create_vseg( process,
98	VSEG_TYPE_CODE,
99	base,
100	size,
101	0, // file_offset unused
102	0, // file_size unused
103	XPTR_NULL, // mapper_xp unused
104	local_cxy );
105
106	if( vseg_kentry == NULL )
107	{
108	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
109	return -1;
110	}
111
112	vmm->kent_vpn_base = base;
113
114	// register args vseg in VSL
115	base = (CONFIG_VMM_KENTRY_BASE +
116	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
117	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
118
119	vseg_args = vmm_create_vseg( process,
120	VSEG_TYPE_DATA,
121	base,
122	size,
123	0, // file_offset unused
124	0, // file_size unused
125	XPTR_NULL, // mapper_xp unused
126	local_cxy );
127
128	if( vseg_args == NULL )
129	{
130	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
131	return -1;
132	}
133
134	vmm->args_vpn_base = base;
135
136	// register the envs vseg in VSL
137	base = (CONFIG_VMM_KENTRY_BASE +
138	CONFIG_VMM_KENTRY_SIZE +
139	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
140	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
141
142	vseg_envs = vmm_create_vseg( process,
143	VSEG_TYPE_DATA,
144	base,
145	size,
146	0, // file_offset unused
147	0, // file_size unused
148	XPTR_NULL, // mapper_xp unused
149	local_cxy );
150
151	if( vseg_envs == NULL )
152	{
153	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
154	return -1;
155	}
156
157	vmm->envs_vpn_base = base;
158
159	// create GPT (empty)
160	error = hal_gpt_create( &vmm->gpt );
161
162	if( error )
163	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
164
165	// initialize GPT lock
166	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
167
168	// architecture specic GPT initialisation
169	// (For TSAR, identity map the kentry_vseg)
170	error = hal_vmm_init( vmm );
171
172	if( error )
173	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
174
175	// initialize STACK allocator
176	vmm->stack_mgr.bitmap = 0;
177	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
178	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
179
180	// initialize MMAP allocator
181	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
182	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
183	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
184	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
185
186	uint32_t i;
187	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
188
189	// initialize instrumentation counters
190	vmm->pgfault_nr = 0;
191
192	hal_fence();
193
194	#if DEBUG_VMM_INIT
195	cycle = (uint32_t)hal_get_cycles();
196	if( DEBUG_VMM_INIT )
197	printk("\n[%s] thread[%x,%x] exit / process %x / entry_point %x / cycle %d\n",
198	__FUNCTION__, this->process->pid, this->trdid, process->pid, process->vmm.entry_point, cycle );
199	#endif
200
201	return 0;
202
203	} // end vmm_init()
204
205	//////////////////////////////////////
206	void vmm_display( process_t * process,
207	bool_t mapping )
208	{
209	vmm_t * vmm = &process->vmm;
210	gpt_t * gpt = &vmm->gpt;
211
212	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
213	process->vmm.gpt.ptr , process->pid , local_cxy );
214
215	// get lock protecting the VSL and the GPT
216	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
217	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
218
219	// scan the list of vsegs
220	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
221	xptr_t iter_xp;
222	xptr_t vseg_xp;
223	vseg_t * vseg;
224	XLIST_FOREACH( root_xp , iter_xp )
225	{
226	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
227	vseg = GET_PTR( vseg_xp );
228
229	printk(" - %s : base = %X / size = %X / npages = %d\n",
230	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
231
232	if( mapping )
233	{
234	vpn_t vpn;
235	ppn_t ppn;
236	uint32_t attr;
237	vpn_t base = vseg->vpn_base;
238	vpn_t size = vseg->vpn_size;
239	for( vpn = base ; vpn < (base+size) ; vpn++ )
240	{
241	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
242	if( attr & GPT_MAPPED )
243	{
244	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
245	}
246	}
247	}
248	}
249
250	// release the locks
251	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
252	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
253
254	} // vmm_display()
255
256	///////////////////////////////////
257	void vmm_vseg_attach( vmm_t * vmm,
258	vseg_t * vseg )
259	{
260	// build extended pointer on rwlock protecting VSL
261	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
262
263	// get rwlock in write mode
264	remote_rwlock_wr_acquire( lock_xp );
265
266	// update vseg descriptor
267	vseg->vmm = vmm;
268
269	// add vseg in vmm list
270	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
271	XPTR( local_cxy , &vseg->xlist ) );
272
273	// release rwlock in write mode
274	remote_rwlock_wr_release( lock_xp );
275	}
276
277	///////////////////////////////////
278	void vmm_vseg_detach( vmm_t * vmm,
279	vseg_t * vseg )
280	{
281	// build extended pointer on rwlock protecting VSL
282	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
283
284	// get rwlock in write mode
285	remote_rwlock_wr_acquire( lock_xp );
286
287	// update vseg descriptor
288	vseg->vmm = NULL;
289
290	// remove vseg from vmm list
291	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
292
293	// release rwlock in write mode
294	remote_rwlock_wr_release( lock_xp );
295	}
296
297	////////////////////////////////////////////////
298	void vmm_global_update_pte( process_t * process,
299	vpn_t vpn,
300	uint32_t attr,
301	ppn_t ppn )
302	{
303	xlist_entry_t * process_root_ptr;
304	xptr_t process_root_xp;
305	xptr_t process_iter_xp;
306
307	xptr_t remote_process_xp;
308	cxy_t remote_process_cxy;
309	process_t * remote_process_ptr;
310	xptr_t remote_gpt_xp;
311
312	pid_t pid;
313	cxy_t owner_cxy;
314	lpid_t owner_lpid;
315
316	#if DEBUG_VMM_UPDATE_PTE
317	uint32_t cycle = (uint32_t)hal_get_cycles();
318	thread_t * this = CURRENT_THREAD;
319	if( DEBUG_VMM_UPDATE_PTE < cycle )
320	printk("\n[%s] thread[%x,%x] enter for process %x / vpn %x / cycle %d\n",
321	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
322	#endif
323
324	// check cluster is reference
325	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
326
327	// get extended pointer on root of process copies xlist in owner cluster
328	pid = process->pid;
329	owner_cxy = CXY_FROM_PID( pid );
330	owner_lpid = LPID_FROM_PID( pid );
331	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
332	process_root_xp = XPTR( owner_cxy , process_root_ptr );
333
334	// loop on destination process copies
335	XLIST_FOREACH( process_root_xp , process_iter_xp )
336	{
337	// get cluster and local pointer on remote process
338	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
339	remote_process_ptr = GET_PTR( remote_process_xp );
340	remote_process_cxy = GET_CXY( remote_process_xp );
341
342	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
343	if( DEBUG_VMM_UPDATE_PTE < cycle )
344	printk("\n[%s] threadr[%x,%x] handling vpn %x for process %x in cluster %x\n",
345	__FUNCTION__, this->process->pid, this->trdid, vpn, process->pid, remote_process_cxy );
346	#endif
347
348	// get extended pointer on remote gpt
349	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
350
351	// update remote GPT
352	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
353	}
354
355	#if DEBUG_VMM_UPDATE_PTE
356	cycle = (uint32_t)hal_get_cycles();
357	if( DEBUG_VMM_UPDATE_PTE < cycle )
358	printk("\n[%s] thread[%x,%x] exit for process %x / vpn %x / cycle %d\n",
359	__FUNCTION__, this->process->pid, this->trdid, process->pid , vpn , cycle );
360	#endif
361
362	} // end vmm_global_update_pte()
363
364	///////////////////////////////////////
365	void vmm_set_cow( process_t * process )
366	{
367	vmm_t * vmm;
368
369	xlist_entry_t * process_root_ptr;
370	xptr_t process_root_xp;
371	xptr_t process_iter_xp;
372
373	xptr_t remote_process_xp;
374	cxy_t remote_process_cxy;
375	process_t * remote_process_ptr;
376	xptr_t remote_gpt_xp;
377
378	xptr_t vseg_root_xp;
379	xptr_t vseg_iter_xp;
380
381	xptr_t vseg_xp;
382	vseg_t * vseg;
383
384	pid_t pid;
385	cxy_t owner_cxy;
386	lpid_t owner_lpid;
387
388	#if DEBUG_VMM_SET_COW
389	uint32_t cycle = (uint32_t)hal_get_cycles();
390	thread_t * this = CURRENT_THREAD;
391	if( DEBUG_VMM_SET_COW < cycle )
392	printk("\n[%s] thread[%x,%x] enter for process %x / cycle %d\n",
393	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
394	#endif
395
396	// check cluster is reference
397	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
398	"local cluster is not process reference cluster\n");
399
400	// get pointer on reference VMM
401	vmm = &process->vmm;
402
403	// get extended pointer on root of process copies xlist in owner cluster
404	pid = process->pid;
405	owner_cxy = CXY_FROM_PID( pid );
406	owner_lpid = LPID_FROM_PID( pid );
407	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
408	process_root_xp = XPTR( owner_cxy , process_root_ptr );
409
410	// get extended pointer on root of vsegs xlist from reference VMM
411	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
412
413	// loop on destination process copies
414	XLIST_FOREACH( process_root_xp , process_iter_xp )
415	{
416	// get cluster and local pointer on remote process
417	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
418	remote_process_ptr = GET_PTR( remote_process_xp );
419	remote_process_cxy = GET_CXY( remote_process_xp );
420
421	#if (DEBUG_VMM_SET_COW & 1)
422	if( DEBUG_VMM_SET_COW < cycle )
423	printk("\n[%s] thread[%x,%x] handling process %x in cluster %x\n",
424	__FUNCTION__, this->process->pid, this->trdid, process->pid , remote_process_cxy );
425	#endif
426
427	// get extended pointer on remote gpt
428	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
429
430	// loop on vsegs in (local) reference process VSL
431	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
432	{
433	// get pointer on vseg
434	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
435	vseg = GET_PTR( vseg_xp );
436
437	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
438	"all vsegs in reference VSL must be local\n" );
439
440	// get vseg type, base and size
441	uint32_t type = vseg->type;
442	vpn_t vpn_base = vseg->vpn_base;
443	vpn_t vpn_size = vseg->vpn_size;
444
445	#if (DEBUG_VMM_SET_COW & 1)
446	if( DEBUG_VMM_SET_COW < cycle )
447	printk("\n[%s] thread[%x,%x] handling vseg %s / vpn_base = %x / vpn_size = %x\n",
448	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), vpn_base, vpn_size );
449	#endif
450	// only DATA, ANON and REMOTE vsegs
451	if( (type == VSEG_TYPE_DATA) \|\|
452	(type == VSEG_TYPE_ANON) \|\|
453	(type == VSEG_TYPE_REMOTE) )
454	{
455	vpn_t vpn;
456	uint32_t attr;
457	ppn_t ppn;
458	xptr_t page_xp;
459	cxy_t page_cxy;
460	page_t * page_ptr;
461	xptr_t forks_xp;
462	xptr_t lock_xp;
463
464	// update flags in remote GPT
465	hal_gpt_set_cow( remote_gpt_xp,
466	vpn_base,
467	vpn_size );
468
469	// atomically increment pending forks counter in physical pages,
470	// for all vseg pages that are mapped in reference cluster
471	if( remote_process_cxy == local_cxy )
472	{
473	// scan all pages in vseg
474	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
475	{
476	// get page attributes and PPN from reference GPT
477	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
478
479	// atomically update pending forks counter if page is mapped
480	if( attr & GPT_MAPPED )
481	{
482	// get pointers and cluster on page descriptor
483	page_xp = ppm_ppn2page( ppn );
484	page_cxy = GET_CXY( page_xp );
485	page_ptr = GET_PTR( page_xp );
486
487	// get extended pointers on "forks" and "lock"
488	forks_xp = XPTR( page_cxy , &page_ptr->forks );
489	lock_xp = XPTR( page_cxy , &page_ptr->lock );
490
491	// take lock protecting "forks" counter
492	remote_busylock_acquire( lock_xp );
493
494	// increment "forks"
495	hal_remote_atomic_add( forks_xp , 1 );
496
497	// release lock protecting "forks" counter
498	remote_busylock_release( lock_xp );
499	}
500	} // end loop on vpn
501	} // end if local
502	} // end if vseg type
503	} // end loop on vsegs
504	} // end loop on process copies
505
506	#if DEBUG_VMM_SET_COW
507	cycle = (uint32_t)hal_get_cycles();
508	if( DEBUG_VMM_SET_COW < cycle )
509	printk("\n[%s] thread[%x,%x] exit for process %x / cycle %d\n",
510	__FUNCTION__, this->process->pid, this->trdid, process->pid , cycle );
511	#endif
512
513	} // end vmm_set-cow()
514
515	/////////////////////////////////////////////////
516	error_t vmm_fork_copy( process_t * child_process,
517	xptr_t parent_process_xp )
518	{
519	error_t error;
520	cxy_t parent_cxy;
521	process_t * parent_process;
522	vmm_t * parent_vmm;
523	xptr_t parent_lock_xp;
524	vmm_t * child_vmm;
525	xptr_t iter_xp;
526	xptr_t parent_vseg_xp;
527	vseg_t * parent_vseg;
528	vseg_t * child_vseg;
529	uint32_t type;
530	bool_t cow;
531	vpn_t vpn;
532	vpn_t vpn_base;
533	vpn_t vpn_size;
534	xptr_t page_xp; // extended pointer on page descriptor
535	page_t * page_ptr;
536	cxy_t page_cxy;
537	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
538	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
539	xptr_t parent_root_xp;
540	bool_t mapped;
541	ppn_t ppn;
542
543	#if DEBUG_VMM_FORK_COPY
544	uint32_t cycle = (uint32_t)hal_get_cycles();
545	thread_t * this = CURRENT_THREAD;
546	if( DEBUG_VMM_FORK_COPY < cycle )
547	printk("\n[%s] thread %x enter / cycle %d\n",
548	__FUNCTION__ , this->process->pid, this->trdid, cycle );
549	#endif
550
551	// get parent process cluster and local pointer
552	parent_cxy = GET_CXY( parent_process_xp );
553	parent_process = GET_PTR( parent_process_xp );
554
555	// get local pointers on parent and child VMM
556	parent_vmm = &parent_process->vmm;
557	child_vmm = &child_process->vmm;
558
559	// get extended pointer on lock protecting the parent VSL
560	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
561
562	// initialize the lock protecting the child VSL
563	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
564
565	// initialize the child VSL as empty
566	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
567	child_vmm->vsegs_nr = 0;
568
569	// create child GPT
570	error = hal_gpt_create( &child_vmm->gpt );
571
572	if( error )
573	{
574	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
575	return -1;
576	}
577
578	// build extended pointer on parent VSL
579	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
580
581	// take the lock protecting the parent VSL in read mode
582	remote_rwlock_rd_acquire( parent_lock_xp );
583
584	// loop on parent VSL xlist
585	XLIST_FOREACH( parent_root_xp , iter_xp )
586	{
587	// get local and extended pointers on current parent vseg
588	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
589	parent_vseg = GET_PTR( parent_vseg_xp );
590
591	// get vseg type
592	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
593
594	#if DEBUG_VMM_FORK_COPY
595	cycle = (uint32_t)hal_get_cycles();
596	if( DEBUG_VMM_FORK_COPY < cycle )
597	printk("\n[%s] thread[%x,%x] found parent vseg %s / vpn_base = %x / cycle %d\n",
598	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
599	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
600	#endif
601
602	// all parent vsegs - but STACK - must be copied in child VSL
603	if( type != VSEG_TYPE_STACK )
604	{
605	// allocate memory for a new child vseg
606	child_vseg = vseg_alloc();
607	if( child_vseg == NULL ) // release all allocated vsegs
608	{
609	vmm_destroy( child_process );
610	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
611	return -1;
612	}
613
614	// copy parent vseg to child vseg
615	vseg_init_from_ref( child_vseg , parent_vseg_xp );
616
617	// register child vseg in child VSL
618	vmm_vseg_attach( child_vmm , child_vseg );
619
620	#if DEBUG_VMM_FORK_COPY
621	cycle = (uint32_t)hal_get_cycles();
622	if( DEBUG_VMM_FORK_COPY < cycle )
623	printk("\n[%s] thread[%x,%x] copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
624	__FUNCTION__ , this->process->pid, this->trdid, vseg_type_str(type),
625	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
626	#endif
627
628	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
629	if( type != VSEG_TYPE_CODE )
630	{
631	// activate the COW for DATA, MMAP, REMOTE vsegs only
632	cow = ( type != VSEG_TYPE_FILE );
633
634	vpn_base = child_vseg->vpn_base;
635	vpn_size = child_vseg->vpn_size;
636
637	// scan pages in parent vseg
638	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
639	{
640	error = hal_gpt_pte_copy( &child_vmm->gpt,
641	XPTR( parent_cxy , &parent_vmm->gpt ),
642	vpn,
643	cow,
644	&ppn,
645	&mapped );
646	if( error )
647	{
648	vmm_destroy( child_process );
649	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
650	return -1;
651	}
652
653	// increment pending forks counter in page if mapped
654	if( mapped )
655	{
656	// get pointers and cluster on page descriptor
657	page_xp = ppm_ppn2page( ppn );
658	page_cxy = GET_CXY( page_xp );
659	page_ptr = GET_PTR( page_xp );
660
661	// get extended pointers on "forks" and "lock"
662	forks_xp = XPTR( page_cxy , &page_ptr->forks );
663	lock_xp = XPTR( page_cxy , &page_ptr->lock );
664
665	// get lock protecting "forks" counter
666	remote_busylock_acquire( lock_xp );
667
668	// increment "forks"
669	hal_remote_atomic_add( forks_xp , 1 );
670
671	// release lock protecting "forks" counter
672	remote_busylock_release( lock_xp );
673
674	#if DEBUG_VMM_FORK_COPY
675	cycle = (uint32_t)hal_get_cycles();
676	if( DEBUG_VMM_FORK_COPY < cycle )
677	printk("\n[%s] thread[%x,%x] copied vpn %x to child GPT / cycle %d\n",
678	__FUNCTION__ , this->process->pid, this->trdid , vpn , cycle );
679	#endif
680	}
681	}
682	} // end if no code & no stack
683	} // end if no stack
684	} // end loop on vsegs
685
686	// release the parent VSL lock in read mode
687	remote_rwlock_rd_release( parent_lock_xp );
688
689	// initialize child GPT (architecture specic)
690	// => For TSAR, identity map the kentry_vseg
691	error = hal_vmm_init( child_vmm );
692
693	if( error )
694	{
695	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
696	return -1;
697	}
698
699	// initialize the child VMM STACK allocator
700	child_vmm->stack_mgr.bitmap = 0;
701	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
702
703	// initialize the child VMM MMAP allocator
704	uint32_t i;
705	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
706	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
707	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
708	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
709
710	// initialize instrumentation counters
711	child_vmm->pgfault_nr = 0;
712
713	// copy base addresses from parent VMM to child VMM
714	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
715	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
716	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
717	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
718	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
719	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
720
721	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
722
723	hal_fence();
724
725	#if DEBUG_VMM_FORK_COPY
726	cycle = (uint32_t)hal_get_cycles();
727	if( DEBUG_VMM_FORK_COPY < cycle )
728	printk("\n[%s] thread[%x,%x] exit successfully / cycle %d\n",
729	__FUNCTION__ , this->process->pid, this->trdid , cycle );
730	#endif
731
732	return 0;
733
734	} // vmm_fork_copy()
735
736	///////////////////////////////////////
737	void vmm_destroy( process_t * process )
738	{
739	xptr_t vseg_xp;
740	vseg_t * vseg;
741
742	#if DEBUG_VMM_DESTROY
743	uint32_t cycle = (uint32_t)hal_get_cycles();
744	thread_t * this = CURRENT_THREAD;
745	if( DEBUG_VMM_DESTROY < cycle )
746	printk("\n[%s] thread[%x,%x] enter for process %x in cluster %x / cycle %d\n",
747	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy, cycle );
748	#endif
749
750	#if (DEBUG_VMM_DESTROY & 1 )
751	if( DEBUG_VMM_DESTROY < cycle )
752	vmm_display( process , true );
753	#endif
754
755	// get pointer on local VMM
756	vmm_t * vmm = &process->vmm;
757
758	// get extended pointer on VSL root and VSL lock
759	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
760
761	// remove all user vsegs registered in VSL
762	while( !xlist_is_empty( root_xp ) )
763	{
764	// get pointer on first vseg in VSL
765	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
766	vseg = GET_PTR( vseg_xp );
767
768	// unmap and release physical pages
769	vmm_unmap_vseg( process , vseg );
770
771	// remove vseg from VSL
772	vmm_vseg_detach( vmm , vseg );
773
774	// release memory allocated to vseg descriptor
775	vseg_free( vseg );
776
777	#if( DEBUG_VMM_DESTROY & 1 )
778	if( DEBUG_VMM_DESTROY < cycle )
779	printk("\n[%s] %s vseg released / vpn_base %x / vpn_size %d\n",
780	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
781	#endif
782
783	}
784
785	// remove all vsegs from zombi_lists in MMAP allocator
786	uint32_t i;
787	for( i = 0 ; i<32 ; i++ )
788	{
789	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
790	{
791	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
792
793	#if( DEBUG_VMM_DESTROY & 1 )
794	if( DEBUG_VMM_DESTROY < cycle )
795	printk("\n[%s] found zombi vseg / vpn_base %x / vpn_size %d\n",
796	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
797	#endif
798	vmm_vseg_detach( vmm , vseg );
799	vseg_free( vseg );
800
801	#if( DEBUG_VMM_DESTROY & 1 )
802	if( DEBUG_VMM_DESTROY < cycle )
803	printk("\n[%s] zombi vseg released / vpn_base %x / vpn_size %d\n",
804	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
805	#endif
806	}
807	}
808
809	// release memory allocated to the GPT itself
810	hal_gpt_destroy( &vmm->gpt );
811
812	#if DEBUG_VMM_DESTROY
813	cycle = (uint32_t)hal_get_cycles();
814	if( DEBUG_VMM_DESTROY < cycle )
815	printk("\n[%s] thread[%x,%x] exit for process %x in cluster %x / cycle %d\n",
816	__FUNCTION__, this->process->pid, this->trdid, process->pid, local_cxy , cycle );
817	#endif
818
819	} // end vmm_destroy()
820
821	/////////////////////////////////////////////////
822	vseg_t * vmm_check_conflict( process_t * process,
823	vpn_t vpn_base,
824	vpn_t vpn_size )
825	{
826	vmm_t * vmm = &process->vmm;
827
828	// scan the VSL
829	vseg_t * vseg;
830	xptr_t iter_xp;
831	xptr_t vseg_xp;
832	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
833
834	XLIST_FOREACH( root_xp , iter_xp )
835	{
836	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
837	vseg = GET_PTR( vseg_xp );
838
839	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
840	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
841	}
842	return NULL;
843
844	} // end vmm_check_conflict()
845
846	////////////////////////////////////////////////////////////////////////////////////////////
847	// This static function is called by the vmm_create_vseg() function, and implements
848	// the VMM stack_vseg specific allocator.
849	////////////////////////////////////////////////////////////////////////////////////////////
850	// @ vmm : pointer on VMM.
851	// @ vpn_base : (return value) first allocated page
852	// @ vpn_size : (return value) number of allocated pages
853	////////////////////////////////////////////////////////////////////////////////////////////
854	static error_t vmm_stack_alloc( vmm_t * vmm,
855	vpn_t * vpn_base,
856	vpn_t * vpn_size )
857	{
858	// get stack allocator pointer
859	stack_mgr_t * mgr = &vmm->stack_mgr;
860
861	// get lock on stack allocator
862	busylock_acquire( &mgr->lock );
863
864	// get first free slot index in bitmap
865	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
866	if( (index < 0) \|\| (index > 31) )
867	{
868	busylock_release( &mgr->lock );
869	return 0xFFFFFFFF;
870	}
871
872	// update bitmap
873	bitmap_set( &mgr->bitmap , index );
874
875	// release lock on stack allocator
876	busylock_release( &mgr->lock );
877
878	// returns vpn_base, vpn_size (one page non allocated)
879	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
880	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
881	return 0;
882
883	} // end vmm_stack_alloc()
884
885	////////////////////////////////////////////////////////////////////////////////////////////
886	// This static function is called by the vmm_create_vseg() function, and implements
887	// the VMM MMAP specific allocator.
888	////////////////////////////////////////////////////////////////////////////////////////////
889	// @ vmm : [in] pointer on VMM.
890	// @ npages : [in] requested number of pages.
891	// @ vpn_base : [out] first allocated page.
892	// @ vpn_size : [out] actual number of allocated pages.
893	////////////////////////////////////////////////////////////////////////////////////////////
894	static error_t vmm_mmap_alloc( vmm_t * vmm,
895	vpn_t npages,
896	vpn_t * vpn_base,
897	vpn_t * vpn_size )
898	{
899	uint32_t index;
900	vseg_t * vseg;
901	vpn_t base;
902	vpn_t size;
903	vpn_t free;
904
905	// mmap vseg size must be power of 2
906	// compute actual size and index in zombi_list array
907	size = POW2_ROUNDUP( npages );
908	index = bits_log2( size );
909
910	// get mmap allocator pointer
911	mmap_mgr_t * mgr = &vmm->mmap_mgr;
912
913	// get lock on mmap allocator
914	busylock_acquire( &mgr->lock );
915
916	// get vseg from zombi_list or from mmap zone
917	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
918	{
919	// check overflow
920	free = mgr->first_free_vpn;
921	if( (free + size) > mgr->vpn_size ) return ENOMEM;
922
923	// update STACK allocator
924	mgr->first_free_vpn += size;
925
926	// compute base
927	base = free;
928	}
929	else // from zombi_list
930	{
931	// get pointer on zombi vseg from zombi_list
932	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
933
934	// remove vseg from free-list
935	list_unlink( &vseg->zlist );
936
937	// compute base
938	base = vseg->vpn_base;
939	}
940
941	// release lock on mmap allocator
942	busylock_release( &mgr->lock );
943
944	// returns vpn_base, vpn_size
945	*vpn_base = base;
946	*vpn_size = size;
947	return 0;
948
949	} // end vmm_mmap_alloc()
950
951	////////////////////////////////////////////////
952	vseg_t * vmm_create_vseg( process_t * process,
953	vseg_type_t type,
954	intptr_t base,
955	uint32_t size,
956	uint32_t file_offset,
957	uint32_t file_size,
958	xptr_t mapper_xp,
959	cxy_t cxy )
960	{
961	vseg_t * vseg; // created vseg pointer
962	vpn_t vpn_base; // first page index
963	vpn_t vpn_size; // number of pages covered by vseg
964	error_t error;
965
966	#if DEBUG_VMM_CREATE_VSEG
967	thread_t * this = CURRENT_THREAD;
968	uint32_t cycle = (uint32_t)hal_get_cycles();
969	if( DEBUG_VMM_CREATE_VSEG < cycle )
970	printk("\n[%s] thread[%x,%x] enter / %s / cxy %x / cycle %d\n",
971	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
972	#endif
973
974	// get pointer on VMM
975	vmm_t * vmm = &process->vmm;
976
977	// compute base, size, vpn_base, vpn_size, depending on vseg type
978	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
979
980	if( type == VSEG_TYPE_STACK )
981	{
982	// get vpn_base and vpn_size from STACK allocator
983	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
984	if( error )
985	{
986	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
987	__FUNCTION__ , process->pid , local_cxy );
988	return NULL;
989	}
990
991	// compute vseg base and size from vpn_base and vpn_size
992	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
993	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
994	}
995	else if( type == VSEG_TYPE_FILE )
996	{
997	// compute page index (in mapper) for first byte
998	vpn_t vpn_min = file_offset >> CONFIG_PPM_PAGE_SHIFT;
999
1000	// compute page index (in mapper) for last byte
1001	vpn_t vpn_max = (file_offset + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1002
1003	// compute offset in first page
1004	uint32_t offset = file_offset & CONFIG_PPM_PAGE_MASK;
1005
1006	// compute number of pages required in virtual space
1007	vpn_t npages = vpn_max - vpn_min + 1;
1008
1009	// get vpn_base and vpn_size from MMAP allocator
1010	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1011	if( error )
1012	{
1013	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1014	__FUNCTION__ , process->pid , local_cxy );
1015	return NULL;
1016	}
1017
1018	// set the vseg base (not always aligned for FILE)
1019	base = (vpn_base << CONFIG_PPM_PAGE_SHIFT) + offset;
1020	}
1021	else if( (type == VSEG_TYPE_ANON) \|\|
1022	(type == VSEG_TYPE_REMOTE) )
1023	{
1024	// compute number of required pages in virtual space
1025	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
1026	if( size & CONFIG_PPM_PAGE_MASK) npages++;
1027
1028	// get vpn_base and vpn_size from MMAP allocator
1029	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
1030	if( error )
1031	{
1032	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
1033	__FUNCTION__ , process->pid , local_cxy );
1034	return NULL;
1035	}
1036
1037	// set vseg base (always aligned for ANON or REMOTE)
1038	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1039	}
1040	else // VSEG_TYPE_DATA or VSEG_TYPE_CODE
1041	{
1042	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1043	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1044
1045	vpn_base = vpn_min;
1046	vpn_size = vpn_max - vpn_min + 1;
1047	}
1048
1049	// check collisions
1050	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1051	if( vseg != NULL )
1052	{
1053	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
1054	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
1055	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1056	return NULL;
1057	}
1058
1059	// allocate physical memory for vseg descriptor
1060	vseg = vseg_alloc();
1061	if( vseg == NULL )
1062	{
1063	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1064	__FUNCTION__ , process->pid );
1065	return NULL;
1066	}
1067
1068	// initialize vseg descriptor
1069	vseg_init( vseg,
1070	type,
1071	base,
1072	size,
1073	vpn_base,
1074	vpn_size,
1075	file_offset,
1076	file_size,
1077	mapper_xp,
1078	cxy );
1079
1080	// attach vseg to VSL
1081	vmm_vseg_attach( vmm , vseg );
1082
1083	#if DEBUG_VMM_CREATE_VSEG
1084	cycle = (uint32_t)hal_get_cycles();
1085	if( DEBUG_VMM_CREATE_VSEG < cycle )
1086	printk("\n[%s] thread[%x,%x] exit / %s / cxy %x / cycle %d\n",
1087	__FUNCTION__, this->process->pid, this->trdid, vseg_type_str(type), cxy, cycle );
1088	#endif
1089
1090	return vseg;
1091
1092	} // vmm_create_vseg()
1093
1094	/////////////////////////////////////
1095	void vmm_remove_vseg( vseg_t * vseg )
1096	{
1097	// get pointers on calling process and VMM
1098	thread_t * this = CURRENT_THREAD;
1099	vmm_t * vmm = &this->process->vmm;
1100	uint32_t type = vseg->type;
1101
1102	// detach vseg from VSL
1103	vmm_vseg_detach( vmm , vseg );
1104
1105	// release the stack slot to VMM stack allocator if STACK type
1106	if( type == VSEG_TYPE_STACK )
1107	{
1108	// get pointer on stack allocator
1109	stack_mgr_t * mgr = &vmm->stack_mgr;
1110
1111	// compute slot index
1112	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
1113
1114	// update stacks_bitmap
1115	busylock_acquire( &mgr->lock );
1116	bitmap_clear( &mgr->bitmap , index );
1117	busylock_release( &mgr->lock );
1118	}
1119
1120	// release the vseg to VMM mmap allocator if MMAP type
1121	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
1122	{
1123	// get pointer on mmap allocator
1124	mmap_mgr_t * mgr = &vmm->mmap_mgr;
1125
1126	// compute zombi_list index
1127	uint32_t index = bits_log2( vseg->vpn_size );
1128
1129	// update zombi_list
1130	busylock_acquire( &mgr->lock );
1131	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
1132	busylock_release( &mgr->lock );
1133	}
1134
1135	// release physical memory allocated for vseg descriptor if no MMAP type
1136	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
1137	{
1138	vseg_free( vseg );
1139	}
1140	} // end vmm_remove_vseg()
1141
1142	/////////////////////////////////////////
1143	void vmm_unmap_vseg( process_t * process,
1144	vseg_t * vseg )
1145	{
1146	vpn_t vpn; // VPN of current PTE
1147	vpn_t vpn_min; // VPN of first PTE
1148	vpn_t vpn_max; // VPN of last PTE (excluded)
1149	ppn_t ppn; // current PTE ppn value
1150	uint32_t attr; // current PTE attributes
1151	kmem_req_t req; // request to release memory
1152	xptr_t page_xp; // extended pointer on page descriptor
1153	cxy_t page_cxy; // page descriptor cluster
1154	page_t * page_ptr; // page descriptor pointer
1155	xptr_t forks_xp; // extended pointer on pending forks counter
1156	xptr_t lock_xp; // extended pointer on lock protecting forks counter
1157	uint32_t forks; // actual number of pendinf forks
1158
1159	#if DEBUG_VMM_UNMAP_VSEG
1160	uint32_t cycle = (uint32_t)hal_get_cycles();
1161	thread_t * this = CURRENT_THREAD;
1162	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1163	printk("\n[%s] thread[%x,%x] enter / process %x / vseg %s / base %x / cycle %d\n",
1164	__FUNCTION__, this->process->pid, this->trdid, process->pid,
1165	vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
1166	#endif
1167
1168	// get pointer on local GPT
1169	gpt_t * gpt = &process->vmm.gpt;
1170
1171	// loop on pages in vseg
1172	vpn_min = vseg->vpn_base;
1173	vpn_max = vpn_min + vseg->vpn_size;
1174	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1175	{
1176	// get GPT entry
1177	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
1178
1179	if( attr & GPT_MAPPED ) // entry is mapped
1180	{
1181
1182	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
1183	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1184	printk("- vpn %x / ppn %x\n" , vpn , ppn );
1185	#endif
1186
1187	// check small page
1188	assert( (attr & GPT_SMALL) , "an user vseg must use small pages" );
1189
1190	// unmap GPT entry in local GPT
1191	hal_gpt_reset_pte( gpt , vpn );
1192
1193	// handle pending forks counter if
1194	// 1) not identity mapped
1195	// 2) reference cluster
1196	if( ((vseg->flags & VSEG_IDENT) == 0) &&
1197	(GET_CXY( process->ref_xp ) == local_cxy) )
1198	{
1199	// get extended pointer on physical page descriptor
1200	page_xp = ppm_ppn2page( ppn );
1201	page_cxy = GET_CXY( page_xp );
1202	page_ptr = GET_PTR( page_xp );
1203
1204	// get extended pointers on forks and lock fields
1205	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1206	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1207
1208	// get pending forks counter
1209	forks = hal_remote_l32( forks_xp );
1210
1211	if( forks ) // decrement pending forks counter
1212	{
1213	hal_remote_atomic_add( forks_xp , -1 );
1214	}
1215	else // release physical page to relevant cluster
1216	{
1217	if( page_cxy == local_cxy ) // local cluster
1218	{
1219	req.type = KMEM_PAGE;
1220	req.ptr = page_ptr;
1221	kmem_free( &req );
1222	}
1223	else // remote cluster
1224	{
1225	rpc_pmem_release_pages_client( page_cxy , page_ptr );
1226	}
1227	}
1228	}
1229	}
1230	}
1231
1232	#if DEBUG_VMM_UNMAP_VSEG
1233	cycle = (uint32_t)hal_get_cycles();
1234	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1235	printk("\n[%s] thread[%x,%x] exit / process %x / vseg %s / base %x / cycle %d\n",
1236	__FUNCTION__, this->process->pid, this->trdid, process->pid,
1237	vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
1238	#endif
1239
1240	} // end vmm_unmap_vseg()
1241
1242	//////////////////////////////////////////////////////////////////////////////////////////
1243	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
1244	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
1245	// containing a given virtual address.
1246	//////////////////////////////////////////////////////////////////////////////////////////
1247	// @ vmm : pointer on the process VMM.
1248	// @ vaddr : virtual address.
1249	// @ return vseg pointer if success / return NULL if not found.
1250	//////////////////////////////////////////////////////////////////////////////////////////
1251	static vseg_t * vmm_vseg_from_vaddr( vmm_t * vmm,
1252	intptr_t vaddr )
1253	{
1254	xptr_t iter_xp;
1255	xptr_t vseg_xp;
1256	vseg_t * vseg;
1257
1258	// get extended pointers on VSL lock and root
1259	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1260	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1261
1262	// get lock protecting the VSL
1263	remote_rwlock_rd_acquire( lock_xp );
1264
1265	// scan the list of vsegs in VSL
1266	XLIST_FOREACH( root_xp , iter_xp )
1267	{
1268	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1269	vseg = GET_PTR( vseg_xp );
1270
1271	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
1272	{
1273	// return success
1274	remote_rwlock_rd_release( lock_xp );
1275	return vseg;
1276	}
1277	}
1278
1279	// return failure
1280	remote_rwlock_rd_release( lock_xp );
1281	return NULL;
1282
1283	} // end vmm_vseg_from_vaddr()
1284
1285	/////////////////////////////////////////////
1286	error_t vmm_resize_vseg( process_t * process,
1287	intptr_t base,
1288	intptr_t size )
1289	{
1290	error_t error;
1291	vseg_t * new;
1292	vpn_t vpn_min;
1293	vpn_t vpn_max;
1294
1295	// get pointer on process VMM
1296	vmm_t * vmm = &process->vmm;
1297
1298	intptr_t addr_min = base;
1299	intptr_t addr_max = base + size;
1300
1301	// get pointer on vseg
1302	vseg_t * vseg = vmm_vseg_from_vaddr( vmm , base );
1303
1304	if( vseg == NULL) return EINVAL;
1305
1306	// get extended pointer on VSL lock
1307	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1308
1309	// get lock protecting VSL
1310	remote_rwlock_wr_acquire( lock_xp );
1311
1312	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
1313	{
1314	error = EINVAL;
1315	}
1316	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
1317	{
1318	vmm_remove_vseg( vseg );
1319	error = 0;
1320	}
1321	else if( vseg->min == addr_min ) // vseg must be resized
1322	{
1323	// update vseg base address
1324	vseg->min = addr_max;
1325
1326	// update vpn_base and vpn_size
1327	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1328	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1329	vseg->vpn_base = vpn_min;
1330	vseg->vpn_size = vpn_max - vpn_min + 1;
1331	error = 0;
1332	}
1333	else if( vseg->max == addr_max ) // vseg must be resized
1334	{
1335	// update vseg max address
1336	vseg->max = addr_min;
1337
1338	// update vpn_base and vpn_size
1339	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1340	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1341	vseg->vpn_base = vpn_min;
1342	vseg->vpn_size = vpn_max - vpn_min + 1;
1343	error = 0;
1344	}
1345	else // vseg cut in three regions
1346	{
1347	// resize existing vseg
1348	vseg->max = addr_min;
1349
1350	// update vpn_base and vpn_size
1351	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1352	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1353	vseg->vpn_base = vpn_min;
1354	vseg->vpn_size = vpn_max - vpn_min + 1;
1355
1356	// create new vseg
1357	new = vmm_create_vseg( process,
1358	vseg->type,
1359	addr_min,
1360	(vseg->max - addr_max),
1361	vseg->file_offset,
1362	vseg->file_size,
1363	vseg->mapper_xp,
1364	vseg->cxy );
1365
1366	if( new == NULL ) error = EINVAL;
1367	else error = 0;
1368	}
1369
1370	// release VMM lock
1371	remote_rwlock_wr_release( lock_xp );
1372
1373	return error;
1374
1375	} // vmm_resize_vseg()
1376
1377	///////////////////////////////////////////
1378	error_t vmm_get_vseg( process_t * process,
1379	intptr_t vaddr,
1380	vseg_t ** found_vseg )
1381	{
1382	xptr_t vseg_xp;
1383	vseg_t * vseg;
1384	vmm_t * vmm;
1385	error_t error;
1386
1387	// get pointer on local VMM
1388	vmm = &process->vmm;
1389
1390	// try to get vseg from local VMM
1391	vseg = vmm_vseg_from_vaddr( vmm , vaddr );
1392
1393	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
1394	{
1395	// get extended pointer on reference process
1396	xptr_t ref_xp = process->ref_xp;
1397
1398	// get cluster and local pointer on reference process
1399	cxy_t ref_cxy = GET_CXY( ref_xp );
1400	process_t * ref_ptr = GET_PTR( ref_xp );
1401
1402	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
1403
1404	// get extended pointer on reference vseg
1405	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
1406
1407	if( error ) return -1; // vseg not found => illegal user vaddr
1408
1409	// allocate a vseg in local cluster
1410	vseg = vseg_alloc();
1411
1412	if( vseg == NULL ) return -1; // cannot allocate a local vseg
1413
1414	// initialise local vseg from reference
1415	vseg_init_from_ref( vseg , vseg_xp );
1416
1417	// register local vseg in local VMM
1418	vmm_vseg_attach( vmm , vseg );
1419	}
1420
1421	// success
1422	*found_vseg = vseg;
1423	return 0;
1424
1425	} // end vmm_get_vseg()
1426
1427	//////////////////////////////////////////////////////////////////////////////////////
1428	// This static function compute the target cluster to allocate a physical page
1429	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
1430	// and returns an extended pointer on the allocated page descriptor.
1431	// It can be called by a thread running in any cluster.
1432	// The vseg cannot have the FILE type.
1433	//////////////////////////////////////////////////////////////////////////////////////
1434	static xptr_t vmm_page_allocate( vseg_t * vseg,
1435	vpn_t vpn )
1436	{
1437
1438	#if DEBUG_VMM_ALLOCATE_PAGE
1439	uint32_t cycle = (uint32_t)hal_get_cycles();
1440	thread_t * this = CURRENT_THREAD;
1441	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1442	printk("\n[%s] thread[%x,%x] enter for vpn %x / cycle %d\n",
1443	__FUNCTION__ , this->process->pid, this->trdid, vpn, cycle );
1444	#endif
1445
1446	// compute target cluster
1447	page_t * page_ptr;
1448	cxy_t page_cxy;
1449	kmem_req_t req;
1450	uint32_t index;
1451
1452	uint32_t type = vseg->type;
1453	uint32_t flags = vseg->flags;
1454	uint32_t x_size = LOCAL_CLUSTER->x_size;
1455	uint32_t y_size = LOCAL_CLUSTER->y_size;
1456
1457	// check vseg type
1458	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1459
1460	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1461	{
1462	index = vpn & ((x_size * y_size) - 1);
1463	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1464
1465	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1466	if ( cluster_is_active( page_cxy ) == false )
1467	{
1468	page_cxy = cluster_random_select();
1469	}
1470	}
1471	else // other cases => cxy specified in vseg
1472	{
1473	page_cxy = vseg->cxy;
1474	}
1475
1476	// allocate a physical page from target cluster
1477	if( page_cxy == local_cxy ) // target cluster is the local cluster
1478	{
1479	req.type = KMEM_PAGE;
1480	req.size = 0;
1481	req.flags = AF_NONE;
1482	page_ptr = (page_t *)kmem_alloc( &req );
1483	}
1484	else // target cluster is not the local cluster
1485	{
1486	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
1487	}
1488
1489	#if DEBUG_VMM_ALLOCATE_PAGE
1490	cycle = (uint32_t)hal_get_cycles();
1491	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1492	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle %d\n",
1493	__FUNCTION__ , this->process->pid, this->trdid, vpn,
1494	ppm_page2ppn( XPTR( page_cxy , page_ptr ) , cycle );
1495	#endif
1496
1497	if( page_ptr == NULL ) return XPTR_NULL;
1498	else return XPTR( page_cxy , page_ptr );
1499
1500	} // end vmm_page_allocate()
1501
1502	////////////////////////////////////////
1503	error_t vmm_get_one_ppn( vseg_t * vseg,
1504	vpn_t vpn,
1505	ppn_t * ppn )
1506	{
1507	error_t error;
1508	xptr_t page_xp; // extended pointer on physical page descriptor
1509	uint32_t page_id; // missing page index in vseg mapper
1510	uint32_t type; // vseg type;
1511
1512	type = vseg->type;
1513	page_id = vpn - vseg->vpn_base;
1514
1515	#if DEBUG_VMM_GET_ONE_PPN
1516	uint32_t cycle = (uint32_t)hal_get_cycles();
1517	thread_t * this = CURRENT_THREAD;
1518	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1519	printk("\n[%s] thread[%x,%x] enter for vpn %x / type %s / page_id %d / cycle %d\n",
1520	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(type), page_id, cycle );
1521	#endif
1522
1523	// FILE type : get the physical page from the file mapper
1524	if( type == VSEG_TYPE_FILE )
1525	{
1526	// get extended pointer on mapper
1527	xptr_t mapper_xp = vseg->mapper_xp;
1528
1529	assert( (mapper_xp != XPTR_NULL),
1530	"mapper not defined for a FILE vseg\n" );
1531
1532	// get extended pointer on page descriptor
1533	page_xp = mapper_remote_get_page( mapper_xp , page_id );
1534
1535	if ( page_xp == XPTR_NULL ) return EINVAL;
1536	}
1537
1538	// Other types : allocate a physical page from target cluster,
1539	// as defined by vseg type and vpn value
1540	else
1541	{
1542	// allocate one physical page
1543	page_xp = vmm_page_allocate( vseg , vpn );
1544
1545	if( page_xp == XPTR_NULL ) return ENOMEM;
1546
1547	// initialise missing page from .elf file mapper for DATA and CODE types
1548	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1549	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1550	{
1551	// get extended pointer on mapper
1552	xptr_t mapper_xp = vseg->mapper_xp;
1553
1554	assert( (mapper_xp != XPTR_NULL),
1555	"mapper not defined for a CODE or DATA vseg\n" );
1556
1557	// compute missing page offset in vseg
1558	uint32_t offset = page_id << CONFIG_PPM_PAGE_SHIFT;
1559
1560	// compute missing page offset in .elf file
1561	uint32_t elf_offset = vseg->file_offset + offset;
1562
1563	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1564	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1565	printk("\n[%s] thread[%x,%x] for vpn = %x / elf_offset = %x\n",
1566	__FUNCTION__, this->process->pid, this->trdid, vpn, elf_offset );
1567	#endif
1568	// compute extended pointer on page base
1569	xptr_t base_xp = ppm_page2base( page_xp );
1570
1571	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1572	uint32_t file_size = vseg->file_size;
1573
1574	if( file_size < offset ) // missing page fully in BSS
1575	{
1576
1577	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1578	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1579	printk("\n[%s] thread[%x,%x] for vpn %x / fully in BSS\n",
1580	__FUNCTION__, this->process->pid, this->trdid, vpn );
1581	#endif
1582	if( GET_CXY( page_xp ) == local_cxy )
1583	{
1584	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1585	}
1586	else
1587	{
1588	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
1589	}
1590	}
1591	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
1592	{
1593
1594	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1595	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1596	printk("\n[%s] thread[%x,%x] for vpn %x / fully in mapper\n",
1597	__FUNCTION__, this->process->pid, this->trdid, vpn );
1598	#endif
1599	error = mapper_move_kernel( mapper_xp,
1600	true, // to_buffer
1601	elf_offset,
1602	base_xp,
1603	CONFIG_PPM_PAGE_SIZE );
1604	if( error ) return EINVAL;
1605	}
1606	else // both in mapper and in BSS :
1607	// - (file_size - offset) bytes from mapper
1608	// - (page_size + offset - file_size) bytes from BSS
1609	{
1610
1611	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1612	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1613	printk("\n[%s] thread[%x,%x] for vpn %x / both mapper & BSS\n",
1614	" %d bytes from mapper / %d bytes from BSS\n",
1615	__FUNCTION__, this->process->pid, this->trdid, vpn,
1616	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
1617	#endif
1618	// initialize mapper part
1619	error = mapper_move_kernel( mapper_xp,
1620	true, // to buffer
1621	elf_offset,
1622	base_xp,
1623	file_size - offset );
1624	if( error ) return EINVAL;
1625
1626	// initialize BSS part
1627	if( GET_CXY( page_xp ) == local_cxy )
1628	{
1629	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
1630	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1631	}
1632	else
1633	{
1634	hal_remote_memset( base_xp + file_size - offset , 0 ,
1635	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1636	}
1637	}
1638	} // end initialisation for CODE or DATA types
1639	}
1640
1641	// return ppn
1642	*ppn = ppm_page2ppn( page_xp );
1643
1644	#if DEBUG_VMM_GET_ONE_PPN
1645	cycle = (uint32_t)hal_get_cycles();
1646	if( DEBUG_VMM_GET_ONE_PPN < cycle )
1647	printk("\n[%s] thread[%x,%x] exit for vpn %x / ppn %x / cycle\n",
1648	__FUNCTION__ , this->process->pid, this->trdid , vpn , *ppn, cycle );
1649	#endif
1650
1651	return 0;
1652
1653	} // end vmm_get_one_ppn()
1654
1655	///////////////////////////////////////////////////
1656	error_t vmm_handle_page_fault( process_t * process,
1657	vpn_t vpn )
1658	{
1659	vseg_t * vseg; // vseg containing vpn
1660	uint32_t new_attr; // new PTE_ATTR value
1661	ppn_t new_ppn; // new PTE_PPN value
1662	uint32_t ref_attr; // PTE_ATTR value in reference GPT
1663	ppn_t ref_ppn; // PTE_PPN value in reference GPT
1664	cxy_t ref_cxy; // reference cluster for missing vpn
1665	process_t * ref_ptr; // reference process for missing vpn
1666	xptr_t local_gpt_xp; // extended pointer on local GPT
1667	xptr_t local_lock_xp; // extended pointer on local GPT lock
1668	xptr_t ref_gpt_xp; // extended pointer on reference GPT
1669	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
1670	error_t error; // value returned by called functions
1671
1672	// get local vseg (access to reference VSL can be required)
1673	error = vmm_get_vseg( process,
1674	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1675	&vseg );
1676
1677	if( error )
1678	{
1679	printk("\n[ERROR] in %s : vpn %x in process %x not in a registered vseg\n",
1680	__FUNCTION__ , vpn , process->pid );
1681
1682	return EXCP_USER_ERROR;
1683	}
1684
1685	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1686	uint32_t cycle = (uint32_t)hal_get_cycles();
1687	thread_t * this = CURRENT_THREAD;
1688	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1689	printk("\n[%s] threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
1690	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
1691	#endif
1692
1693	//////////////// private vseg => access only the local GPT
1694	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1695	{
1696	// build extended pointer on local GPT and local GPT lock
1697	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1698	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1699
1700	// take local GPT lock in write mode
1701	remote_rwlock_wr_acquire( local_lock_xp );
1702
1703	// check VPN still unmapped in local GPT
1704
1705	// do nothing if VPN has been mapped by a a concurrent page_fault
1706	hal_gpt_get_pte( local_gpt_xp,
1707	vpn,
1708	&new_attr,
1709	&new_ppn );
1710
1711	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1712	{
1713	// allocate and initialise a physical page depending on the vseg type
1714	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1715
1716	if( error )
1717	{
1718	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1719	__FUNCTION__ , process->pid , vpn );
1720
1721	// release local GPT lock in write mode
1722	remote_rwlock_wr_release( local_lock_xp );
1723
1724	return EXCP_KERNEL_PANIC;
1725	}
1726
1727	// define new_attr from vseg flags
1728	new_attr = GPT_MAPPED \| GPT_SMALL;
1729	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1730	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1731	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1732	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1733
1734	// set PTE (PPN & attribute) to local GPT
1735	error = hal_gpt_set_pte( local_gpt_xp,
1736	vpn,
1737	new_attr,
1738	new_ppn );
1739	if ( error )
1740	{
1741	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1742	__FUNCTION__ , process->pid , vpn );
1743
1744	// release local GPT lock in write mode
1745	remote_rwlock_wr_release( local_lock_xp );
1746
1747	return EXCP_KERNEL_PANIC;
1748	}
1749	}
1750
1751	// release local GPT lock in write mode
1752	remote_rwlock_wr_release( local_lock_xp );
1753
1754	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1755	cycle = (uint32_t)hal_get_cycles();
1756	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1757	printk("\n[%s] private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1758	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1759	#endif
1760	return EXCP_NON_FATAL;
1761
1762	} // end local GPT access
1763
1764	//////////// public vseg => access reference GPT
1765	else
1766	{
1767	// get reference process cluster and local pointer
1768	ref_cxy = GET_CXY( process->ref_xp );
1769	ref_ptr = GET_PTR( process->ref_xp );
1770
1771	// build extended pointer on reference GPT and reference GPT lock
1772	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1773	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1774
1775	// build extended pointer on local GPT and local GPT lock
1776	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1777	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1778
1779	// take reference GPT lock in read mode
1780	remote_rwlock_rd_acquire( ref_lock_xp );
1781
1782	// get directly PPN & attributes from reference GPT
1783	// this can avoids a costly RPC for a false page fault
1784	hal_gpt_get_pte( ref_gpt_xp,
1785	vpn,
1786	&ref_attr,
1787	&ref_ppn );
1788
1789	// release reference GPT lock in read mode
1790	remote_rwlock_rd_release( ref_lock_xp );
1791
1792	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
1793	{
1794	// take local GPT lock in write mode
1795	remote_rwlock_wr_acquire( local_lock_xp );
1796
1797	// check VPN still unmapped in local GPT
1798	hal_gpt_get_pte( local_gpt_xp,
1799	vpn,
1800	&new_attr,
1801	&new_ppn );
1802
1803	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1804	{
1805	// update local GPT from reference GPT
1806	error = hal_gpt_set_pte( local_gpt_xp,
1807	vpn,
1808	ref_attr,
1809	ref_ppn );
1810	if( error )
1811	{
1812	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn %x\n",
1813	__FUNCTION__ , process->pid , vpn );
1814
1815	// release local GPT lock in write mode
1816	remote_rwlock_wr_release( local_lock_xp );
1817
1818	return EXCP_KERNEL_PANIC;
1819	}
1820	}
1821	else // VPN has been mapped by a a concurrent page_fault
1822	{
1823	// keep PTE from local GPT
1824	ref_attr = new_attr;
1825	ref_ppn = new_ppn;
1826	}
1827
1828	// release local GPT lock in write mode
1829	remote_rwlock_wr_release( local_lock_xp );
1830
1831	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1832	cycle = (uint32_t)hal_get_cycles();
1833	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1834	printk("\n[%s] false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1835	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
1836	#endif
1837	return EXCP_NON_FATAL;
1838	}
1839	else // true page fault => update reference GPT
1840	{
1841	// take reference GPT lock in write mode
1842	remote_rwlock_wr_acquire( ref_lock_xp );
1843
1844	// check VPN still unmapped in reference GPT
1845	// do nothing if VPN has been mapped by a a concurrent page_fault
1846	hal_gpt_get_pte( ref_gpt_xp,
1847	vpn,
1848	&ref_attr,
1849	&ref_ppn );
1850
1851	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
1852	{
1853	// allocate and initialise a physical page depending on the vseg type
1854	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1855
1856	if( error )
1857	{
1858	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1859	__FUNCTION__ , process->pid , vpn );
1860
1861	// release reference GPT lock in write mode
1862	remote_rwlock_wr_release( ref_lock_xp );
1863
1864	return EXCP_KERNEL_PANIC;
1865	}
1866
1867	// define new_attr from vseg flags
1868	new_attr = GPT_MAPPED \| GPT_SMALL;
1869	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1870	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1871	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1872	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1873
1874	// update reference GPT
1875	error = hal_gpt_set_pte( ref_gpt_xp,
1876	vpn,
1877	new_attr,
1878	new_ppn );
1879
1880	// update local GPT (protected by reference GPT lock)
1881	error \|= hal_gpt_set_pte( local_gpt_xp,
1882	vpn,
1883	new_attr,
1884	new_ppn );
1885
1886	if( error )
1887	{
1888	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
1889	__FUNCTION__ , process->pid , vpn );
1890
1891	// release reference GPT lock in write mode
1892	remote_rwlock_wr_release( ref_lock_xp );
1893
1894	return EXCP_KERNEL_PANIC;
1895	}
1896	}
1897
1898	// release reference GPT lock in write mode
1899	remote_rwlock_wr_release( ref_lock_xp );
1900
1901	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1902	cycle = (uint32_t)hal_get_cycles();
1903	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1904	printk("\n[%s] true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1905	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1906	#endif
1907	return EXCP_NON_FATAL;
1908	}
1909	}
1910	} // end vmm_handle_page_fault()
1911
1912	////////////////////////////////////////////
1913	error_t vmm_handle_cow( process_t * process,
1914	vpn_t vpn )
1915	{
1916	vseg_t * vseg; // vseg containing vpn
1917	cxy_t ref_cxy; // reference cluster for missing vpn
1918	process_t * ref_ptr; // reference process for missing vpn
1919	xptr_t gpt_xp; // extended pointer on GPT
1920	xptr_t gpt_lock_xp; // extended pointer on GPT lock
1921	uint32_t old_attr; // current PTE_ATTR value
1922	ppn_t old_ppn; // current PTE_PPN value
1923	uint32_t new_attr; // new PTE_ATTR value
1924	ppn_t new_ppn; // new PTE_PPN value
1925	error_t error;
1926
1927	#if DEBUG_VMM_HANDLE_COW
1928	uint32_t cycle = (uint32_t)hal_get_cycles();
1929	thread_t * this = CURRENT_THREAD;
1930	if( DEBUG_VMM_HANDLE_COW < cycle )
1931	printk("\n[%s] thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
1932	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
1933	#endif
1934
1935	// get local vseg
1936	error = vmm_get_vseg( process,
1937	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1938	&vseg );
1939
1940	if( error )
1941	{
1942	printk("\n[PANIC] in %s : vpn %x in process %x not in a registered vseg\n",
1943	__FUNCTION__, vpn, process->pid );
1944
1945	return EXCP_KERNEL_PANIC;
1946	}
1947
1948	// get reference GPT cluster and local pointer
1949	ref_cxy = GET_CXY( process->ref_xp );
1950	ref_ptr = GET_PTR( process->ref_xp );
1951
1952	// build relevant extended pointers on GPT and GPT lock
1953	// - access local GPT for a private vseg
1954	// - access reference GPT for a public vseg
1955	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1956	{
1957	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1958	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1959	}
1960	else
1961	{
1962	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1963	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1964	}
1965
1966	// take GPT lock in write mode
1967	remote_rwlock_wr_acquire( gpt_lock_xp );
1968
1969	// get current PTE from reference GPT
1970	hal_gpt_get_pte( gpt_xp,
1971	vpn,
1972	&old_attr,
1973	&old_ppn );
1974
1975	// the PTE must be mapped for a COW
1976	if( (old_attr & GPT_MAPPED) == 0 )
1977	{
1978	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
1979	__FUNCTION__, vpn, process->pid );
1980
1981	// release GPT lock in write mode
1982	remote_rwlock_wr_acquire( gpt_lock_xp );
1983
1984	return EXCP_KERNEL_PANIC;
1985	}
1986
1987	// get extended pointer, cluster and local pointer on physical page descriptor
1988	xptr_t page_xp = ppm_ppn2page( old_ppn );
1989	cxy_t page_cxy = GET_CXY( page_xp );
1990	page_t * page_ptr = GET_PTR( page_xp );
1991
1992	// get extended pointers on forks and lock field in page descriptor
1993	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
1994	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
1995
1996	// take lock protecting "forks" counter
1997	remote_busylock_acquire( forks_lock_xp );
1998
1999	// get number of pending forks from page descriptor
2000	uint32_t forks = hal_remote_l32( forks_xp );
2001
2002	if( forks ) // pending fork => allocate a new page, and copy old to new
2003	{
2004	// allocate a new physical page
2005	page_xp = vmm_page_allocate( vseg , vpn );
2006	if( page_xp == XPTR_NULL )
2007	{
2008	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2009	__FUNCTION__ , vpn, process->pid );
2010
2011	// release GPT lock in write mode
2012	remote_rwlock_wr_acquire( gpt_lock_xp );
2013
2014	// release lock protecting "forks" counter
2015	remote_busylock_release( forks_lock_xp );
2016
2017	return EXCP_KERNEL_PANIC;
2018	}
2019
2020	// compute allocated page PPN
2021	new_ppn = ppm_page2ppn( page_xp );
2022
2023	// copy old page content to new page
2024	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
2025	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
2026	memcpy( GET_PTR( new_base_xp ),
2027	GET_PTR( old_base_xp ),
2028	CONFIG_PPM_PAGE_SIZE );
2029
2030	// decrement pending forks counter in page descriptor
2031	hal_remote_atomic_add( forks_xp , -1 );
2032
2033	#if(DEBUG_VMM_HANDLE_COW & 1)
2034	if( DEBUG_VMM_HANDLE_COW < cycle )
2035	printk("\n[%s] thread[%x,%x] : pending forks => allocate a new PPN %x\n",
2036	__FUNCTION__, process->pid, this->trdid, new_ppn );
2037	#endif
2038
2039	}
2040	else // no pending fork => keep the existing page
2041	{
2042
2043	#if(DEBUG_VMM_HANDLE_COW & 1)
2044	if( DEBUG_VMM_HANDLE_COW < cycle )
2045	printk("\n[%s] thread[%x,%x] no pending forks => keep existing PPN %x\n",
2046	__FUNCTION__, process->pid, this->trdid, new_ppn );
2047	#endif
2048	new_ppn = old_ppn;
2049	}
2050
2051	// release lock protecting "forks" counter
2052	remote_busylock_release( forks_lock_xp );
2053
2054	// build new_attr : reset COW and set WRITABLE,
2055	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
2056
2057	// update the relevan GPT
2058	// - private vseg => update local GPT
2059	// - public vseg => update all GPT copies
2060	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2061	{
2062	hal_gpt_set_pte( gpt_xp,
2063	vpn,
2064	new_attr,
2065	new_ppn );
2066	}
2067	else
2068	{
2069	if( ref_cxy == local_cxy ) // reference cluster is local
2070	{
2071	vmm_global_update_pte( process,
2072	vpn,
2073	new_attr,
2074	new_ppn );
2075	}
2076	else // reference cluster is remote
2077	{
2078	rpc_vmm_global_update_pte_client( ref_cxy,
2079	ref_ptr,
2080	vpn,
2081	new_attr,
2082	new_ppn );
2083	}
2084	}
2085
2086	// release GPT lock in write mode
2087	remote_rwlock_wr_release( gpt_lock_xp );
2088
2089	#if DEBUG_VMM_HANDLE_COW
2090	cycle = (uint32_t)hal_get_cycles();
2091	if( DEBUG_VMM_HANDLE_COW < cycle )
2092	printk("\n[%s] thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2093	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2094	#endif
2095
2096	return EXCP_NON_FATAL;
2097
2098	} // end vmm_handle_cow()
2099

Note: See TracBrowser for help on using the repository browser.

Download in other formats: