Context Navigation

source: trunk/kernel/mm/vmm.c @ 593

Last change on this file since 593 was 585, checked in by alain, 6 years ago
Introduce two separate vmm_handle_page_fault() & vmm_handle_cow() functions in vmm.c
File size: 72.8 KB

Line
1	/*
2	* vmm.c - virtual memory manager related operations interface.
3	*
4	* Authors Ghassan Almaless (2008,2009,2010,2011, 2012)
5	* Mohamed Lamine Karaoui (2015)
6	* Alain Greiner (2016)
7	*
8	* Copyright (c) UPMC Sorbonne Universites
9	*
10	* This file is part of ALMOS-MKH.
11	*
12	* ALMOS-MKH is free software; you can redistribute it and/or modify it
13	* under the terms of the GNU General Public License as published by
14	* the Free Software Foundation; version 2.0 of the License.
15	*
16	* ALMOS-MKH is distributed in the hope that it will be useful, but
17	* WITHOUT ANY WARRANTY; without even the implied warranty of
18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19	* General Public License for more details.
20	*
21	* You should have received a copy of the GNU General Public License
22	* along with ALMOS-MKH; if not, write to the Free Software Foundation,
23	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24	*/
25
26	#include <kernel_config.h>
27	#include <hal_kernel_types.h>
28	#include <hal_special.h>
29	#include <hal_gpt.h>
30	#include <hal_vmm.h>
31	#include <hal_macros.h>
32	#include <printk.h>
33	#include <memcpy.h>
34	#include <remote_rwlock.h>
35	#include <remote_queuelock.h>
36	#include <list.h>
37	#include <xlist.h>
38	#include <bits.h>
39	#include <process.h>
40	#include <thread.h>
41	#include <vseg.h>
42	#include <cluster.h>
43	#include <scheduler.h>
44	#include <vfs.h>
45	#include <mapper.h>
46	#include <page.h>
47	#include <kmem.h>
48	#include <vmm.h>
49	#include <hal_exception.h>
50
51	//////////////////////////////////////////////////////////////////////////////////
52	// Extern global variables
53	//////////////////////////////////////////////////////////////////////////////////
54
55	extern process_t process_zero; // allocated in cluster.c
56
57	///////////////////////////////////////
58	error_t vmm_init( process_t * process )
59	{
60	error_t error;
61	vseg_t * vseg_kentry;
62	vseg_t * vseg_args;
63	vseg_t * vseg_envs;
64	intptr_t base;
65	intptr_t size;
66
67	#if DEBUG_VMM_INIT
68	thread_t * this = CURRENT_THREAD;
69	uint32_t cycle = (uint32_t)hal_get_cycles();
70	if( DEBUG_VMM_INIT )
71	printk("\n[DBG] %s : thread %x in process %x enter for process %x / cycle %d\n",
72	__FUNCTION__ , this->trdid, this->process->pid, process->pid , cycle );
73	#endif
74
75	// get pointer on VMM
76	vmm_t * vmm = &process->vmm;
77
78	// initialize local list of vsegs
79	vmm->vsegs_nr = 0;
80	xlist_root_init( XPTR( local_cxy , &vmm->vsegs_root ) );
81	remote_rwlock_init( XPTR( local_cxy , &vmm->vsegs_lock ) , LOCK_VMM_VSL );
82
83	assert( ((CONFIG_VMM_KENTRY_SIZE + CONFIG_VMM_ARGS_SIZE + CONFIG_VMM_ENVS_SIZE)
84	<= CONFIG_VMM_ELF_BASE) , "UTILS zone too small\n" );
85
86	assert( (CONFIG_THREADS_MAX_PER_CLUSTER <= 32) ,
87	"no more than 32 threads per cluster for a single process\n");
88
89	assert( ((CONFIG_VMM_STACK_SIZE * CONFIG_THREADS_MAX_PER_CLUSTER) <=
90	(CONFIG_VMM_VSPACE_SIZE - CONFIG_VMM_STACK_BASE)) ,
91	"STACK zone too small\n");
92
93	// register kentry vseg in VSL
94	base = CONFIG_VMM_KENTRY_BASE << CONFIG_PPM_PAGE_SHIFT;
95	size = CONFIG_VMM_KENTRY_SIZE << CONFIG_PPM_PAGE_SHIFT;
96
97	vseg_kentry = vmm_create_vseg( process,
98	VSEG_TYPE_CODE,
99	base,
100	size,
101	0, // file_offset unused
102	0, // file_size unused
103	XPTR_NULL, // mapper_xp unused
104	local_cxy );
105
106	if( vseg_kentry == NULL )
107	{
108	printk("\n[ERROR] in %s : cannot register kentry vseg\n", __FUNCTION__ );
109	return -1;
110	}
111
112	vmm->kent_vpn_base = base;
113
114	// register args vseg in VSL
115	base = (CONFIG_VMM_KENTRY_BASE +
116	CONFIG_VMM_KENTRY_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
117	size = CONFIG_VMM_ARGS_SIZE << CONFIG_PPM_PAGE_SHIFT;
118
119	vseg_args = vmm_create_vseg( process,
120	VSEG_TYPE_DATA,
121	base,
122	size,
123	0, // file_offset unused
124	0, // file_size unused
125	XPTR_NULL, // mapper_xp unused
126	local_cxy );
127
128	if( vseg_args == NULL )
129	{
130	printk("\n[ERROR] in %s : cannot register args vseg\n", __FUNCTION__ );
131	return -1;
132	}
133
134	vmm->args_vpn_base = base;
135
136	// register the envs vseg in VSL
137	base = (CONFIG_VMM_KENTRY_BASE +
138	CONFIG_VMM_KENTRY_SIZE +
139	CONFIG_VMM_ARGS_SIZE ) << CONFIG_PPM_PAGE_SHIFT;
140	size = CONFIG_VMM_ENVS_SIZE << CONFIG_PPM_PAGE_SHIFT;
141
142	vseg_envs = vmm_create_vseg( process,
143	VSEG_TYPE_DATA,
144	base,
145	size,
146	0, // file_offset unused
147	0, // file_size unused
148	XPTR_NULL, // mapper_xp unused
149	local_cxy );
150
151	if( vseg_envs == NULL )
152	{
153	printk("\n[ERROR] in %s : cannot register envs vseg\n", __FUNCTION__ );
154	return -1;
155	}
156
157	vmm->envs_vpn_base = base;
158
159	// create GPT (empty)
160	error = hal_gpt_create( &vmm->gpt );
161
162	if( error )
163	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
164
165	// initialize GPT lock
166	remote_rwlock_init( XPTR( local_cxy , &vmm->gpt_lock ) , LOCK_VMM_GPT );
167
168	// architecture specic GPT initialisation
169	// (For TSAR, identity map the kentry_vseg)
170	error = hal_vmm_init( vmm );
171
172	if( error )
173	printk("\n[ERROR] in %s : cannot initialize GPT\n", __FUNCTION__ );
174
175	// initialize STACK allocator
176	vmm->stack_mgr.bitmap = 0;
177	vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
178	busylock_init( &vmm->stack_mgr.lock , LOCK_VMM_STACK );
179
180	// initialize MMAP allocator
181	vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
182	vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
183	vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
184	busylock_init( &vmm->mmap_mgr.lock , LOCK_VMM_MMAP );
185
186	uint32_t i;
187	for( i = 0 ; i < 32 ; i++ ) list_root_init( &vmm->mmap_mgr.zombi_list[i] );
188
189	// initialize instrumentation counters
190	vmm->pgfault_nr = 0;
191
192	hal_fence();
193
194	#if DEBUG_VMM_INIT
195	cycle = (uint32_t)hal_get_cycles();
196	if( DEBUG_VMM_INIT )
197	printk("\n[DBG] %s : thread %x in process %x exit / process %x / entry_point = %x / cycle %d\n",
198	__FUNCTION__, this->trdid, this->process->pid, process->pid , process->vmm.entry_point , cycle );
199	#endif
200
201	return 0;
202
203	} // end vmm_init()
204
205	//////////////////////////////////////
206	void vmm_display( process_t * process,
207	bool_t mapping )
208	{
209	vmm_t * vmm = &process->vmm;
210	gpt_t * gpt = &vmm->gpt;
211
212	printk("\n***** VSL and GPT(%x) for process %x in cluster %x\n\n",
213	process->vmm.gpt.ptr , process->pid , local_cxy );
214
215	// get lock protecting the VSL and the GPT
216	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->vsegs_lock ) );
217	remote_rwlock_rd_acquire( XPTR( local_cxy , &vmm->gpt_lock ) );
218
219	// scan the list of vsegs
220	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
221	xptr_t iter_xp;
222	xptr_t vseg_xp;
223	vseg_t * vseg;
224	XLIST_FOREACH( root_xp , iter_xp )
225	{
226	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
227	vseg = GET_PTR( vseg_xp );
228
229	printk(" - %s : base = %X / size = %X / npages = %d\n",
230	vseg_type_str( vseg->type ) , vseg->min , vseg->max - vseg->min , vseg->vpn_size );
231
232	if( mapping )
233	{
234	vpn_t vpn;
235	ppn_t ppn;
236	uint32_t attr;
237	vpn_t base = vseg->vpn_base;
238	vpn_t size = vseg->vpn_size;
239	for( vpn = base ; vpn < (base+size) ; vpn++ )
240	{
241	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
242	if( attr & GPT_MAPPED )
243	{
244	printk(" . vpn = %X / attr = %X / ppn = %X\n", vpn , attr , ppn );
245	}
246	}
247	}
248	}
249
250	// release the locks
251	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->vsegs_lock ) );
252	remote_rwlock_rd_release( XPTR( local_cxy , &vmm->gpt_lock ) );
253
254	} // vmm_display()
255
256	///////////////////////////////////
257	void vmm_vseg_attach( vmm_t * vmm,
258	vseg_t * vseg )
259	{
260	// build extended pointer on rwlock protecting VSL
261	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
262
263	// get rwlock in write mode
264	remote_rwlock_wr_acquire( lock_xp );
265
266	// update vseg descriptor
267	vseg->vmm = vmm;
268
269	// add vseg in vmm list
270	xlist_add_last( XPTR( local_cxy , &vmm->vsegs_root ),
271	XPTR( local_cxy , &vseg->xlist ) );
272
273	// release rwlock in write mode
274	remote_rwlock_wr_release( lock_xp );
275	}
276
277	///////////////////////////////////
278	void vmm_vseg_detach( vmm_t * vmm,
279	vseg_t * vseg )
280	{
281	// build extended pointer on rwlock protecting VSL
282	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
283
284	// get rwlock in write mode
285	remote_rwlock_wr_acquire( lock_xp );
286
287	// update vseg descriptor
288	vseg->vmm = NULL;
289
290	// remove vseg from vmm list
291	xlist_unlink( XPTR( local_cxy , &vseg->xlist ) );
292
293	// release rwlock in write mode
294	remote_rwlock_wr_release( lock_xp );
295	}
296
297	/////////////////////i//////////////////////////
298	void vmm_global_update_pte( process_t * process,
299	vpn_t vpn,
300	uint32_t attr,
301	ppn_t ppn )
302	{
303	xlist_entry_t * process_root_ptr;
304	xptr_t process_root_xp;
305	xptr_t process_iter_xp;
306
307	xptr_t remote_process_xp;
308	cxy_t remote_process_cxy;
309	process_t * remote_process_ptr;
310	xptr_t remote_gpt_xp;
311
312	pid_t pid;
313	cxy_t owner_cxy;
314	lpid_t owner_lpid;
315
316	#if DEBUG_VMM_UPDATE_PTE
317	uint32_t cycle = (uint32_t)hal_get_cycles();
318	if( DEBUG_VMM_UPDATE_PTE < cycle )
319	printk("\n[DBG] %s : thread %x enter for process %x / vpn %x / cycle %d\n",
320	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
321	#endif
322
323	// check cluster is reference
324	assert( (GET_CXY( process->ref_xp ) == local_cxy) , "not called in reference cluster\n");
325
326	// get extended pointer on root of process copies xlist in owner cluster
327	pid = process->pid;
328	owner_cxy = CXY_FROM_PID( pid );
329	owner_lpid = LPID_FROM_PID( pid );
330	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
331	process_root_xp = XPTR( owner_cxy , process_root_ptr );
332
333	// loop on destination process copies
334	XLIST_FOREACH( process_root_xp , process_iter_xp )
335	{
336	// get cluster and local pointer on remote process
337	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
338	remote_process_ptr = GET_PTR( remote_process_xp );
339	remote_process_cxy = GET_CXY( remote_process_xp );
340
341	#if (DEBUG_VMM_UPDATE_PTE & 0x1)
342	if( DEBUG_VMM_UPDATE_PTE < cycle )
343	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
344	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
345	#endif
346
347	// get extended pointer on remote gpt
348	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
349
350	// update remote GPT
351	hal_gpt_update_pte( remote_gpt_xp, vpn, attr, ppn );
352	}
353
354	#if DEBUG_VMM_UPDATE_PTE
355	cycle = (uint32_t)hal_get_cycles();
356	if( DEBUG_VMM_UPDATE_PTE < cycle )
357	printk("\n[DBG] %s : thread %x exit for process %x / vpn %x / cycle %d\n",
358	__FUNCTION__ , CURRENT_THREAD , process->pid , vpn , cycle );
359	#endif
360
361	} // end vmm_global_update_pte()
362
363	///////////////////////////////////////
364	void vmm_set_cow( process_t * process )
365	{
366	vmm_t * vmm;
367
368	xlist_entry_t * process_root_ptr;
369	xptr_t process_root_xp;
370	xptr_t process_iter_xp;
371
372	xptr_t remote_process_xp;
373	cxy_t remote_process_cxy;
374	process_t * remote_process_ptr;
375	xptr_t remote_gpt_xp;
376
377	xptr_t vseg_root_xp;
378	xptr_t vseg_iter_xp;
379
380	xptr_t vseg_xp;
381	vseg_t * vseg;
382
383	pid_t pid;
384	cxy_t owner_cxy;
385	lpid_t owner_lpid;
386
387	#if DEBUG_VMM_SET_COW
388	uint32_t cycle = (uint32_t)hal_get_cycles();
389	if( DEBUG_VMM_SET_COW < cycle )
390	printk("\n[DBG] %s : thread %x enter for process %x / cycle %d\n",
391	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
392	#endif
393
394	// check cluster is reference
395	assert( (GET_CXY( process->ref_xp ) == local_cxy) ,
396	"local cluster is not process reference cluster\n");
397
398	// get pointer on reference VMM
399	vmm = &process->vmm;
400
401	// get extended pointer on root of process copies xlist in owner cluster
402	pid = process->pid;
403	owner_cxy = CXY_FROM_PID( pid );
404	owner_lpid = LPID_FROM_PID( pid );
405	process_root_ptr = &LOCAL_CLUSTER->pmgr.copies_root[owner_lpid];
406	process_root_xp = XPTR( owner_cxy , process_root_ptr );
407
408	// get extended pointer on root of vsegs xlist from reference VMM
409	vseg_root_xp = XPTR( local_cxy , &vmm->vsegs_root );
410
411	// loop on destination process copies
412	XLIST_FOREACH( process_root_xp , process_iter_xp )
413	{
414	// get cluster and local pointer on remote process
415	remote_process_xp = XLIST_ELEMENT( process_iter_xp , process_t , copies_list );
416	remote_process_ptr = GET_PTR( remote_process_xp );
417	remote_process_cxy = GET_CXY( remote_process_xp );
418
419	#if (DEBUG_VMM_SET_COW &0x1)
420	if( DEBUG_VMM_SET_COW < cycle )
421	printk("\n[DBG] %s : thread %x handling process %x in cluster %x\n",
422	__FUNCTION__ , CURRENT_THREAD , process->pid , remote_process_cxy );
423	#endif
424
425	// get extended pointer on remote gpt
426	remote_gpt_xp = XPTR( remote_process_cxy , &remote_process_ptr->vmm.gpt );
427
428	// loop on vsegs in (local) reference process VSL
429	XLIST_FOREACH( vseg_root_xp , vseg_iter_xp )
430	{
431	// get pointer on vseg
432	vseg_xp = XLIST_ELEMENT( vseg_iter_xp , vseg_t , xlist );
433	vseg = GET_PTR( vseg_xp );
434
435	assert( (GET_CXY( vseg_xp ) == local_cxy) ,
436	"all vsegs in reference VSL must be local\n" );
437
438	// get vseg type, base and size
439	uint32_t type = vseg->type;
440	vpn_t vpn_base = vseg->vpn_base;
441	vpn_t vpn_size = vseg->vpn_size;
442
443	#if (DEBUG_VMM_SET_COW & 0x1)
444	if( DEBUG_VMM_SET_COW < cycle )
445	printk("\n[DBG] %s : thread %x handling vseg %s / vpn_base = %x / vpn_size = %x\n",
446	__FUNCTION__, CURRENT_THREAD , vseg_type_str(type), vpn_base, vpn_size );
447	#endif
448	// only DATA, ANON and REMOTE vsegs
449	if( (type == VSEG_TYPE_DATA) \|\|
450	(type == VSEG_TYPE_ANON) \|\|
451	(type == VSEG_TYPE_REMOTE) )
452	{
453	vpn_t vpn;
454	uint32_t attr;
455	ppn_t ppn;
456	xptr_t page_xp;
457	cxy_t page_cxy;
458	page_t * page_ptr;
459	xptr_t forks_xp;
460	xptr_t lock_xp;
461
462	// update flags in remote GPT
463	hal_gpt_set_cow( remote_gpt_xp,
464	vpn_base,
465	vpn_size );
466
467	// atomically increment pending forks counter in physical pages,
468	// for all vseg pages that are mapped in reference cluster
469	if( remote_process_cxy == local_cxy )
470	{
471	// scan all pages in vseg
472	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
473	{
474	// get page attributes and PPN from reference GPT
475	hal_gpt_get_pte( remote_gpt_xp , vpn , &attr , &ppn );
476
477	// atomically update pending forks counter if page is mapped
478	if( attr & GPT_MAPPED )
479	{
480	// get pointers and cluster on page descriptor
481	page_xp = ppm_ppn2page( ppn );
482	page_cxy = GET_CXY( page_xp );
483	page_ptr = GET_PTR( page_xp );
484
485	// get extended pointers on "forks" and "lock"
486	forks_xp = XPTR( page_cxy , &page_ptr->forks );
487	lock_xp = XPTR( page_cxy , &page_ptr->lock );
488
489	// take lock protecting "forks" counter
490	remote_busylock_acquire( lock_xp );
491
492	// increment "forks"
493	hal_remote_atomic_add( forks_xp , 1 );
494
495	// release lock protecting "forks" counter
496	remote_busylock_release( lock_xp );
497	}
498	} // end loop on vpn
499	} // end if local
500	} // end if vseg type
501	} // end loop on vsegs
502	} // end loop on process copies
503
504	#if DEBUG_VMM_SET_COW
505	cycle = (uint32_t)hal_get_cycles();
506	if( DEBUG_VMM_SET_COW < cycle )
507	printk("\n[DBG] %s : thread %x exit for process %x / cycle %d\n",
508	__FUNCTION__ , CURRENT_THREAD , process->pid , cycle );
509	#endif
510
511	} // end vmm_set-cow()
512
513	/////////////////////////////////////////////////
514	error_t vmm_fork_copy( process_t * child_process,
515	xptr_t parent_process_xp )
516	{
517	error_t error;
518	cxy_t parent_cxy;
519	process_t * parent_process;
520	vmm_t * parent_vmm;
521	xptr_t parent_lock_xp;
522	vmm_t * child_vmm;
523	xptr_t iter_xp;
524	xptr_t parent_vseg_xp;
525	vseg_t * parent_vseg;
526	vseg_t * child_vseg;
527	uint32_t type;
528	bool_t cow;
529	vpn_t vpn;
530	vpn_t vpn_base;
531	vpn_t vpn_size;
532	xptr_t page_xp; // extended pointer on page descriptor
533	page_t * page_ptr;
534	cxy_t page_cxy;
535	xptr_t forks_xp; // extended pointer on forks counter in page descriptor
536	xptr_t lock_xp; // extended pointer on lock protecting the forks counter
537	xptr_t parent_root_xp;
538	bool_t mapped;
539	ppn_t ppn;
540
541	#if DEBUG_VMM_FORK_COPY
542	uint32_t cycle = (uint32_t)hal_get_cycles();
543	if( DEBUG_VMM_FORK_COPY < cycle )
544	printk("\n[DBG] %s : thread %x enter / cycle %d\n",
545	__FUNCTION__ , CURRENT_THREAD, cycle );
546	#endif
547
548	// get parent process cluster and local pointer
549	parent_cxy = GET_CXY( parent_process_xp );
550	parent_process = GET_PTR( parent_process_xp );
551
552	// get local pointers on parent and child VMM
553	parent_vmm = &parent_process->vmm;
554	child_vmm = &child_process->vmm;
555
556	// get extended pointer on lock protecting the parent VSL
557	parent_lock_xp = XPTR( parent_cxy , &parent_vmm->vsegs_lock );
558
559	// initialize the lock protecting the child VSL
560	remote_rwlock_init( XPTR( local_cxy , &child_vmm->vsegs_lock ), LOCK_VMM_STACK );
561
562	// initialize the child VSL as empty
563	xlist_root_init( XPTR( local_cxy, &child_vmm->vsegs_root ) );
564	child_vmm->vsegs_nr = 0;
565
566	// create child GPT
567	error = hal_gpt_create( &child_vmm->gpt );
568
569	if( error )
570	{
571	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
572	return -1;
573	}
574
575	// build extended pointer on parent VSL
576	parent_root_xp = XPTR( parent_cxy , &parent_vmm->vsegs_root );
577
578	// take the lock protecting the parent VSL in read mode
579	remote_rwlock_rd_acquire( parent_lock_xp );
580
581	// loop on parent VSL xlist
582	XLIST_FOREACH( parent_root_xp , iter_xp )
583	{
584	// get local and extended pointers on current parent vseg
585	parent_vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
586	parent_vseg = GET_PTR( parent_vseg_xp );
587
588	// get vseg type
589	type = hal_remote_l32( XPTR( parent_cxy , &parent_vseg->type ) );
590
591	#if DEBUG_VMM_FORK_COPY
592	cycle = (uint32_t)hal_get_cycles();
593	if( DEBUG_VMM_FORK_COPY < cycle )
594	printk("\n[DBG] %s : thread %x found parent vseg %s / vpn_base = %x / cycle %d\n",
595	__FUNCTION__ , CURRENT_THREAD, vseg_type_str(type),
596	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
597	#endif
598
599	// all parent vsegs - but STACK - must be copied in child VSL
600	if( type != VSEG_TYPE_STACK )
601	{
602	// allocate memory for a new child vseg
603	child_vseg = vseg_alloc();
604	if( child_vseg == NULL ) // release all allocated vsegs
605	{
606	vmm_destroy( child_process );
607	printk("\n[ERROR] in %s : cannot create vseg for child\n", __FUNCTION__ );
608	return -1;
609	}
610
611	// copy parent vseg to child vseg
612	vseg_init_from_ref( child_vseg , parent_vseg_xp );
613
614	// register child vseg in child VSL
615	vmm_vseg_attach( child_vmm , child_vseg );
616
617	#if DEBUG_VMM_FORK_COPY
618	cycle = (uint32_t)hal_get_cycles();
619	if( DEBUG_VMM_FORK_COPY < cycle )
620	printk("\n[DBG] %s : thread %x copied vseg %s / vpn_base = %x to child VSL / cycle %d\n",
621	__FUNCTION__ , CURRENT_THREAD , vseg_type_str(type),
622	hal_remote_l32( XPTR( parent_cxy , &parent_vseg->vpn_base ) ) , cycle );
623	#endif
624
625	// copy DATA, MMAP, REMOTE, FILE parent GPT entries to child GPT
626	if( type != VSEG_TYPE_CODE )
627	{
628	// activate the COW for DATA, MMAP, REMOTE vsegs only
629	cow = ( type != VSEG_TYPE_FILE );
630
631	vpn_base = child_vseg->vpn_base;
632	vpn_size = child_vseg->vpn_size;
633
634	// scan pages in parent vseg
635	for( vpn = vpn_base ; vpn < (vpn_base + vpn_size) ; vpn++ )
636	{
637	error = hal_gpt_pte_copy( &child_vmm->gpt,
638	XPTR( parent_cxy , &parent_vmm->gpt ),
639	vpn,
640	cow,
641	&ppn,
642	&mapped );
643	if( error )
644	{
645	vmm_destroy( child_process );
646	printk("\n[ERROR] in %s : cannot copy GPT\n", __FUNCTION__ );
647	return -1;
648	}
649
650	// increment pending forks counter in page if mapped
651	if( mapped )
652	{
653	// get pointers and cluster on page descriptor
654	page_xp = ppm_ppn2page( ppn );
655	page_cxy = GET_CXY( page_xp );
656	page_ptr = GET_PTR( page_xp );
657
658	// get extended pointers on "forks" and "lock"
659	forks_xp = XPTR( page_cxy , &page_ptr->forks );
660	lock_xp = XPTR( page_cxy , &page_ptr->lock );
661
662	// get lock protecting "forks" counter
663	remote_busylock_acquire( lock_xp );
664
665	// increment "forks"
666	hal_remote_atomic_add( forks_xp , 1 );
667
668	// release lock protecting "forks" counter
669	remote_busylock_release( lock_xp );
670
671	#if DEBUG_VMM_FORK_COPY
672	cycle = (uint32_t)hal_get_cycles();
673	if( DEBUG_VMM_FORK_COPY < cycle )
674	printk("\n[DBG] %s : thread %x copied vpn %x to child GPT / cycle %d\n",
675	__FUNCTION__ , CURRENT_THREAD , vpn , cycle );
676	#endif
677	}
678	}
679	} // end if no code & no stack
680	} // end if no stack
681	} // end loop on vsegs
682
683	// release the parent VSL lock in read mode
684	remote_rwlock_rd_release( parent_lock_xp );
685
686	// initialize child GPT (architecture specic)
687	// => For TSAR, identity map the kentry_vseg
688	error = hal_vmm_init( child_vmm );
689
690	if( error )
691	{
692	printk("\n[ERROR] in %s : cannot create GPT\n", __FUNCTION__ );
693	return -1;
694	}
695
696	// initialize the child VMM STACK allocator
697	child_vmm->stack_mgr.bitmap = 0;
698	child_vmm->stack_mgr.vpn_base = CONFIG_VMM_STACK_BASE;
699
700	// initialize the child VMM MMAP allocator
701	uint32_t i;
702	child_vmm->mmap_mgr.vpn_base = CONFIG_VMM_HEAP_BASE;
703	child_vmm->mmap_mgr.vpn_size = CONFIG_VMM_STACK_BASE - CONFIG_VMM_HEAP_BASE;
704	child_vmm->mmap_mgr.first_free_vpn = CONFIG_VMM_HEAP_BASE;
705	for( i = 0 ; i < 32 ; i++ ) list_root_init( &child_vmm->mmap_mgr.zombi_list[i] );
706
707	// initialize instrumentation counters
708	child_vmm->pgfault_nr = 0;
709
710	// copy base addresses from parent VMM to child VMM
711	child_vmm->kent_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->kent_vpn_base));
712	child_vmm->args_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->args_vpn_base));
713	child_vmm->envs_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->envs_vpn_base));
714	child_vmm->heap_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->heap_vpn_base));
715	child_vmm->code_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->code_vpn_base));
716	child_vmm->data_vpn_base = (vpn_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->data_vpn_base));
717
718	child_vmm->entry_point = (intptr_t)hal_remote_lpt(XPTR(parent_cxy, &parent_vmm->entry_point));
719
720	hal_fence();
721
722	#if DEBUG_VMM_FORK_COPY
723	cycle = (uint32_t)hal_get_cycles();
724	if( DEBUG_VMM_FORK_COPY < cycle )
725	printk("\n[DBG] %s : thread %x exit successfully / cycle %d\n",
726	__FUNCTION__ , CURRENT_THREAD , cycle );
727	#endif
728
729	return 0;
730
731	} // vmm_fork_copy()
732
733	///////////////////////////////////////
734	void vmm_destroy( process_t * process )
735	{
736	xptr_t vseg_xp;
737	vseg_t * vseg;
738
739	#if DEBUG_VMM_DESTROY
740	uint32_t cycle = (uint32_t)hal_get_cycles();
741	if( DEBUG_VMM_DESTROY < cycle )
742	printk("\n[DBG] %s : thread %x enter for process %x in cluster %x / cycle %d\n",
743	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy, cycle );
744	#endif
745
746	#if (DEBUG_VMM_DESTROY & 1 )
747	if( DEBUG_VMM_DESTROY < cycle )
748	vmm_display( process , true );
749	#endif
750
751	// get pointer on local VMM
752	vmm_t * vmm = &process->vmm;
753
754	// get extended pointer on VSL root and VSL lock
755	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
756
757	// remove all user vsegs registered in VSL
758	while( !xlist_is_empty( root_xp ) )
759	{
760	// get pointer on first vseg in VSL
761	vseg_xp = XLIST_FIRST( root_xp , vseg_t , xlist );
762	vseg = GET_PTR( vseg_xp );
763
764	// unmap and release physical pages
765	vmm_unmap_vseg( process , vseg );
766
767	// remove vseg from VSL
768	vmm_vseg_detach( vmm , vseg );
769
770	// release memory allocated to vseg descriptor
771	vseg_free( vseg );
772
773	#if( DEBUG_VMM_DESTROY & 1 )
774	if( DEBUG_VMM_DESTROY < cycle )
775	printk("\n[DBG] %s : %s vseg released / vpn_base %x / vpn_size %d\n",
776	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
777	#endif
778
779	}
780
781	// remove all vsegs from zombi_lists in MMAP allocator
782	uint32_t i;
783	for( i = 0 ; i<32 ; i++ )
784	{
785	while( !list_is_empty( &vmm->mmap_mgr.zombi_list[i] ) )
786	{
787	vseg = LIST_FIRST( &vmm->mmap_mgr.zombi_list[i] , vseg_t , zlist );
788
789	#if( DEBUG_VMM_DESTROY & 1 )
790	if( DEBUG_VMM_DESTROY < cycle )
791	printk("\n[DBG] %s : found zombi vseg / vpn_base %x / vpn_size %d\n",
792	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
793	#endif
794	vmm_vseg_detach( vmm , vseg );
795	vseg_free( vseg );
796
797	#if( DEBUG_VMM_DESTROY & 1 )
798	if( DEBUG_VMM_DESTROY < cycle )
799	printk("\n[DBG] %s : zombi vseg released / vpn_base %x / vpn_size %d\n",
800	__FUNCTION__ , vseg_type_str( vseg->type ), vseg->vpn_base, vseg->vpn_size );
801	#endif
802	}
803	}
804
805	// release memory allocated to the GPT itself
806	hal_gpt_destroy( &vmm->gpt );
807
808	#if DEBUG_VMM_DESTROY
809	cycle = (uint32_t)hal_get_cycles();
810	if( DEBUG_VMM_DESTROY < cycle )
811	printk("\n[DBG] %s : thread %x exit for process %x in cluster %x / cycle %d\n",
812	__FUNCTION__, CURRENT_THREAD->trdid, process->pid, local_cxy , cycle );
813	#endif
814
815	} // end vmm_destroy()
816
817	/////////////////////////////////////////////////
818	vseg_t * vmm_check_conflict( process_t * process,
819	vpn_t vpn_base,
820	vpn_t vpn_size )
821	{
822	vmm_t * vmm = &process->vmm;
823
824	// scan the VSL
825	vseg_t * vseg;
826	xptr_t iter_xp;
827	xptr_t vseg_xp;
828	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
829
830	XLIST_FOREACH( root_xp , iter_xp )
831	{
832	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
833	vseg = GET_PTR( vseg_xp );
834
835	if( ((vpn_base + vpn_size) > vseg->vpn_base) &&
836	(vpn_base < (vseg->vpn_base + vseg->vpn_size)) ) return vseg;
837	}
838	return NULL;
839
840	} // end vmm_check_conflict()
841
842	////////////////////////////////////////////////////////////////////////////////////////////
843	// This static function is called by the vmm_create_vseg() function, and implements
844	// the VMM stack_vseg specific allocator.
845	////////////////////////////////////////////////////////////////////////////////////////////
846	// @ vmm : pointer on VMM.
847	// @ vpn_base : (return value) first allocated page
848	// @ vpn_size : (return value) number of allocated pages
849	////////////////////////////////////////////////////////////////////////////////////////////
850	static error_t vmm_stack_alloc( vmm_t * vmm,
851	vpn_t * vpn_base,
852	vpn_t * vpn_size )
853	{
854	// get stack allocator pointer
855	stack_mgr_t * mgr = &vmm->stack_mgr;
856
857	// get lock on stack allocator
858	busylock_acquire( &mgr->lock );
859
860	// get first free slot index in bitmap
861	int32_t index = bitmap_ffc( &mgr->bitmap , 4 );
862	if( (index < 0) \|\| (index > 31) )
863	{
864	busylock_release( &mgr->lock );
865	return 0xFFFFFFFF;
866	}
867
868	// update bitmap
869	bitmap_set( &mgr->bitmap , index );
870
871	// release lock on stack allocator
872	busylock_release( &mgr->lock );
873
874	// returns vpn_base, vpn_size (one page non allocated)
875	vpn_base = mgr->vpn_base + index CONFIG_VMM_STACK_SIZE + 1;
876	*vpn_size = CONFIG_VMM_STACK_SIZE - 1;
877	return 0;
878
879	} // end vmm_stack_alloc()
880
881	////////////////////////////////////////////////////////////////////////////////////////////
882	// This static function is called by the vmm_create_vseg() function, and implements
883	// the VMM MMAP specific allocator.
884	////////////////////////////////////////////////////////////////////////////////////////////
885	// @ vmm : [in] pointer on VMM.
886	// @ npages : [in] requested number of pages.
887	// @ vpn_base : [out] first allocated page.
888	// @ vpn_size : [out] actual number of allocated pages.
889	////////////////////////////////////////////////////////////////////////////////////////////
890	static error_t vmm_mmap_alloc( vmm_t * vmm,
891	vpn_t npages,
892	vpn_t * vpn_base,
893	vpn_t * vpn_size )
894	{
895	uint32_t index;
896	vseg_t * vseg;
897	vpn_t base;
898	vpn_t size;
899	vpn_t free;
900
901	// mmap vseg size must be power of 2
902	// compute actual size and index in zombi_list array
903	size = POW2_ROUNDUP( npages );
904	index = bits_log2( size );
905
906	// get mmap allocator pointer
907	mmap_mgr_t * mgr = &vmm->mmap_mgr;
908
909	// get lock on mmap allocator
910	busylock_acquire( &mgr->lock );
911
912	// get vseg from zombi_list or from mmap zone
913	if( list_is_empty( &mgr->zombi_list[index] ) ) // from mmap zone
914	{
915	// check overflow
916	free = mgr->first_free_vpn;
917	if( (free + size) > mgr->vpn_size ) return ENOMEM;
918
919	// update STACK allocator
920	mgr->first_free_vpn += size;
921
922	// compute base
923	base = free;
924	}
925	else // from zombi_list
926	{
927	// get pointer on zombi vseg from zombi_list
928	vseg = LIST_FIRST( &mgr->zombi_list[index] , vseg_t , zlist );
929
930	// remove vseg from free-list
931	list_unlink( &vseg->zlist );
932
933	// compute base
934	base = vseg->vpn_base;
935	}
936
937	// release lock on mmap allocator
938	busylock_release( &mgr->lock );
939
940	// returns vpn_base, vpn_size
941	*vpn_base = base;
942	*vpn_size = size;
943	return 0;
944
945	} // end vmm_mmap_alloc()
946
947	////////////////////////////////////////////////
948	vseg_t * vmm_create_vseg( process_t * process,
949	vseg_type_t type,
950	intptr_t base,
951	uint32_t size,
952	uint32_t file_offset,
953	uint32_t file_size,
954	xptr_t mapper_xp,
955	cxy_t cxy )
956	{
957	vseg_t * vseg; // created vseg pointer
958	vpn_t vpn_base; // first page index
959	vpn_t vpn_size; // number of pages
960	error_t error;
961
962	#if DEBUG_VMM_CREATE_VSEG
963	uint32_t cycle = (uint32_t)hal_get_cycles();
964	if( DEBUG_VMM_CREATE_VSEG < cycle )
965	printk("\n[DBG] %s : thread %x enter / process %x / base %x / size %x / %s / cxy %x / cycle %d\n",
966	__FUNCTION__, CURRENT_THREAD, process->pid, base, size, vseg_type_str(type), cxy, cycle );
967	#endif
968
969	// get pointer on VMM
970	vmm_t * vmm = &process->vmm;
971
972	// compute base, size, vpn_base, vpn_size, depending on vseg type
973	// we use the VMM specific allocators for "stack", "file", "anon", & "remote" vsegs
974	if( type == VSEG_TYPE_STACK )
975	{
976	// get vpn_base and vpn_size from STACK allocator
977	error = vmm_stack_alloc( vmm , &vpn_base , &vpn_size );
978	if( error )
979	{
980	printk("\n[ERROR] in %s : no space for stack vseg / process %x in cluster %x\n",
981	__FUNCTION__ , process->pid , local_cxy );
982	return NULL;
983	}
984
985	// compute vseg base and size from vpn_base and vpn_size
986	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
987	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
988	}
989	else if( (type == VSEG_TYPE_ANON) \|\|
990	(type == VSEG_TYPE_FILE) \|\|
991	(type == VSEG_TYPE_REMOTE) )
992	{
993	// get vpn_base and vpn_size from MMAP allocator
994	vpn_t npages = size >> CONFIG_PPM_PAGE_SHIFT;
995	error = vmm_mmap_alloc( vmm , npages , &vpn_base , &vpn_size );
996	if( error )
997	{
998	printk("\n[ERROR] in %s : no vspace for mmap vseg / process %x in cluster %x\n",
999	__FUNCTION__ , process->pid , local_cxy );
1000	return NULL;
1001	}
1002
1003	// compute vseg base and size from vpn_base and vpn_size
1004	base = vpn_base << CONFIG_PPM_PAGE_SHIFT;
1005	size = vpn_size << CONFIG_PPM_PAGE_SHIFT;
1006	}
1007	else
1008	{
1009	uint32_t vpn_min = base >> CONFIG_PPM_PAGE_SHIFT;
1010	uint32_t vpn_max = (base + size - 1) >> CONFIG_PPM_PAGE_SHIFT;
1011
1012	vpn_base = vpn_min;
1013	vpn_size = vpn_max - vpn_min + 1;
1014	}
1015
1016	// check collisions
1017	vseg = vmm_check_conflict( process , vpn_base , vpn_size );
1018	if( vseg != NULL )
1019	{
1020	printk("\n[ERROR] in %s for process %x : new vseg [vpn_base = %x / vpn_size = %x]\n"
1021	" overlap existing vseg [vpn_base = %x / vpn_size = %x]\n",
1022	__FUNCTION__ , process->pid, vpn_base, vpn_size, vseg->vpn_base, vseg->vpn_size );
1023	return NULL;
1024	}
1025
1026	// allocate physical memory for vseg descriptor
1027	vseg = vseg_alloc();
1028	if( vseg == NULL )
1029	{
1030	printk("\n[ERROR] in %s for process %x : cannot allocate memory for vseg\n",
1031	__FUNCTION__ , process->pid );
1032	return NULL;
1033	}
1034
1035	// initialize vseg descriptor
1036	vseg_init( vseg,
1037	type,
1038	base,
1039	size,
1040	vpn_base,
1041	vpn_size,
1042	file_offset,
1043	file_size,
1044	mapper_xp,
1045	cxy );
1046
1047	// attach vseg to VSL
1048	vmm_vseg_attach( vmm , vseg );
1049
1050	#if DEBUG_VMM_CREATE_VSEG
1051	cycle = (uint32_t)hal_get_cycles();
1052	if( DEBUG_VMM_CREATE_VSEG < cycle )
1053	printk("\n[DBG] %s : thread %x exit / process %x / %s / cxy %x / cycle %d\n",
1054	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str(type), cxy, cycle );
1055	#endif
1056
1057	return vseg;
1058
1059	} // vmm_create_vseg()
1060
1061	/////////////////////////////////////
1062	void vmm_remove_vseg( vseg_t * vseg )
1063	{
1064	// get pointers on calling process and VMM
1065	thread_t * this = CURRENT_THREAD;
1066	vmm_t * vmm = &this->process->vmm;
1067	uint32_t type = vseg->type;
1068
1069	// detach vseg from VSL
1070	vmm_vseg_detach( vmm , vseg );
1071
1072	// release the stack slot to VMM stack allocator if STACK type
1073	if( type == VSEG_TYPE_STACK )
1074	{
1075	// get pointer on stack allocator
1076	stack_mgr_t * mgr = &vmm->stack_mgr;
1077
1078	// compute slot index
1079	uint32_t index = ((vseg->vpn_base - mgr->vpn_base - 1) / CONFIG_VMM_STACK_SIZE);
1080
1081	// update stacks_bitmap
1082	busylock_acquire( &mgr->lock );
1083	bitmap_clear( &mgr->bitmap , index );
1084	busylock_release( &mgr->lock );
1085	}
1086
1087	// release the vseg to VMM mmap allocator if MMAP type
1088	if( (type == VSEG_TYPE_ANON) \|\| (type == VSEG_TYPE_FILE) \|\| (type == VSEG_TYPE_REMOTE) )
1089	{
1090	// get pointer on mmap allocator
1091	mmap_mgr_t * mgr = &vmm->mmap_mgr;
1092
1093	// compute zombi_list index
1094	uint32_t index = bits_log2( vseg->vpn_size );
1095
1096	// update zombi_list
1097	busylock_acquire( &mgr->lock );
1098	list_add_first( &mgr->zombi_list[index] , &vseg->zlist );
1099	busylock_release( &mgr->lock );
1100	}
1101
1102	// release physical memory allocated for vseg descriptor if no MMAP type
1103	if( (type != VSEG_TYPE_ANON) && (type != VSEG_TYPE_FILE) && (type != VSEG_TYPE_REMOTE) )
1104	{
1105	vseg_free( vseg );
1106	}
1107	} // end vmm_remove_vseg()
1108
1109	/////////////////////////////////////////
1110	void vmm_unmap_vseg( process_t * process,
1111	vseg_t * vseg )
1112	{
1113	vpn_t vpn; // VPN of current PTE
1114	vpn_t vpn_min; // VPN of first PTE
1115	vpn_t vpn_max; // VPN of last PTE (excluded)
1116	ppn_t ppn; // current PTE ppn value
1117	uint32_t attr; // current PTE attributes
1118	kmem_req_t req; // request to release memory
1119	xptr_t page_xp; // extended pointer on page descriptor
1120	cxy_t page_cxy; // page descriptor cluster
1121	page_t * page_ptr; // page descriptor pointer
1122	xptr_t forks_xp; // extended pointer on pending forks counter
1123	xptr_t lock_xp; // extended pointer on lock protecting forks counter
1124	uint32_t forks; // actual number of pendinf forks
1125
1126	#if DEBUG_VMM_UNMAP_VSEG
1127	uint32_t cycle = (uint32_t)hal_get_cycles();
1128	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1129	printk("\n[DBG] %s : thread %x enter / process %x / vseg %s / base %x / cycle %d\n",
1130	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
1131	#endif
1132
1133	// get pointer on local GPT
1134	gpt_t * gpt = &process->vmm.gpt;
1135
1136	// loop on pages in vseg
1137	vpn_min = vseg->vpn_base;
1138	vpn_max = vpn_min + vseg->vpn_size;
1139	for( vpn = vpn_min ; vpn < vpn_max ; vpn++ )
1140	{
1141	// get GPT entry
1142	hal_gpt_get_pte( XPTR( local_cxy , gpt ) , vpn , &attr , &ppn );
1143
1144	if( attr & GPT_MAPPED ) // entry is mapped
1145	{
1146
1147	#if( DEBUG_VMM_UNMAP_VSEG & 1 )
1148	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1149	printk("- vpn %x / ppn %x\n" , vpn , ppn );
1150	#endif
1151
1152	// check small page
1153	assert( (attr & GPT_SMALL) , "an user vseg must use small pages" );
1154
1155	// unmap GPT entry in local GPT
1156	hal_gpt_reset_pte( gpt , vpn );
1157
1158	// handle pending forks counter if
1159	// 1) not identity mapped
1160	// 2) reference cluster
1161	if( ((vseg->flags & VSEG_IDENT) == 0) &&
1162	(GET_CXY( process->ref_xp ) == local_cxy) )
1163	{
1164	// get extended pointer on physical page descriptor
1165	page_xp = ppm_ppn2page( ppn );
1166	page_cxy = GET_CXY( page_xp );
1167	page_ptr = GET_PTR( page_xp );
1168
1169	// get extended pointers on forks and lock fields
1170	forks_xp = XPTR( page_cxy , &page_ptr->forks );
1171	lock_xp = XPTR( page_cxy , &page_ptr->lock );
1172
1173	// get pending forks counter
1174	forks = hal_remote_l32( forks_xp );
1175
1176	if( forks ) // decrement pending forks counter
1177	{
1178	hal_remote_atomic_add( forks_xp , -1 );
1179	}
1180	else // release physical page to relevant cluster
1181	{
1182	if( page_cxy == local_cxy ) // local cluster
1183	{
1184	req.type = KMEM_PAGE;
1185	req.ptr = page_ptr;
1186	kmem_free( &req );
1187	}
1188	else // remote cluster
1189	{
1190	rpc_pmem_release_pages_client( page_cxy , page_ptr );
1191	}
1192	}
1193	}
1194	}
1195	}
1196
1197	#if DEBUG_VMM_UNMAP_VSEG
1198	cycle = (uint32_t)hal_get_cycles();
1199	if( DEBUG_VMM_UNMAP_VSEG < cycle )
1200	printk("\n[DBG] %s : thread %x exit / process %x / vseg %s / base %x / cycle %d\n",
1201	__FUNCTION__, CURRENT_THREAD, process->pid, vseg_type_str( vseg->type ), vseg->vpn_base, cycle );
1202	#endif
1203
1204	} // end vmm_unmap_vseg()
1205
1206	//////////////////////////////////////////////////////////////////////////////////////////
1207	// This low-level static function is called by the vmm_get_vseg(), vmm_get_pte(),
1208	// and vmm_resize_vseg() functions. It scan the local VSL to find the unique vseg
1209	// containing a given virtual address.
1210	//////////////////////////////////////////////////////////////////////////////////////////
1211	// @ vmm : pointer on the process VMM.
1212	// @ vaddr : virtual address.
1213	// @ return vseg pointer if success / return NULL if not found.
1214	//////////////////////////////////////////////////////////////////////////////////////////
1215	static vseg_t * vseg_from_vaddr( vmm_t * vmm,
1216	intptr_t vaddr )
1217	{
1218	xptr_t iter_xp;
1219	xptr_t vseg_xp;
1220	vseg_t * vseg;
1221
1222	// get extended pointers on VSL lock and root
1223	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1224	xptr_t root_xp = XPTR( local_cxy , &vmm->vsegs_root );
1225
1226	// get lock protecting the VSL
1227	remote_rwlock_rd_acquire( lock_xp );
1228
1229	// scan the list of vsegs in VSL
1230	XLIST_FOREACH( root_xp , iter_xp )
1231	{
1232	vseg_xp = XLIST_ELEMENT( iter_xp , vseg_t , xlist );
1233	vseg = GET_PTR( vseg_xp );
1234	if( (vaddr >= vseg->min) && (vaddr < vseg->max) )
1235	{
1236	// return success
1237	remote_rwlock_rd_release( lock_xp );
1238	return vseg;
1239	}
1240	}
1241
1242	// return failure
1243	remote_rwlock_rd_release( lock_xp );
1244	return NULL;
1245
1246	} // end vseg_from_vaddr()
1247
1248	/////////////////////////////////////////////
1249	error_t vmm_resize_vseg( process_t * process,
1250	intptr_t base,
1251	intptr_t size )
1252	{
1253	error_t error;
1254	vseg_t * new;
1255	vpn_t vpn_min;
1256	vpn_t vpn_max;
1257
1258	// get pointer on process VMM
1259	vmm_t * vmm = &process->vmm;
1260
1261	intptr_t addr_min = base;
1262	intptr_t addr_max = base + size;
1263
1264	// get pointer on vseg
1265	vseg_t * vseg = vseg_from_vaddr( vmm , base );
1266
1267	if( vseg == NULL) return EINVAL;
1268
1269	// get extended pointer on VSL lock
1270	xptr_t lock_xp = XPTR( local_cxy , &vmm->vsegs_lock );
1271
1272	// get lock protecting VSL
1273	remote_rwlock_wr_acquire( lock_xp );
1274
1275	if( (vseg->min > addr_min) \|\| (vseg->max < addr_max) ) // region not included in vseg
1276	{
1277	error = EINVAL;
1278	}
1279	else if( (vseg->min == addr_min) && (vseg->max == addr_max) ) // vseg must be removed
1280	{
1281	vmm_remove_vseg( vseg );
1282	error = 0;
1283	}
1284	else if( vseg->min == addr_min ) // vseg must be resized
1285	{
1286	// update vseg base address
1287	vseg->min = addr_max;
1288
1289	// update vpn_base and vpn_size
1290	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1291	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1292	vseg->vpn_base = vpn_min;
1293	vseg->vpn_size = vpn_max - vpn_min + 1;
1294	error = 0;
1295	}
1296	else if( vseg->max == addr_max ) // vseg must be resized
1297	{
1298	// update vseg max address
1299	vseg->max = addr_min;
1300
1301	// update vpn_base and vpn_size
1302	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1303	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1304	vseg->vpn_base = vpn_min;
1305	vseg->vpn_size = vpn_max - vpn_min + 1;
1306	error = 0;
1307	}
1308	else // vseg cut in three regions
1309	{
1310	// resize existing vseg
1311	vseg->max = addr_min;
1312
1313	// update vpn_base and vpn_size
1314	vpn_min = vseg->min >> CONFIG_PPM_PAGE_SHIFT;
1315	vpn_max = (vseg->max - 1) >> CONFIG_PPM_PAGE_SHIFT;
1316	vseg->vpn_base = vpn_min;
1317	vseg->vpn_size = vpn_max - vpn_min + 1;
1318
1319	// create new vseg
1320	new = vmm_create_vseg( process,
1321	vseg->type,
1322	addr_min,
1323	(vseg->max - addr_max),
1324	vseg->file_offset,
1325	vseg->file_size,
1326	vseg->mapper_xp,
1327	vseg->cxy );
1328
1329	if( new == NULL ) error = EINVAL;
1330	else error = 0;
1331	}
1332
1333	// release VMM lock
1334	remote_rwlock_wr_release( lock_xp );
1335
1336	return error;
1337
1338	} // vmm_resize_vseg()
1339
1340	///////////////////////////////////////////
1341	error_t vmm_get_vseg( process_t * process,
1342	intptr_t vaddr,
1343	vseg_t ** found_vseg )
1344	{
1345	xptr_t vseg_xp;
1346	error_t error;
1347	vseg_t * vseg;
1348	vmm_t * vmm;
1349
1350	// get pointer on local VMM
1351	vmm = &process->vmm;
1352
1353	// try to get vseg from local VMM
1354	vseg = vseg_from_vaddr( vmm , vaddr );
1355
1356	if( vseg == NULL ) // vseg not found in local cluster => try to get it from ref
1357	{
1358	// get extended pointer on reference process
1359	xptr_t ref_xp = process->ref_xp;
1360
1361	// get cluster and local pointer on reference process
1362	cxy_t ref_cxy = GET_CXY( ref_xp );
1363	process_t * ref_ptr = GET_PTR( ref_xp );
1364
1365	if( local_cxy == ref_cxy ) return -1; // local cluster is the reference
1366
1367	// get extended pointer on reference vseg
1368	rpc_vmm_get_vseg_client( ref_cxy , ref_ptr , vaddr , &vseg_xp , &error );
1369
1370	if( error ) return -1; // vseg not found => illegal user vaddr
1371
1372	// allocate a vseg in local cluster
1373	vseg = vseg_alloc();
1374
1375	if( vseg == NULL ) return -1; // cannot allocate a local vseg
1376
1377	// initialise local vseg from reference
1378	vseg_init_from_ref( vseg , vseg_xp );
1379
1380	// register local vseg in local VMM
1381	vmm_vseg_attach( &process->vmm , vseg );
1382	}
1383
1384	// success
1385	*found_vseg = vseg;
1386	return 0;
1387
1388	} // end vmm_get_vseg()
1389
1390	//////////////////////////////////////////////////////////////////////////////////////
1391	// This static function compute the target cluster to allocate a physical page
1392	// for a given <vpn> in a given <vseg>, allocates the page (with an RPC if required)
1393	// and returns an extended pointer on the allocated page descriptor.
1394	// It can be called by a thread running in any cluster.
1395	// The vseg cannot have the FILE type.
1396	//////////////////////////////////////////////////////////////////////////////////////
1397	static xptr_t vmm_page_allocate( vseg_t * vseg,
1398	vpn_t vpn )
1399	{
1400
1401	#if DEBUG_VMM_ALLOCATE_PAGE
1402	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1403	printk("\n[DBG] in %s : thread %x enter for vpn %x\n",
1404	__FUNCTION__ , CURRENT_THREAD, vpn );
1405	#endif
1406
1407	// compute target cluster
1408	page_t * page_ptr;
1409	cxy_t page_cxy;
1410	kmem_req_t req;
1411	uint32_t index;
1412
1413	uint32_t type = vseg->type;
1414	uint32_t flags = vseg->flags;
1415	uint32_t x_size = LOCAL_CLUSTER->x_size;
1416	uint32_t y_size = LOCAL_CLUSTER->y_size;
1417
1418	// check vseg type
1419	assert( ( type != VSEG_TYPE_FILE ) , "illegal vseg type\n" );
1420
1421	if( flags & VSEG_DISTRIB ) // distributed => cxy depends on vpn LSB
1422	{
1423	index = vpn & ((x_size * y_size) - 1);
1424	page_cxy = HAL_CXY_FROM_XY( (index / y_size) , (index % y_size) );
1425
1426	// If the cluster selected from VPN's LSBs is empty, we select one randomly
1427	if ( cluster_is_active( page_cxy ) == false )
1428	{
1429	page_cxy = cluster_random_select();
1430	}
1431	}
1432	else // other cases => cxy specified in vseg
1433	{
1434	page_cxy = vseg->cxy;
1435	}
1436
1437	// allocate a physical page from target cluster
1438	if( page_cxy == local_cxy ) // target cluster is the local cluster
1439	{
1440	req.type = KMEM_PAGE;
1441	req.size = 0;
1442	req.flags = AF_NONE;
1443	page_ptr = (page_t *)kmem_alloc( &req );
1444	}
1445	else // target cluster is not the local cluster
1446	{
1447	rpc_pmem_get_pages_client( page_cxy , 0 , &page_ptr );
1448	}
1449
1450	#if DEBUG_VMM_ALLOCATE_PAGE
1451	if( DEBUG_VMM_ALLOCATE_PAGE < (uint32_t)hal_get_cycles() )
1452	printk("\n[DBG] in %s : thread %x exit for vpn = %d / ppn = %x\n",
1453	__FUNCTION__ , CURRENT_THREAD, vpn, ppm_page2ppn( XPTR( page_cxy , page_ptr ) ) );
1454	#endif
1455
1456	if( page_ptr == NULL ) return XPTR_NULL;
1457	else return XPTR( page_cxy , page_ptr );
1458
1459	} // end vmm_page_allocate()
1460
1461	////////////////////////////////////////
1462	error_t vmm_get_one_ppn( vseg_t * vseg,
1463	vpn_t vpn,
1464	ppn_t * ppn )
1465	{
1466	error_t error;
1467	xptr_t page_xp; // extended pointer on physical page descriptor
1468	page_t * page_ptr; // local pointer on physical page descriptor
1469	uint32_t index; // missing page index in vseg mapper
1470	uint32_t type; // vseg type;
1471
1472	type = vseg->type;
1473	index = vpn - vseg->vpn_base;
1474
1475	#if DEBUG_VMM_GET_ONE_PPN
1476	thread_t * this = CURRENT_THREAD;
1477	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1478	printk("\n[DBG] %s : thread %x enter for vpn = %x / type = %s / index = %d\n",
1479	__FUNCTION__, this, vpn, vseg_type_str(type), index );
1480	#endif
1481
1482	// FILE type : get the physical page from the file mapper
1483	if( type == VSEG_TYPE_FILE )
1484	{
1485	// get extended pointer on mapper
1486	xptr_t mapper_xp = vseg->mapper_xp;
1487
1488	assert( (mapper_xp != XPTR_NULL),
1489	"mapper not defined for a FILE vseg\n" );
1490
1491	// get mapper cluster and local pointer
1492	cxy_t mapper_cxy = GET_CXY( mapper_xp );
1493	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
1494
1495	// get page descriptor from mapper
1496	if( mapper_cxy == local_cxy ) // mapper is local
1497	{
1498	page_ptr = mapper_get_page( mapper_ptr , index );
1499	}
1500	else // mapper is remote
1501	{
1502	rpc_mapper_get_page_client( mapper_cxy , mapper_ptr , index , &page_ptr );
1503	}
1504
1505	if ( page_ptr == NULL ) return EINVAL;
1506
1507	page_xp = XPTR( mapper_cxy , page_ptr );
1508	}
1509
1510	// Other types : allocate a physical page from target cluster,
1511	// as defined by vseg type and vpn value
1512	else
1513	{
1514	// allocate one physical page
1515	page_xp = vmm_page_allocate( vseg , vpn );
1516
1517	if( page_xp == XPTR_NULL ) return ENOMEM;
1518
1519	// initialise missing page from .elf file mapper for DATA and CODE types
1520	// the vseg->mapper_xp field is an extended pointer on the .elf file mapper
1521	if( (type == VSEG_TYPE_CODE) \|\| (type == VSEG_TYPE_DATA) )
1522	{
1523	// get extended pointer on mapper
1524	xptr_t mapper_xp = vseg->mapper_xp;
1525
1526	assert( (mapper_xp != XPTR_NULL),
1527	"mapper not defined for a CODE or DATA vseg\n" );
1528
1529	// get mapper cluster and local pointer
1530	cxy_t mapper_cxy = GET_CXY( mapper_xp );
1531	mapper_t * mapper_ptr = GET_PTR( mapper_xp );
1532
1533	// compute missing page offset in vseg
1534	uint32_t offset = index << CONFIG_PPM_PAGE_SHIFT;
1535
1536	// compute missing page offset in .elf file
1537	uint32_t elf_offset = vseg->file_offset + offset;
1538
1539	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1540	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1541	printk("\n[DBG] %s : thread %x for vpn = %x / elf_offset = %x\n",
1542	__FUNCTION__, this, vpn, elf_offset );
1543	#endif
1544	// compute extended pointer on page base
1545	xptr_t base_xp = ppm_page2base( page_xp );
1546
1547	// file_size (in .elf mapper) can be smaller than vseg_size (BSS)
1548	uint32_t file_size = vseg->file_size;
1549
1550	if( file_size < offset ) // missing page fully in BSS
1551	{
1552
1553	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1554	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1555	printk("\n[DBG] %s : thread%x for vpn = %x / fully in BSS\n",
1556	__FUNCTION__, this, vpn );
1557	#endif
1558	if( GET_CXY( page_xp ) == local_cxy )
1559	{
1560	memset( GET_PTR( base_xp ) , 0 , CONFIG_PPM_PAGE_SIZE );
1561	}
1562	else
1563	{
1564	hal_remote_memset( base_xp , 0 , CONFIG_PPM_PAGE_SIZE );
1565	}
1566	}
1567	else if( file_size >= (offset + CONFIG_PPM_PAGE_SIZE) ) // fully in mapper
1568	{
1569
1570	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1571	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1572	printk("\n[DBG] %s : thread %x, for vpn = %x / fully in mapper\n",
1573	__FUNCTION__, this, vpn );
1574	#endif
1575	if( mapper_cxy == local_cxy )
1576	{
1577	error = mapper_move_kernel( mapper_ptr,
1578	true, // to_buffer
1579	elf_offset,
1580	base_xp,
1581	CONFIG_PPM_PAGE_SIZE );
1582	}
1583	else
1584	{
1585	rpc_mapper_move_buffer_client( mapper_cxy,
1586	mapper_ptr,
1587	true, // to buffer
1588	false, // kernel buffer
1589	elf_offset,
1590	base_xp,
1591	CONFIG_PPM_PAGE_SIZE,
1592	&error );
1593	}
1594	if( error ) return EINVAL;
1595	}
1596	else // both in mapper and in BSS :
1597	// - (file_size - offset) bytes from mapper
1598	// - (page_size + offset - file_size) bytes from BSS
1599	{
1600
1601	#if (DEBUG_VMM_GET_ONE_PPN & 0x1)
1602	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1603	printk("\n[DBG] %s : thread %x for vpn = %x / both mapper & BSS\n"
1604	" %d bytes from mapper / %d bytes from BSS\n",
1605	__FUNCTION__, this, vpn,
1606	file_size - offset , offset + CONFIG_PPM_PAGE_SIZE - file_size );
1607	#endif
1608	// initialize mapper part
1609	if( mapper_cxy == local_cxy )
1610	{
1611	error = mapper_move_kernel( mapper_ptr,
1612	true, // to buffer
1613	elf_offset,
1614	base_xp,
1615	file_size - offset );
1616	}
1617	else
1618	{
1619	rpc_mapper_move_buffer_client( mapper_cxy,
1620	mapper_ptr,
1621	true, // to buffer
1622	false, // kernel buffer
1623	elf_offset,
1624	base_xp,
1625	file_size - offset,
1626	&error );
1627	}
1628	if( error ) return EINVAL;
1629
1630	// initialize BSS part
1631	if( GET_CXY( page_xp ) == local_cxy )
1632	{
1633	memset( GET_PTR( base_xp ) + file_size - offset , 0 ,
1634	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1635	}
1636	else
1637	{
1638	hal_remote_memset( base_xp + file_size - offset , 0 ,
1639	offset + CONFIG_PPM_PAGE_SIZE - file_size );
1640	}
1641	}
1642	} // end initialisation for CODE or DATA types
1643	}
1644
1645	// return ppn
1646	*ppn = ppm_page2ppn( page_xp );
1647
1648	#if DEBUG_VMM_GET_ONE_PPN
1649	if( DEBUG_VMM_GET_ONE_PPN < (uint32_t)hal_get_cycles() )
1650	printk("\n[DBG] %s : thread %x exit for vpn = %x / ppn = %x\n",
1651	__FUNCTION__ , this , vpn , *ppn );
1652	#endif
1653
1654	return 0;
1655
1656	} // end vmm_get_one_ppn()
1657
1658	///////////////////////////////////////////////////
1659	error_t vmm_handle_page_fault( process_t * process,
1660	vpn_t vpn )
1661	{
1662	vseg_t * vseg; // vseg containing vpn
1663	uint32_t new_attr; // new PTE_ATTR value
1664	ppn_t new_ppn; // new PTE_PPN value
1665	uint32_t ref_attr; // PTE_ATTR value in reference GPT
1666	ppn_t ref_ppn; // PTE_PPN value in reference GPT
1667	cxy_t ref_cxy; // reference cluster for missing vpn
1668	process_t * ref_ptr; // reference process for missing vpn
1669	xptr_t local_gpt_xp; // extended pointer on local GPT
1670	xptr_t local_lock_xp; // extended pointer on local GPT lock
1671	xptr_t ref_gpt_xp; // extended pointer on reference GPT
1672	xptr_t ref_lock_xp; // extended pointer on reference GPT lock
1673	error_t error; // value returned by called functions
1674
1675	// get local vseg (access to reference VSL can be required)
1676	error = vmm_get_vseg( process,
1677	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1678	&vseg );
1679
1680	if( error )
1681	{
1682	printk("\n[ERROR] in %s : illegal VPN %x in process %x\n",
1683	__FUNCTION__ , vpn , process->pid );
1684
1685	return EXCP_USER_ERROR;
1686	}
1687
1688	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1689	uint32_t cycle = (uint32_t)hal_get_cycles();
1690	thread_t * this = CURRENT_THREAD;
1691	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1692	printk("\n[DBG] %s : threadr[%x,%x] enter for vpn %x / %s / cycle %d\n",
1693	__FUNCTION__, this->process->pid, this->trdid, vpn, vseg_type_str(vseg->type), cycle );
1694	#endif
1695
1696	//////////////// private vseg => access only the local GPT
1697	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1698	{
1699	// build extended pointer on local GPT and local GPT lock
1700	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1701	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1702
1703	// take local GPT lock in write mode
1704	remote_rwlock_wr_acquire( local_lock_xp );
1705
1706	// check VPN still unmapped in local GPT
1707	// do nothing if VPN has been mapped by a a concurrent page_fault
1708	hal_gpt_get_pte( local_gpt_xp,
1709	vpn,
1710	&new_attr,
1711	&new_ppn );
1712
1713	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1714	{
1715	// allocate and initialise a physical page depending on the vseg type
1716	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1717
1718	if( error )
1719	{
1720	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1721	__FUNCTION__ , process->pid , vpn );
1722
1723	// release local GPT lock in write mode
1724	remote_rwlock_wr_release( local_lock_xp );
1725
1726	return EXCP_KERNEL_PANIC;
1727	}
1728
1729	// define new_attr from vseg flags
1730	new_attr = GPT_MAPPED \| GPT_SMALL;
1731	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1732	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1733	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1734	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1735
1736	// set PTE (PPN & attribute) to local GPT
1737	error = hal_gpt_set_pte( local_gpt_xp,
1738	vpn,
1739	new_attr,
1740	new_ppn );
1741	if ( error )
1742	{
1743	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1744	__FUNCTION__ , process->pid , vpn );
1745
1746	// release local GPT lock in write mode
1747	remote_rwlock_wr_release( local_lock_xp );
1748
1749	return EXCP_KERNEL_PANIC;
1750	}
1751	}
1752
1753	// release local GPT lock in write mode
1754	remote_rwlock_wr_release( local_lock_xp );
1755
1756	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1757	cycle = (uint32_t)hal_get_cycles();
1758	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1759	printk("\n[DBG] %s : private page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1760	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1761	#endif
1762	return EXCP_NON_FATAL;
1763
1764	} // end local GPT access
1765
1766	//////////// public vseg => access reference GPT
1767	else
1768	{
1769	// get reference process cluster and local pointer
1770	ref_cxy = GET_CXY( process->ref_xp );
1771	ref_ptr = GET_PTR( process->ref_xp );
1772
1773	// build extended pointer on reference GPT and reference GPT lock
1774	ref_gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1775	ref_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1776
1777	// build extended pointer on local GPT and local GPT lock
1778	local_gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1779	local_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1780
1781	// take reference GPT lock in read mode
1782	remote_rwlock_rd_acquire( ref_lock_xp );
1783
1784	// get directly PPN & attributes from reference GPT
1785	// this can avoids a costly RPC for a false page fault
1786	hal_gpt_get_pte( ref_gpt_xp,
1787	vpn,
1788	&ref_attr,
1789	&ref_ppn );
1790
1791	// release reference GPT lock in read mode
1792	remote_rwlock_rd_release( ref_lock_xp );
1793
1794	if( ref_attr & GPT_MAPPED ) // false page fault => update local GPT
1795	{
1796	// take local GPT lock in write mode
1797	remote_rwlock_wr_acquire( local_lock_xp );
1798
1799	// check VPN still unmapped in local GPT
1800	hal_gpt_get_pte( local_gpt_xp,
1801	vpn,
1802	&new_attr,
1803	&new_ppn );
1804
1805	if( (new_attr & GPT_MAPPED) == 0 ) // VPN still unmapped
1806	{
1807	// update local GPT from reference GPT
1808	error = hal_gpt_set_pte( local_gpt_xp,
1809	vpn,
1810	ref_attr,
1811	ref_ppn );
1812	if( error )
1813	{
1814	printk("\n[ERROR] in %s : cannot update local GPT / process %x / vpn = %x\n",
1815	__FUNCTION__ , process->pid , vpn );
1816
1817	// release local GPT lock in write mode
1818	remote_rwlock_wr_release( local_lock_xp );
1819
1820	return EXCP_KERNEL_PANIC;
1821	}
1822	}
1823	else // VPN has been mapped by a a concurrent page_fault
1824	{
1825	// keep PTE from local GPT
1826	ref_attr = new_attr;
1827	ref_ppn = new_ppn;
1828	}
1829
1830	// release local GPT lock in write mode
1831	remote_rwlock_wr_release( local_lock_xp );
1832
1833	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1834	cycle = (uint32_t)hal_get_cycles();
1835	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1836	printk("\n[DBG] %s : false page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1837	__FUNCTION__, vpn, ref_ppn, ref_attr, cycle );
1838	#endif
1839	return EXCP_NON_FATAL;
1840	}
1841	else // true page fault => update reference GPT
1842	{
1843	// take reference GPT lock in write mode
1844	remote_rwlock_wr_acquire( ref_lock_xp );
1845
1846	// check VPN still unmapped in reference GPT
1847	// do nothing if VPN has been mapped by a a concurrent page_fault
1848	hal_gpt_get_pte( ref_gpt_xp,
1849	vpn,
1850	&ref_attr,
1851	&ref_ppn );
1852
1853	if( (ref_attr & GPT_MAPPED) == 0 ) // VPN actually unmapped
1854	{
1855	// allocate and initialise a physical page depending on the vseg type
1856	error = vmm_get_one_ppn( vseg , vpn , &new_ppn );
1857
1858	if( error )
1859	{
1860	printk("\n[ERROR] in %s : no memory / process = %x / vpn = %x\n",
1861	__FUNCTION__ , process->pid , vpn );
1862
1863	// release reference GPT lock in write mode
1864	remote_rwlock_wr_release( ref_lock_xp );
1865
1866	return EXCP_KERNEL_PANIC;
1867	}
1868
1869	// define new_attr from vseg flags
1870	new_attr = GPT_MAPPED \| GPT_SMALL;
1871	if( vseg->flags & VSEG_USER ) new_attr \|= GPT_USER;
1872	if( vseg->flags & VSEG_WRITE ) new_attr \|= GPT_WRITABLE;
1873	if( vseg->flags & VSEG_EXEC ) new_attr \|= GPT_EXECUTABLE;
1874	if( vseg->flags & VSEG_CACHE ) new_attr \|= GPT_CACHABLE;
1875
1876	// update reference GPT
1877	error = hal_gpt_set_pte( ref_gpt_xp,
1878	vpn,
1879	new_attr,
1880	new_ppn );
1881
1882	// update local GPT (protected by reference GPT lock)
1883	error \|= hal_gpt_set_pte( local_gpt_xp,
1884	vpn,
1885	new_attr,
1886	new_ppn );
1887
1888	if( error )
1889	{
1890	printk("\n[ERROR] in %s : cannot update GPT / process %x / vpn = %x\n",
1891	__FUNCTION__ , process->pid , vpn );
1892
1893	// release reference GPT lock in write mode
1894	remote_rwlock_wr_release( ref_lock_xp );
1895
1896	return EXCP_KERNEL_PANIC;
1897	}
1898	}
1899
1900	// release reference GPT lock in write mode
1901	remote_rwlock_wr_release( ref_lock_xp );
1902
1903	#if DEBUG_VMM_HANDLE_PAGE_FAULT
1904	cycle = (uint32_t)hal_get_cycles();
1905	if( DEBUG_VMM_HANDLE_PAGE_FAULT < cycle )
1906	printk("\n[DBG] %s : true page fault handled / vpn %x / ppn %x / attr %x / cycle %d\n",
1907	__FUNCTION__, vpn, new_ppn, new_attr, cycle );
1908	#endif
1909	return EXCP_NON_FATAL;
1910	}
1911	}
1912	} // end vmm_handle_page_fault()
1913
1914	////////////////////////////////////////////
1915	error_t vmm_handle_cow( process_t * process,
1916	vpn_t vpn )
1917	{
1918	vseg_t * vseg; // vseg containing vpn
1919	cxy_t ref_cxy; // reference cluster for missing vpn
1920	process_t * ref_ptr; // reference process for missing vpn
1921	xptr_t gpt_xp; // extended pointer on GPT
1922	xptr_t gpt_lock_xp; // extended pointer on GPT lock
1923	uint32_t old_attr; // current PTE_ATTR value
1924	ppn_t old_ppn; // current PTE_PPN value
1925	uint32_t new_attr; // new PTE_ATTR value
1926	ppn_t new_ppn; // new PTE_PPN value
1927	error_t error;
1928
1929	#if DEBUG_VMM_HANDLE_COW
1930	uint32_t cycle = (uint32_t)hal_get_cycles();
1931	thread_t * this = CURRENT_THREAD;
1932	if( DEBUG_VMM_HANDLE_COW < cycle )
1933	printk("\n[DBG] %s : thread[%x,%x] enter for vpn %x / core[%x,%d] / cycle %d\n",
1934	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
1935	#endif
1936
1937	// get local vseg
1938	error = vmm_get_vseg( process,
1939	(intptr_t)vpn<<CONFIG_PPM_PAGE_SHIFT,
1940	&vseg );
1941
1942	if( error )
1943	{
1944	printk("\n[PANIC] in %s : VPN %x in process %x not in registered vseg\n",
1945	__FUNCTION__, vpn, process->pid );
1946
1947	return EXCP_KERNEL_PANIC;
1948	}
1949
1950	// get reference GPT cluster and local pointer
1951	ref_cxy = GET_CXY( process->ref_xp );
1952	ref_ptr = GET_PTR( process->ref_xp );
1953
1954	// build relevant extended pointers on GPT and GPT lock
1955	// - access local GPT for a private vseg
1956	// - access reference GPT for a public vseg
1957	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
1958	{
1959	gpt_xp = XPTR( local_cxy , &process->vmm.gpt );
1960	gpt_lock_xp = XPTR( local_cxy , &process->vmm.gpt_lock );
1961	}
1962	else
1963	{
1964	gpt_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt );
1965	gpt_lock_xp = XPTR( ref_cxy , &ref_ptr->vmm.gpt_lock );
1966	}
1967
1968	// take GPT lock in write mode
1969	remote_rwlock_wr_acquire( gpt_lock_xp );
1970
1971	// get current PTE from reference GPT
1972	hal_gpt_get_pte( gpt_xp,
1973	vpn,
1974	&old_attr,
1975	&old_ppn );
1976
1977	// the PTE must be mapped for a COW
1978	if( (old_attr & GPT_MAPPED) == 0 )
1979	{
1980	printk("\n[PANIC] in %s : VPN %x in process %x unmapped\n",
1981	__FUNCTION__, vpn, process->pid );
1982
1983	// release GPT lock in write mode
1984	remote_rwlock_wr_acquire( gpt_lock_xp );
1985
1986	return EXCP_KERNEL_PANIC;
1987	}
1988
1989	// get extended pointer, cluster and local pointer on physical page descriptor
1990	xptr_t page_xp = ppm_ppn2page( old_ppn );
1991	cxy_t page_cxy = GET_CXY( page_xp );
1992	page_t * page_ptr = GET_PTR( page_xp );
1993
1994	// get extended pointers on forks and lock field in page descriptor
1995	xptr_t forks_xp = XPTR( page_cxy , &page_ptr->forks );
1996	xptr_t forks_lock_xp = XPTR( page_cxy , &page_ptr->lock );
1997
1998	// take lock protecting "forks" counter
1999	remote_busylock_acquire( forks_lock_xp );
2000
2001	// get number of pending forks from page descriptor
2002	uint32_t forks = hal_remote_l32( forks_xp );
2003
2004	if( forks ) // pending fork => allocate a new page, and copy old to new
2005	{
2006	// allocate a new physical page
2007	page_xp = vmm_page_allocate( vseg , vpn );
2008	if( page_xp == XPTR_NULL )
2009	{
2010	printk("\n[PANIC] in %s : no memory for vpn %x in process %x\n",
2011	__FUNCTION__ , vpn, process->pid );
2012
2013	// release GPT lock in write mode
2014	remote_rwlock_wr_acquire( gpt_lock_xp );
2015
2016	// release lock protecting "forks" counter
2017	remote_busylock_release( forks_lock_xp );
2018
2019	return EXCP_KERNEL_PANIC;
2020	}
2021
2022	// compute allocated page PPN
2023	new_ppn = ppm_page2ppn( page_xp );
2024
2025	// copy old page content to new page
2026	xptr_t old_base_xp = ppm_ppn2base( old_ppn );
2027	xptr_t new_base_xp = ppm_ppn2base( new_ppn );
2028	memcpy( GET_PTR( new_base_xp ),
2029	GET_PTR( old_base_xp ),
2030	CONFIG_PPM_PAGE_SIZE );
2031
2032	// decrement pending forks counter in page descriptor
2033	hal_remote_atomic_add( forks_xp , -1 );
2034
2035	#if(DEBUG_VMM_HANDLE_COW & 1)
2036	if( DEBUG_VMM_HANDLE_COW < cycle )
2037	printk("\n[DBG] %s : thread[%x,%x] : pending forks => allocate a new PPN %x\n",
2038	__FUNCTION__, process->pid, this->trdid, new_ppn );
2039	#endif
2040
2041	}
2042	else // no pending fork => keep the existing page
2043	{
2044
2045	#if(DEBUG_VMM_HANDLE_COW & 1)
2046	if( DEBUG_VMM_HANDLE_COW < cycle )
2047	printk("\n[DBG] %s : thread[%x,%x] no pending forks => keep existing PPN %x\n",
2048	__FUNCTION__, process->pid, this->trdid, new_ppn );
2049	#endif
2050	new_ppn = old_ppn;
2051	}
2052
2053	// release lock protecting "forks" counter
2054	remote_busylock_release( forks_lock_xp );
2055
2056	// build new_attr : reset COW and set WRITABLE,
2057	new_attr = (old_attr \| GPT_WRITABLE) & (~GPT_COW);
2058
2059	// update the relevan GPT
2060	// - private vseg => update local GPT
2061	// - public vseg => update all GPT copies
2062	if( (vseg->type == VSEG_TYPE_STACK) \|\| (vseg->type == VSEG_TYPE_CODE) )
2063	{
2064	hal_gpt_set_pte( gpt_xp,
2065	vpn,
2066	new_attr,
2067	new_ppn );
2068	}
2069	else
2070	{
2071	if( ref_cxy == local_cxy ) // reference cluster is local
2072	{
2073	vmm_global_update_pte( process,
2074	vpn,
2075	new_attr,
2076	new_ppn );
2077	}
2078	else // reference cluster is remote
2079	{
2080	rpc_vmm_global_update_pte_client( ref_cxy,
2081	ref_ptr,
2082	vpn,
2083	new_attr,
2084	new_ppn );
2085	}
2086	}
2087
2088	// release GPT lock in write mode
2089	remote_rwlock_wr_release( gpt_lock_xp );
2090
2091	#if DEBUG_VMM_HANDLE_COW
2092	cycle = (uint32_t)hal_get_cycles();
2093	if( DEBUG_VMM_HANDLE_COW < cycle )
2094	printk("\n[DBG] %s : thread[%x,%x] exit for vpn %x / core[%x,%d] / cycle %d\n",
2095	__FUNCTION__, process->pid, this->trdid, vpn, local_cxy, this->core->lid, cycle );
2096	#endif
2097
2098	return EXCP_NON_FATAL;
2099
2100	} // end vmm_handle_cow()
2101

Note: See TracBrowser for help on using the repository browser.

Download in other formats: