source: trunk/kernel/mm/mapper.c @ 560

/*
 * mapper.c - Map memory, file or device in process virtual address space.
 *
 * Authors   Mohamed Lamine Karaoui (2015)
 *           Alain Greiner (2016,2017,2018)
 *
 * Copyright (c)  UPMC Sorbonne Universites
 *
 * This file is part of ALMOS-MKH.
 *
 * ALMOS-MKH is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.0 of the License.
 *
 * ALMOS-MKH is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with ALMOS-MKH; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <kernel_config.h>
#include <hal_kernel_types.h>
#include <hal_special.h>
#include <hal_uspace.h>
#include <grdxt.h>
#include <rwlock.h>
#include <printk.h>
#include <memcpy.h>
#include <thread.h>
#include <core.h>
#include <process.h>
#include <kmem.h>
#include <kcm.h>
#include <page.h>
#include <cluster.h>
#include <vfs.h>
#include <mapper.h>

//////////////////////////////////////////////
mapper_t * mapper_create( vfs_fs_type_t type )
{
    mapper_t * mapper;
    kmem_req_t req;
    error_t    error;

    // allocate memory for associated mapper
    req.type  = KMEM_MAPPER;
    req.size  = sizeof(mapper_t);
    req.flags = AF_KERNEL | AF_ZERO;
    mapper    = (mapper_t *)kmem_alloc( &req );

    if( mapper == NULL )
    {
        printk("\n[ERROR] in %s : no memory for mapper descriptor\n", __FUNCTION__ );
        return NULL;
    }

    // initialize refcount & inode
    mapper->refcount = 0;
    mapper->inode    = NULL;

    // initialize radix tree
    error = grdxt_init( &mapper->radix,
                        CONFIG_VMM_GRDXT_W1,
                        CONFIG_VMM_GRDXT_W2,
                        CONFIG_VMM_GRDXT_W3 );

    if( error )
    {
        printk("\n[ERROR] in %s : cannot initialize radix tree\n", __FUNCTION__ );
        req.type  = KMEM_MAPPER;
        req.ptr   = mapper;
        kmem_free( &req );
        return NULL;
    }

    // initialize mapper type
    mapper->type = type;

    // initialize mapper lock
    rwlock_init(  &mapper->lock );

    // initialize waiting threads xlist (empty)
    xlist_root_init( XPTR( local_cxy , &mapper->wait_root ) );

    // initialize vsegs xlist (empty)
    xlist_root_init( XPTR( local_cxy , &mapper->vsegs_root ) );

    return mapper;

}  // end mapper_create()

///////////////////////////////////////////
error_t mapper_destroy( mapper_t * mapper )
{
    page_t   * page;
    uint32_t   found_index = 0;
    uint32_t   start_index = 0;
    kmem_req_t req;
    error_t    error;

    // scan radix three and release all registered pages to PPM
    do
    {
        // get page from radix tree
        page = (page_t *)grdxt_get_first( &mapper->radix , start_index , &found_index );

        if( page != NULL )
        {
            // remove page from mapper and release to PPM
            error = mapper_release_page( mapper , page );

            if ( error ) return error;

            // update start_key value for next page
            start_index = found_index;
        }
    }
    while( page != NULL );

    // release the memory allocated to radix-tree itself
    grdxt_destroy( &mapper->radix );

    // release memory for mapper descriptor
    req.type = KMEM_MAPPER;
    req.ptr  = mapper;
    kmem_free( &req );

    return 0;

}  // end mapper_destroy()

////////////////////////////////////////////
page_t * mapper_get_page( mapper_t * mapper,
                          uint32_t   index )
{
    kmem_req_t    req;
    page_t      * page;
    error_t       error;

#if DEBUG_MAPPER_GET_PAGE
uint32_t cycle = (uint32_t)hal_get_cycles();
if( DEBUG_MAPPER_GET_PAGE < cycle )
printk("\n[DBG] %s : thread %x enter for page %d / mapper %x / cycle %d\n",
__FUNCTION__ , CURRENT_THREAD , index , mapper , cycle );
#endif

    thread_t * this = CURRENT_THREAD;

    // take mapper lock in READ_MODE
    rwlock_rd_lock( &mapper->lock );

    // search page in radix tree
    page = (page_t *)grdxt_lookup( &mapper->radix , index );

    // test if page available in mapper
    if( ( page == NULL) || page_is_flag( page , PG_INLOAD ) )  // page not available
    {

        // release the lock in READ_MODE and take it in WRITE_MODE
        rwlock_rd_unlock( &mapper->lock );
        rwlock_wr_lock( &mapper->lock );

        // second test on missing page because the page status can have been modified
        // by another thread, when passing from READ_MODE to WRITE_MODE.
        // from this point there is no concurrent accesses to mapper.

        page = grdxt_lookup( &mapper->radix , index );

        if ( page == NULL )   // missing page => create it and load it from file system
        {

#if (DEBUG_MAPPER_GET_PAGE & 1)
if( DEBUG_MAPPER_GET_PAGE < cycle )
printk("\n[DBG] %s : missing page => load from device\n", __FUNCTION__ );
#endif
            // allocate one page from PPM
            req.type  = KMEM_PAGE;
            req.size  = 0;
            req.flags = AF_NONE;
            page = kmem_alloc( &req );

            if( page == NULL )
            {
                printk("\n[ERROR] in %s : thread %x cannot allocate a page in cluster %x\n",
                       __FUNCTION__ , this->trdid , local_cxy );
                rwlock_wr_unlock( &mapper->lock );
                return NULL;
            }

            // initialize the page descriptor
            page_init( page );
            page_set_flag( page , PG_INIT | PG_INLOAD );
            page_refcount_up( page );
            page->mapper = mapper;
            page->index  = index;

            // insert page in mapper radix tree
            error = grdxt_insert( &mapper->radix, index , page );

            // release mapper lock from WRITE_MODE
            rwlock_wr_unlock( &mapper->lock );

            if( error )
            {
                printk("\n[ERROR] in %s : thread %x cannot insert page in mapper\n",
                       __FUNCTION__ , this->trdid );
                mapper_release_page( mapper , page );
                page_clear_flag( page , PG_ALL );
                req.ptr  = page;
                req.type = KMEM_PAGE;
                kmem_free(&req);
                return NULL;
            }

            // launch I/O operation to load page from file system
            error = vfs_mapper_move_page( page, 
                                          true );   // to mapper
            if( error )
            {
                printk("\n[ERROR] in %s : thread %x cannot load page from device\n",
                       __FUNCTION__ , this->trdid );
                mapper_release_page( mapper , page );
                page_clear_flag( page , PG_ALL );
                req.ptr  = page;
                req.type = KMEM_PAGE;
                kmem_free( &req );
                return NULL;
            }

            // reset the page INLOAD flag to make the page available to all readers
            page_clear_flag( page , PG_INLOAD );
        }
        else if( page_is_flag( page , PG_INLOAD ) )   // page is loaded by another thread
        {
            // release mapper lock from WRITE_MODE
            rwlock_wr_unlock( &mapper->lock );

            // wait load completion
            while( 1 )
            {
                // exit waiting loop when loaded
                if( page_is_flag( page , PG_INLOAD ) == false ) break;

                // deschedule
                sched_yield("waiting page loading");
            }
        }
    }
    else                          // page available in mapper
    {
        rwlock_rd_unlock( &mapper->lock );
    }

#if DEBUG_MAPPER_GET_PAGE
cycle = (uint32_t)hal_get_cycles();
if( DEBUG_MAPPER_GET_PAGE < cycle )
printk("\n[DBG] %s : thread %x exit for page %d / ppn %x / cycle %d\n",
__FUNCTION__, CURRENT_THREAD, index, ppm_page2ppn(XPTR(local_cxy, page)), cycle );
#endif

    return page;

}  // end mapper_get_page()

///////////////////////////////////////////////
error_t mapper_release_page( mapper_t * mapper,
                             page_t   * page )
{
    error_t error;

    // lauch IO operation to update page to file system
    error = vfs_mapper_move_page( page , false );    // from mapper

    if( error )
    {
        printk("\n[ERROR] in %s : cannot update file system\n", __FUNCTION__ );
        return EIO;
    }

    // take mapper lock in WRITE_MODE
    rwlock_wr_lock( &mapper->lock );

    // remove physical page from radix tree
    grdxt_remove( &mapper->radix , page->index );

    // release mapper lock from WRITE_MODE
    rwlock_wr_unlock( &mapper->lock );

    // release page to PPM
    kmem_req_t   req;
    req.type  = KMEM_PAGE;
    req.ptr   = page;
    kmem_free( &req );

    return 0;

}  // end mapper_release_page()

///////////////////////////////////////////////////
error_t mapper_move_user( mapper_t * mapper,
                          bool_t     to_buffer,
                          uint32_t   file_offset,
                          void     * buffer,
                          uint32_t   size )
{
    uint32_t   page_offset;    // first byte to move to/from a mapper page
    uint32_t   page_count;     // number of bytes to move to/from a mapper page
    uint32_t   index;          // current mapper page index
    uint32_t   done;           // number of moved bytes
    page_t   * page;           // current mapper page descriptor
    uint8_t  * map_ptr;        // current mapper  address
    uint8_t  * buf_ptr;        // current buffer  address

#if DEBUG_MAPPER_MOVE_USER
uint32_t cycle = (uint32_t)hal_get_cycles();
if( DEBUG_MAPPER_MOVE_USER < cycle )
printk("\n[DBG] %s : thread %x enter / to_buf %d / buffer %x / cycle %d\n",
__FUNCTION__ , CURRENT_THREAD , to_buffer , buffer , cycle );
#endif

    // compute offsets of first and last bytes in file
    uint32_t min_byte = file_offset;
    uint32_t max_byte = file_offset + size -1;

    // compute indexes of pages for first and last byte in mapper
    uint32_t first = min_byte >> CONFIG_PPM_PAGE_SHIFT;
    uint32_t last  = max_byte >> CONFIG_PPM_PAGE_SHIFT;

    done = 0;

    // loop on pages in mapper
    for( index = first ; index <= last ; index++ )
    {
        // compute page_offset
        if( index == first ) page_offset = min_byte & CONFIG_PPM_PAGE_MASK;
        else                 page_offset = 0;

        // compute number of bytes in page
        if      ( first == last  ) page_count = size;
        else if ( index == first ) page_count = CONFIG_PPM_PAGE_SIZE - page_offset;
        else if ( index == last  ) page_count = (max_byte & CONFIG_PPM_PAGE_MASK) + 1;
        else                       page_count = CONFIG_PPM_PAGE_SIZE;

#if (DEBUG_MAPPER_MOVE_USER & 1)
if( DEBUG_MAPPER_MOVE_USER < cycle )
printk("\n[DBG] %s : index = %d / offset = %d / count = %d\n",
__FUNCTION__ , index , page_offset , page_count );
#endif

        // get page descriptor
        page = mapper_get_page( mapper , index );

        if ( page == NULL ) return EINVAL;

        // compute pointer in mapper
        xptr_t base_xp = ppm_page2base( XPTR( local_cxy, page ) );
        map_ptr = (uint8_t *)GET_PTR( base_xp ) + page_offset;

        // compute pointer in buffer
        buf_ptr = (uint8_t *)buffer + done;

        // move fragment
        if( to_buffer )
        {
            hal_copy_to_uspace( buf_ptr , map_ptr , page_count );
        }
        else
        {
            page_do_dirty( page );
            hal_copy_from_uspace( map_ptr , buf_ptr , page_count );
        }

        done += page_count;
    }

#if DEBUG_MAPPER_MOVE_USER
cycle = (uint32_t)hal_get_cycles();
if( DEBUG_MAPPER_MOVE_USER < cycle )
printk("\n[DBG] %s : thread %x exit / to_buf %d / buffer %x / cycle %d\n",
__FUNCTION__ , CURRENT_THREAD , to_buffer , buffer , cycle );
#endif

    return 0;

}  // end mapper_move_user()

////////////////////////////////////////////////
error_t mapper_move_kernel( mapper_t  *  mapper,
                            bool_t       to_buffer,
                            uint32_t     file_offset,
                            xptr_t       buffer_xp,
                            uint32_t     size )
{
    uint32_t   page_offset;    // first byte to move to/from a mapper page
    uint32_t   page_count;     // number of bytes to move to/from a mapper page
    uint32_t   index;          // current mapper page index
    uint32_t   done;           // number of moved bytes
    page_t   * page;           // current mapper page descriptor

    uint8_t  * src_ptr;        // source buffer local pointer
    cxy_t      src_cxy;        // source cluster
    uint8_t  * dst_ptr;        // destination buffer local pointer
    cxy_t      dst_cxy;        // destination cluster

    // get buffer cluster and local pointer
    cxy_t     buffer_cxy = GET_CXY( buffer_xp );
    uint8_t * buffer_ptr = (uint8_t *)GET_PTR( buffer_xp );

#if DEBUG_MAPPER_MOVE_KERNEL
uint32_t cycle = (uint32_t)hal_get_cycles();
if( DEBUG_MAPPER_MOVE_KERNEL < cycle )
printk("\n[DBG] %s : thread %x enter / to_buf %d / buf_cxy %x / buf_ptr %x / cycle %d\n",
__FUNCTION__ , CURRENT_THREAD , to_buffer , buffer_cxy , buffer_ptr , cycle );
#endif

    // compute offsets of first and last bytes in file
    uint32_t min_byte = file_offset;
    uint32_t max_byte = file_offset + size -1;

    // compute indexes for first and last pages in mapper
    uint32_t first = min_byte >> CONFIG_PPM_PAGE_SHIFT;
    uint32_t last  = max_byte >> CONFIG_PPM_PAGE_SHIFT;

#if (DEBUG_MAPPER_MOVE_KERNEL & 1)
if( DEBUG_MAPPER_MOVE_KERNEL < cycle )
printk("\n[DBG] %s : first_page %d / last_page %d\n", __FUNCTION__, first, last );
#endif

    // compute source and destination clusters
    if( to_buffer )
    {
        dst_cxy = buffer_cxy;
        src_cxy = local_cxy;
    }
    else
    {
        src_cxy = buffer_cxy;
        dst_cxy = local_cxy;
    }

    done = 0;

    // loop on pages in mapper
    for( index = first ; index <= last ; index++ )
    {
        // compute page_offset
        if( index == first ) page_offset = min_byte & CONFIG_PPM_PAGE_MASK;
        else                 page_offset = 0;

        // compute number of bytes to move in page
        if      ( first == last  ) page_count = size;
        else if ( index == first ) page_count = CONFIG_PPM_PAGE_SIZE - page_offset;
        else if ( index == last  ) page_count = (max_byte & CONFIG_PPM_PAGE_MASK) + 1;
        else                       page_count = CONFIG_PPM_PAGE_SIZE;

#if (DEBUG_MAPPER_MOVE_KERNEL & 1)
if( DEBUG_MAPPER_MOVE_KERNEL < cycle )
printk("\n[DBG] %s : page_index = %d / offset = %d / bytes = %d\n",
__FUNCTION__ , index , page_offset , page_count );
#endif

        // get page descriptor
        page = mapper_get_page( mapper , index );

        if ( page == NULL ) return EINVAL;

        // get page base address
        xptr_t    base_xp  = ppm_page2base( XPTR( local_cxy , page ) );
        uint8_t * base_ptr = (uint8_t *)GET_PTR( base_xp );

        // compute source and destination pointers
        if( to_buffer )
        {
            dst_ptr = buffer_ptr + done;
            src_ptr = base_ptr + page_offset;
        }
        else
        {
            src_ptr = buffer_ptr + done;
            dst_ptr = base_ptr + page_offset;

            page_do_dirty( page );
        }

        // move fragment
        hal_remote_memcpy( XPTR( dst_cxy , dst_ptr ), XPTR( src_cxy , src_ptr ), page_count );

        done += page_count;
    }

#if DEBUG_MAPPER_MOVE_KERNEL
cycle = (uint32_t)hal_get_cycles();
if( DEBUG_MAPPER_MOVE_KERNEL < cycle )
printk("\n[DBG] %s : thread %x exit / to_buf %d / buf_cxy %x / buf_ptr %x / cycle %d\n",
__FUNCTION__ , CURRENT_THREAD , to_buffer , buffer_cxy , buffer_ptr , cycle );
#endif

    return 0;

}  // end mapper_move_kernel()