source: soft/giet_vm/giet_drivers/ioc_driver.c @ 269

///////////////////////////////////////////////////////////////////////////////////
// File     : ioc_driver.c
// Date     : 23/05/2013
// Author   : alain greiner
// Copyright (c) UPMC-LIP6
///////////////////////////////////////////////////////////////////////////////////
// The ioc_driver.c and ioc_driver.h files are part ot the GIET-VM kernel.
// This driver supports the SocLib vci_block_device component, that is
// a single channel, block oriented, external storage contrÃŽler.
//
// It can exist only one block-device controler in the architecture.
//
// The _ioc_read() and _ioc_write() functions use the _ioc_access() function,
// that is always blocking, but can be called in 4 modes:
//
// - In BOOT_PA mode, the _ioc_access() function use the buffer virtual address 
//   as a physical address (as the page tables are not build) and use a polling
//   policy on the IOC_STATUS register to detect transfer completion, as
//   hardware interrupts are not activated. This mode is used by the
//   boot code to load the map.bin file into memory.
//
// - In BOOT_VA mode, the _ioc_access() function makes a V2P translation to
//   compute the buffer physical address, and use a polling policy on IOC_STATUS
//   register to detect transfer completion. This mode is used by the boot code
//   to load the various .elf files into memory.
//
// - In KERNEL mode, the _ioc_access() function makes a V2P translation to
//   compute the buffer physical address, and use a descheduling strategy:
//   The ISR executed when transfer completes should restart the calling task. 
//   There is no checking of user access right to the memory buffer.
//   This mode must be used to access IOC, for an "open" system call.
//
// - In USER mode, the _ioc_access() function makes a V2P translation to
//   compute the buffer physical address, and use a descheduling strategy:
//   The ISR executed when transfer completes should restart the calling task, 
//   The user access right to the memory buffer must be checked.
//   This mode must be used to access IOC, for a "read/write" system call.
//
// As the IOC component can be used by several programs running in parallel,
// the _ioc_lock variable guaranties exclusive access to the device.  The
// _ioc_read() and _ioc_write() functions use atomic LL/SC to get the lock.
//
// The IOMMU can be activated or not:
// 
// 1) When the IOMMU is used, a fixed size 2Mbytes vseg is allocated to 
// the IOC peripheral, in the I/O virtual space, and the user buffer is
// dynamically remapped in the IOMMU page table. The corresponding entry 
// in the IOMMU PT1 is defined by the kernel _ioc_iommu_ix1 variable.
// The number of pages to be unmapped is stored in the _ioc_npages variable.
// The number of PT2 entries is dynamically computed and stored in the
// kernel _ioc_iommu_npages variable. It cannot be larger than 512.
// The user buffer is unmapped by the _ioc_completed() function when 
// the transfer is completed.
//
// 2/ If the IOMMU is not used, we check that  the user buffer is mapped to a
// contiguous physical buffer (this is generally true because the user space
// page tables are statically constructed to use contiguous physical memory).
//
// Finally, the memory buffer must fulfill the following conditions:
// - The buffer must be word aligned, 
// - The buffer must be mapped in user space for an user access, 
// - The buffer must be writable in case of (to_mem) access,
// - The total number of physical pages occupied by the user buffer cannot
//   be larger than 512 pages if the IOMMU is activated,
// - All physical pages occupied by the user buffer must be contiguous
//   if the IOMMU is not activated.
// An error code is returned if these conditions are not verified.
///////////////////////////////////////////////////////////////////////////////////
// The seg_ioc_base virtual base addresses must be defined in giet_vsegs.ld file.
///////////////////////////////////////////////////////////////////////////////////

#include <giet_config.h>
#include <ioc_driver.h>
#include <utils.h>
#include <tty_driver.h>
#include <iob_driver.h>
#include <ctx_handler.h>
#include <mmc_driver.h>
#include <vmem.h>

#if !defined( USE_IOB )
# error: You must define USE_IOB in the hard_config.h file
#endif

#if !defined(GIET_USE_IOMMU) 
# error: You must define GIET_USE_IOMMU in the giet_config.h file
#endif

#define in_unckdata __attribute__((section (".unckdata")))

///////////////////// IOC global variables

in_unckdata unsigned int          _ioc_lock = 0;
in_unckdata volatile unsigned int _ioc_status = 0;
in_unckdata volatile unsigned int _ioc_gtid;
in_unckdata volatile unsigned int _ioc_iommu_ix1 = 0;
in_unckdata volatile unsigned int _ioc_iommu_npages; 

///////////////////////////////////////////////////////////////////////////////
//      _ioc_access()
// This function transfer data between a memory buffer and the block device.
// The buffer lentgth is (count*block_size) bytes.
// Arguments are:
// - to_mem     : from external storage to memory when non 0.
// - kernel     : kernel buffer with identity mapping when non 0. 
// - lba        : first block index on the external storage.
// - buf_vaddr  : virtual base address of the memory buffer.
// - count      : number of blocks to be transfered.
// Returns 0 if success, > 0 if error.
///////////////////////////////////////////////////////////////////////////////
static unsigned int _ioc_access( unsigned int to_mem,
                                 unsigned int mode,
                                 unsigned int lba,
                                 unsigned int buf_vaddr,
                                 unsigned int count) 
{

#if GIET_DEBUG_IOC_DRIVER
_tty_get_lock( 0 );
_puts("\n[IOC DEBUG] Enter _ioc_access() at cycle ");
_putd( _get_proctime() );
_puts(" for processor ");
_putd( _get_procid() );
_puts("\n - mode    = ");
_putd( mode );
_puts("\n - vaddr   = ");
_putx( buf_vaddr );
_puts("\n - sectors = ");
_putd( count );
_puts("\n - lba     = ");
_putx( lba );
_puts("\n");
_tty_release_lock( 0 );
#endif

    unsigned int error;            // return value
    unsigned int pt_vbase;         // page table vbase address
    unsigned int vpn_min;          // first virtuel page index covering buffer
    unsigned int vpn_max;          // last virtual page index covering buffer
    unsigned int vpn;              // current virtual page index
    unsigned int ppn;              // physical page number
    unsigned int flags;            // page protection flags
    unsigned int ix2;              // page index in IOMMU PT1 page table
    unsigned int ppn_first;        // first physical page number for user buffer
    unsigned int buf_xaddr = 0;    // user buffer virtual address in IO space (if IOMMU)
    paddr_t      buf_paddr = 0;    // user buffer physical address (if no IOMMU),

    // check buffer alignment
    if ((unsigned int) buf_vaddr & 0x3)
    {
        _tty_get_lock( 0 );
        _puts("\n[GIET ERROR] in _ioc_access() : buffer not word aligned\n");
        _tty_release_lock( 0 );
        return 1; 
    }

    unsigned int * ioc_address = (unsigned int *) &seg_ioc_base ;

    unsigned int length = count << 9;  // count * 512 bytes

    // computing target buffer physical address
    if ( mode == IOC_BOOT_PA_MODE )                // identity mapping
    {
        buf_paddr = (paddr_t)buf_vaddr;
    }
    else                                           // V2P translation
    {
        // get page table virtual address
        pt_vbase = _get_context_slot(CTX_PTAB_ID);
        vpn_min = buf_vaddr >> 12;
        vpn_max = (buf_vaddr + length - 1) >> 12;

        // loop on all virtual pages covering the user buffer
        for (vpn = vpn_min, ix2 = 0 ; 
             vpn <= vpn_max ; 
             vpn++, ix2++ ) 
        {
            // get ppn and flags for each vpn
            unsigned int ko = _v2p_translate( (page_table_t*)pt_vbase,
                                              vpn,
                                              &ppn,
                                              &flags);
            // check access rights
            if ( ko )
            {
                _tty_get_lock( 0 );
                _puts("\n[GIET ERROR] in _ioc_access() : buffer unmapped\n");
                _tty_release_lock( 0 );
                return 1; 
            }

            if ( (mode == IOC_USER_MODE) && ((flags & PTE_U) == 0) )
            {
                _tty_get_lock( 0 );
                _puts("\n[GIET ERROR] in _ioc_access() : buffer not user accessible\n");
                _tty_release_lock( 0 );
                return 1; 
            }

            if ( ((flags & PTE_W) == 0 ) && to_mem )
            {
                _tty_get_lock( 0 );
                _puts("\n[GIET ERROR] in _ioc_access() : buffer not writable\n");
                _tty_release_lock( 0 );
                return 1; 
            }

            // save first ppn value
            if (ix2 == 0) ppn_first = ppn;

#if GIET_USE_IOMMU 
 
            // check buffer length < 2 Mbytes
            if (ix2 > 511) // check buffer length < 2 Mbytes
            {
                _tty_get_lock( 0 );
                _puts("\n[GIET ERROR] in _ioc_access() : user buffer > 2 Mbytes\n");
                _tty_release_lock( 0 );
                return 1; 
            }
            // map the physical page in IOMMU page table
            _iommu_add_pte2( _ioc_iommu_ix1,    // PT1 index
                             ix2,               // PT2 index
                             ppn,               // Physical page number    
                             flags );           // Protection flags

            // compute user buffer virtual adress in IO space
            buf_xaddr = (_ioc_iommu_ix1) << 21 | (buf_vaddr & 0xFFF);

#else

            // check that physical pages are contiguous
            if ((ppn - ppn_first) != ix2) 
            {
                _tty_get_lock( 0 );
                _puts("[GIET ERROR] in _ioc_access() : split physical buffer\n");
                _tty_release_lock( 0 );
                return 1; 
            }

            // compute user buffer physical adress
            buf_paddr = (((paddr_t)ppn_first) << 12) | (buf_vaddr & 0xFFF);
#endif            

        } // end for vpn
    }

#if GIET_USE_IOMMU 

    // register the number of pages to be unmapped in IOMMU
    _ioc_iommu_npages = (vpn_max - vpn_min) + 1;

#endif

    if ( to_mem ) // memory write : invalidate data caches 
    {
        // L1 cache
        if ( GIET_NO_HARD_CC ) _dcache_buf_invalidate((void *) buf_vaddr, length);

        // L2 cache (only if IOB used)
        if ( USE_IOB ) _memc_inval( buf_paddr, length );
    }
    else         // memory read : update data caches
    {
        // L1 cache : nothing to do if L1 write-through

        // L2 cache (only if IOB used)
        if ( USE_IOB ) _memc_sync( buf_paddr, length );
    }

    // get the lock protecting IOC
    _get_lock(&_ioc_lock);

    // set the _ioc_status polling variable
    _ioc_status = BLOCK_DEVICE_BUSY;

#if GIET_DEBUG_IOC_DRIVER
_tty_get_lock( 0 );
_puts("\n[IOC DEBUG] _ioc_access() : configure IOC\n");
_puts(" - buf_paddr = ");
_putl( buf_paddr );
_puts("\n");
_puts(" - count     = ");
_putd( count );
_puts("\n");
_puts(" - lba       = ");
_putx( lba );
_puts("\n");
_tty_release_lock( 0 );
#endif

    // send command to IOC   
    if ( GIET_USE_IOMMU ) 
    {
        ioc_address[BLOCK_DEVICE_BUFFER] = buf_xaddr;
        ioc_address[BLOCK_DEVICE_COUNT]  = count;
        ioc_address[BLOCK_DEVICE_LBA]    = lba;
    }
    else
    {
        ioc_address[BLOCK_DEVICE_BUFFER]     = (unsigned int)buf_paddr;
        ioc_address[BLOCK_DEVICE_BUFFER_EXT] = (unsigned int)(buf_paddr>>32);
        ioc_address[BLOCK_DEVICE_COUNT]      = count;
        ioc_address[BLOCK_DEVICE_LBA]        = lba;
    }

    // There is two policies for transfer completion 
        // detection, depending on the mode argument:

    if ( (mode == IOC_BOOT_PA_MODE) ||    // We poll directly the IOC_STATUS register
         (mode == IOC_BOOT_VA_MODE) )     // as IRQs are masked.
    {
        // Launch transfert
        if (to_mem == 0) ioc_address[BLOCK_DEVICE_OP] = BLOCK_DEVICE_WRITE;
        else             ioc_address[BLOCK_DEVICE_OP] = BLOCK_DEVICE_READ;

        unsigned int status = _ioc_get_status();
        while( (status != BLOCK_DEVICE_READ_SUCCESS)  &&
               (status != BLOCK_DEVICE_READ_ERROR)    &&
               (status != BLOCK_DEVICE_WRITE_SUCCESS) &&
               (status != BLOCK_DEVICE_WRITE_ERROR) ) 
        {
            status = _ioc_get_status();

#if GIET_DEBUG_IOC_DRIVER
_tty_get_lock( 0 );
_puts("\n[IOC DEBUG] _ioc_access() : ... waiting on IOC_STATUS register ...\n");
_tty_release_lock( 0 );
#endif

        }
        // analyse status
        error = ( (status == BLOCK_DEVICE_READ_ERROR) ||
                  (status == BLOCK_DEVICE_WRITE_ERROR) );

        // release lock
        _release_lock(&_ioc_lock);      
    }
    else                           // in USER or KERNEL mode, we deschedule the task.
                                   // When the task is rescheduled by the ISR, we reset 
                                   // the _ioc_status variable, and release the lock
    {
        // We need a critical section, because we must reset the RUN bit
                // before to launch the transfer, and we want to avoid to be descheduled
                // between these two operations. 

        // Enter critical section
        _it_disable(); 
        
        // set _ioc_gtid and reset runnable 
        unsigned int ltid = _get_proc_task_id();
        unsigned int pid = _get_procid();
        _ioc_gtid = (pid<<16) + ltid;
        _set_task_slot( pid, ltid, CTX_RUN_ID, 0 );  
        
        // Launch transfert
        if (to_mem == 0) ioc_address[BLOCK_DEVICE_OP] = BLOCK_DEVICE_WRITE;
        else             ioc_address[BLOCK_DEVICE_OP] = BLOCK_DEVICE_READ;

        // Exit critical section 
        _it_enable();

        // deschedule task
        _ctx_switch();                      

        // analyse status
        error = ( (_ioc_status == BLOCK_DEVICE_READ_ERROR) ||
                  (_ioc_status == BLOCK_DEVICE_WRITE_ERROR) );

        // reset _ioc_status and release lock
        _ioc_status = BLOCK_DEVICE_IDLE; 
        _release_lock(&_ioc_lock);      
    }

#if GIET_DEBUG_IOC_DRIVER
_tty_get_lock( 0 );
_puts("\n[IOC DEBUG] _ioc_access completed at cycle ");
_putd( _get_proctime() );
_puts(" for processor ");
_putd( _get_procid() );
_puts(" : error = ");
_putd( (unsigned int)error );
_puts("\n");
_tty_release_lock( 0 );
#endif

    return error;
} // end _ioc_access()

///////////////////////////////////////////////////////////////////////////////
//       _ioc_init()
// This function cheks block size, and activates the IOC interrupts.
// Return 0 for success.
///////////////////////////////////////////////////////////////////////////////
unsigned int _ioc_init()
{
    unsigned int * ioc_address = (unsigned int *) &seg_ioc_base ;
    
    if ( ioc_address[BLOCK_DEVICE_BLOCK_SIZE] != 512 )
    {
        _puts("\n[GIET ERROR] in _ioc_init() : block size must be 512 bytes\n");
        _exit();
    }

    ioc_address[BLOCK_DEVICE_IRQ_ENABLE] = 1;
    return 0;
}

///////////////////////////////////////////////////////////////////////////////
//     _ioc_read()
// Transfer data from the block device to a memory buffer. 
// - mode     : BOOT / KERNEL / USER
// - lba      : first block index on the block device
// - buffer   : base address of the memory buffer (must be word aligned)
// - count    : number of blocks to be transfered.
// Returns 0 if success, > 0 if error.
///////////////////////////////////////////////////////////////////////////////
unsigned int _ioc_read( unsigned int mode,  
                        unsigned int lba, 
                        void*        buffer, 
                        unsigned int count) 
{
    return _ioc_access( 1,        // read access
                        mode,  
                        lba,
                        (unsigned int) buffer,
                        count );
}

///////////////////////////////////////////////////////////////////////////////
//     _ioc_write()
// Transfer data from a memory buffer to the block device. 
// - mode     : BOOT / KERNEL / USER
// - lba      : first block index on the block device
// - buffer   : base address of the memory buffer (must be word aligned)
// - count    : number of blocks to be transfered.
// Returns 0 if success, > 0 if error.
///////////////////////////////////////////////////////////////////////////////
unsigned int _ioc_write( unsigned int mode,  
                         unsigned int lba, 
                         const void*  buffer, 
                         unsigned int count ) 
{
    return _ioc_access( 0,        // write access
                        mode,  
                        lba,
                        (unsigned int) buffer,
                        count );
}

///////////////////////////////////////////////////////////////////////////////
//     _ioc_get_status()
// This function returns the transfert status, and acknowledge the IRQ
// if the IOC controler is not busy.
///////////////////////////////////////////////////////////////////////////////
unsigned int _ioc_get_status() 
{
    // get IOC base address
    unsigned int * ioc_address = (unsigned int *) &seg_ioc_base;

    return ioc_address[BLOCK_DEVICE_STATUS];
}

///////////////////////////////////////////////////////////////////////////////
//     _ioc_get_block_size()
// This function returns the block_size with which the IOC has been configured.
///////////////////////////////////////////////////////////////////////////////
unsigned int _ioc_get_block_size() 
{
    // get IOC base address
    unsigned int * ioc_address = (unsigned int *) &seg_ioc_base;
    
    return  ioc_address[BLOCK_DEVICE_BLOCK_SIZE];
}


// Local Variables:
// tab-width: 4
// c-basic-offset: 4
// c-file-offsets:((innamespace . 0)(inline-open . 0))
// indent-tabs-mode: nil
// End:
// vim: filetype=c:expandtab:shiftwidth=4:tabstop=4:softtabstop=4