source: trunk/hal/tsar_mips32/drivers/soclib_bdv.c @ 678

/*
 * soclib_bdv.c - soclib simple block device driver implementation.
 *
 * Author     Alain Greiner (2016,2017,2018,2019,2020)
 *
 * Copyright (c) UPMC Sorbonne Universites
 *
 * This file is part of ALMOS-MKH.
 *
 * ALMOS-MKH.is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.0 of the License.
 *
 * ALMOS-MKH.is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with ALMOS-MKH.; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <soclib_bdv.h>
#include <hal_kernel_types.h>
#include <chdev.h>
#include <dev_ioc.h>
#include <printk.h>
#include <thread.h>
#include <hal_irqmask.h>

///////////////////////////////////////
void soclib_bdv_init( chdev_t * chdev )
{
    // set driver specific fields in IOC chdev
    chdev->cmd = &soclib_bdv_cmd;
    chdev->isr = &soclib_bdv_isr;

    // get extended pointer on SOCLIB_BDV peripheral base 
        xptr_t  base_xp = chdev->base;

    // get hardware device cluster and local pointer
    cxy_t      base_cxy  = GET_CXY( base_xp );
    uint32_t * base_ptr  = GET_PTR( base_xp );

    // get block_size and block_count  
        uint32_t block_size  = hal_remote_l32( XPTR( base_cxy , base_ptr + BDV_BLOCK_SIZE_REG ) );
        uint32_t block_count = hal_remote_l32( XPTR( base_cxy , base_ptr + BDV_SIZE_REG ) );

    // set IOC chdev extension fields
    chdev->ext.ioc.size  = block_size;
    chdev->ext.ioc.count = block_count;

} // end soclib_bdv_init()


//////////////////////////////////////////////////////////////
void __attribute__ ((noinline)) soclib_bdv_cmd( xptr_t th_xp )
{
    uint32_t   cmd_type;    // IOC_READ / IOC_WRITE / IOC_SYNC_READ / IOC_SYNC_WRITE 
    uint32_t   lba;
    uint32_t   count;
    xptr_t     buf_xp;
    xptr_t     ioc_xp;
    uint32_t   status;      // I/0 operation status (from BDV)
    reg_t      save_sr;     // for critical section
    uint32_t   op;          // BDV_OP_READ / BDV_OP_WRITE 

    // get client thread cluster and local pointer
    cxy_t      th_cxy = GET_CXY( th_xp );
    thread_t * th_ptr = GET_PTR( th_xp );

#if (DEBUG_HAL_IOC_RX || DEBUG_HAL_IOC_TX)
uint32_t    cycle        = (uint32_t)hal_get_cycles();
thread_t  * this         = CURRENT_THREAD;
process_t * process      = hal_remote_lpt( XPTR( th_cxy , &th_ptr->process ) );
pid_t       client_pid   = hal_remote_l32( XPTR( th_cxy , &process->pid ) );
trdid_t     client_trdid = hal_remote_l32( XPTR( th_cxy , &th_ptr->trdid ) );
#endif 

    // get command arguments and extended pointer on IOC device
    cmd_type =         hal_remote_l32( XPTR( th_cxy , &th_ptr->ioc_cmd.type   ) );
    lba      =         hal_remote_l32( XPTR( th_cxy , &th_ptr->ioc_cmd.lba    ) );
    count    =         hal_remote_l32( XPTR( th_cxy , &th_ptr->ioc_cmd.count  ) );
    buf_xp   = (xptr_t)hal_remote_l64( XPTR( th_cxy , &th_ptr->ioc_cmd.buf_xp ) );
    ioc_xp   = (xptr_t)hal_remote_l64( XPTR( th_cxy , &th_ptr->ioc_cmd.dev_xp ) );

    // decode command
    if     ( (cmd_type == IOC_READ)  || (cmd_type == IOC_SYNC_READ)  ) op = BDV_OP_READ;
    else if( (cmd_type == IOC_WRITE) || (cmd_type == IOC_SYNC_WRITE) ) op = BDV_OP_WRITE;
    else     assert( false , "illegal command" );

    // get cluster and local pointer on IOC chdev
    cxy_t      ioc_cxy = GET_CXY( ioc_xp );
    chdev_t  * ioc_ptr = GET_PTR( ioc_xp );

    // get cluster and pointers for SOCLIB-BDV peripheral segment base
    xptr_t     seg_xp  = (xptr_t)hal_remote_l64( XPTR( ioc_cxy , &ioc_ptr->base ) );
    cxy_t      seg_cxy = GET_CXY( seg_xp );
    uint32_t * seg_ptr = GET_PTR( seg_xp );

    // split buffer address in two 32 bits words
    uint32_t   buf_lsb = (uint32_t)(buf_xp);
    uint32_t   buf_msb = (uint32_t)(buf_xp>>32);

#if DEBUG_HAL_IOC_RX 
if( DEBUG_HAL_IOC_RX < cycle )
printk("\n[%s] thread[%x,%x] enters / client[%x,%x] / cmd %d / lba %x / buf(%x,%x) / cycle %d\n",
__FUNCTION__ , this->process->pid, this->trdid, client_pid, client_trdid,
cmd_type, lba, buf_msb, buf_lsb, cycle );
#endif

#if DEBUG_HAL_IOC_TX
if( DEBUG_HAL_IOC_TX < cycle )
printk("\n[%s] thread[%x,%x] enters / client[%x,%x] / cmd %d / lba %x / buf(%x,%x) / cycle %d\n",
__FUNCTION__ , this->process->pid, this->trdid, client_pid, client_trdid,
cmd_type, lba, buf_msb, buf_lsb, cycle );
#endif

    // select operation
    if( (cmd_type == IOC_READ) || (cmd_type == IOC_SYNC_READ) ) op = BDV_OP_READ; 
    else                                                        op = BDV_OP_WRITE;

    // set SOCLIB_BDV registers to configure the I/O operation
    hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_IRQ_ENABLE_REG ) , 1       );
    hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_BUFFER_REG     ) , buf_lsb ); 
    hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_BUFFER_EXT_REG ) , buf_msb ); 
    hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_LBA_REG        ) , lba     );
    hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_COUNT_REG      ) , count   );

    // waiting policy  depends on the command type
    // - for IOC_READ / IOC_WRITE commands, this function is called by the server thread
    //   => block and deschedule after launching the I/O transfer. 
    //   The I/O operation status is reported in the command by the ISR, and the
    //   server thread is re-activated by the ISR.
    // - for IOC_SYNC_READ / IOC_SYNC_WRITE, this function is called by the client thread
    //   => mask the IOC IRQ and poll the BDV status register until I/O transfer completion,
    //   and report status in the command.

    if( (cmd_type == IOC_SYNC_READ) || (cmd_type == IOC_SYNC_WRITE) )  // polling policy
    {
        // get core handling the IOC IRQ
        thread_t * server = (thread_t *)hal_remote_lpt( XPTR( ioc_cxy , &ioc_ptr->server ) );
        core_t   * core   = (core_t *)hal_remote_lpt( XPTR( ioc_cxy , &server->core ) );
        lid_t      lid    = (lid_t)hal_remote_l32( XPTR( ioc_cxy , &core->lid ) );

        // mask the IOC IRQ
        dev_pic_disable_irq( lid , ioc_xp );

        // launch I/O operation on BDV device
        hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_OP_REG ) , op );
        
        // wait completion 
        while (1)
        {
            status = hal_remote_l32( XPTR( seg_cxy , seg_ptr + BDV_STATUS_REG ) );

            if( (status == BDV_READ_SUCCESS) ||
                (status == BDV_WRITE_SUCCESS) ) // successfully completed
            {
                // unmask IOC IRQ
                dev_pic_enable_irq( lid , ioc_xp );

                // report success in command
                hal_remote_s32( XPTR( th_cxy , &th_ptr->ioc_cmd.error ) , 0 );

#if DEBUG_HAL_IOC_RX
cycle = (uint32_t)hal_get_cycles();
if( (DEBUG_HAL_IOC_RX < cycle) && (cmd_type == IOC_SYNC_READ) )
printk("\n[%s] thread[%x,%x] SYNC_READ success for client thread[%x,%x] / cycle %d\n",
__FUNCTION__, this->process->pid, this->trdid, client_pid, client_trdid, cycle );
#endif

#if DEBUG_HAL_IOC_TX
cycle = (uint32_t)hal_get_cycles();
if( (DEBUG_HAL_IOC_TX < cycle) && (cmd_type == IOC_SYNC_WRITE) )
printk("\n[%s] thread[%x,%x] SYNC_WRITE success for client thread[%x,%x] / cycle %d\n",
__FUNCTION__, this->process->pid, this->trdid, client_pid, client_trdid, cycle );
#endif
                break;
            }
            else if( status == BDV_BUSY )      // non completed
            {
                continue;
            }
            else                               // error reported
            {
                // unmask IOC IRQ
                dev_pic_enable_irq( lid , ioc_xp );

                // report failure in command
                hal_remote_s32( XPTR( th_cxy , &th_ptr->ioc_cmd.error ) , 1 );

#if DEBUG_HAL_IOC_RX
cycle = (uint32_t)hal_get_cycles();
if( (DEBUG_HAL_IOC_RX < cycle) && (cmd_type == IOC_SYNC_READ) )
printk("\n[%s] thread[%x,%x] SYNC_READ failure for client thread[%x,%x] / cycle %d\n",
__FUNCTION__, this->process->pid, this->trdid, client_pid, client_trdid, cycle );
#endif

#if DEBUG_HAL_IOC_TX
cycle = (uint32_t)hal_get_cycles();
if( (DEBUG_HAL_IOC_TX < cycle) && (cmd_type == IOC_SYNC_WRITE) )
printk("\n[%s] thread[%x,%x] SYNC_WRITE failure for client thread[%x,%x] / cycle %d\n",
__FUNCTION__, this->process->pid, this->trdid, client_pid, client_trdid, cycle );
#endif
                break;
            }
        }
    }
    else                                                     // descheduling + IRQ policy
    { 
        // enter critical section to atomically
        // lauch I/O operation and deschedule  
        hal_disable_irq( &save_sr );

        // launch I/O operation on BDV device
        hal_remote_s32( XPTR( seg_cxy , seg_ptr + BDV_OP_REG ) , op );
        
        // server thread blocks on ISR
        thread_block( XPTR( local_cxy , CURRENT_THREAD ) , THREAD_BLOCKED_ISR );

#if DEBUG_HAL_IOC_RX 
if( (DEBUG_HAL_IOC_RX < cycle) && (cmd_type != IOC_WRITE ) )
printk("\n[%s] thread[%x,%x] blocks & deschedules after lauching READ transfer\n",
__FUNCTION__ , this->process->pid, this->trdid );
#endif

#if DEBUG_HAL_IOC_TX
if( (DEBUG_HAL_IOC_TX < cycle) && (cmd_type == IOC_WRITE) )
printk("\n[%s] thread[%x,%x] blocks & deschedules after lauching WRITE transfer\n",
__FUNCTION__ , this->process->pid, this->trdid );
#endif
        // server thread deschedules
        sched_yield("blocked on ISR");

        // exit critical section
        hal_restore_irq( save_sr );
    
#if DEBUG_HAL_IOC_RX
cycle = (uint32_t)hal_get_cycles();
if( (DEBUG_HAL_IOC_RX < cycle) && (cmd_type != IOC_WRITE) )
printk("\n[%s] thread[%x,%x] exit after READ for client thread[%x,%x] / cycle %d\n",
__FUNCTION__, this->process->pid, this->trdid, client_pid, client_trdid, cycle );
#endif

#if DEBUG_HAL_IOC_TX
cycle = (uint32_t)hal_get_cycles();
if( (DEBUG_HAL_IOC_TX < cycle) && (cmd_type == IOC_WRITE) )
printk("\n[%s] thread[%x,%x] exit after WRITE for client thread[%x,%x] / cycle %d\n",
__FUNCTION__, this->process->pid, this->trdid, client_pid, client_trdid, cycle );
#endif

    }

} // end soclib_bdv_cmd()


/////////////////////////////////////////////////////////////////
void __attribute__ ((noinline)) soclib_bdv_isr( chdev_t * chdev )
{
    error_t  error = 0;

    // get extended pointer on server thread
    xptr_t server_xp = XPTR( local_cxy , chdev->server );

    // get extended pointer on client thread
    xptr_t root      = XPTR( local_cxy , &chdev->wait_root );
    xptr_t client_xp = XLIST_FIRST( root , thread_t , wait_list );

    // get client thread cluster and local pointer
    cxy_t      client_cxy = GET_CXY( client_xp );
    thread_t * client_ptr = GET_PTR( client_xp );

    // get command type
    uint32_t   cmd_type = hal_remote_l32( XPTR( client_cxy , &client_ptr->ioc_cmd.type ) );
    
#if (DEBUG_HAL_IOC_RX || DEBUG_HAL_IOC_TX)
uint32_t    cycle        = (uint32_t)hal_get_cycles();
process_t * process      = hal_remote_lpt( XPTR( client_cxy , &client_ptr->process ) );
pid_t       client_pid   = hal_remote_l32( XPTR( client_cxy , &process->pid ) );
trdid_t     client_trdid = hal_remote_l32( XPTR( client_cxy , &client_ptr->trdid ) );
thread_t  * server       = GET_PTR( server_xp );
pid_t       server_pid   = server->process->pid;
trdid_t     server_trdid = server->trdid;
#endif

    // get SOCLIB_BDV device cluster and local pointer
    cxy_t      bdv_cxy  = GET_CXY( chdev->base );
    uint32_t * bdv_ptr  = GET_PTR( chdev->base );

    // get BDV status register and acknowledge IRQ
        uint32_t status = hal_remote_l32( XPTR( bdv_cxy , bdv_ptr + BDV_STATUS_REG ) );   

    if( cmd_type == IOC_READ )
    {
            error = (status != BDV_READ_SUCCESS);

#if DEBUG_HAL_IOC_RX
if( DEBUG_HAL_IOC_RX < cycle )
printk("\n[%s] RX transfer completed for client[%x,%x] / server[%x,%x] / cycle %d\n",
__FUNCTION__, client_pid, client_trdid, server_pid, server_trdid, cycle );
#endif

    }
        else if( cmd_type == IOC_WRITE )
    {
            error = (status != BDV_WRITE_SUCCESS);

#if DEBUG_HAL_IOC_TX
if( DEBUG_HAL_IOC_TX < cycle )
printk("\n[%s] TX transfer completed for client[%x,%x] / server[%x,%x] / cycle %d\n",
__FUNCTION__, client_pid, client_trdid, server_pid, server_trdid, cycle );
#endif

    }
    else
    {
        assert( false , "illegal command %d", cmd_type );
    } 

    // set operation status in command
    hal_remote_s32( XPTR( client_cxy , &client_ptr->ioc_cmd.error ) , error );

    // unblock server thread 
    thread_unblock( server_xp , THREAD_BLOCKED_ISR );

} // end soclib_bdv_isr()