source: trunk/hal/tsar_mips32/drivers/soclib_pic.c @ 650

/*
 * soclib_pic.c - soclib PIC driver implementation.
 *
 * Author  Alain Greiner (2016,2017,2018,2019)
 *
 * Copyright (c) UPMC Sorbonne Universites
 *
 * This file is part of ALMOS-MKH.
 *
 * ALMOS-MKH is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.0 of the License.
 *
 * ALMOS-MKH is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with ALMOS-MKH; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <hal_kernel_types.h>
#include <chdev.h>
#include <soclib_pic.h>
#include <errno.h>
#include <string.h>
#include <bits.h>
#include <vfs.h>
#include <rpc.h>
#include <cluster.h>
#include <printk.h>
#include <core.h>
#include <thread.h>

//////////////////////////////////////////////////////////////////////////////////////
//         Extern variables
//////////////////////////////////////////////////////////////////////////////////////

extern  chdev_directory_t chdev_dir;    // defined in chdev.h / allocated in kerneL-init.c

extern  iopic_input_t  iopic_input;  // defined in dev_pic.h / allocated in kernel_init.c
extern  lapic_input_t  lapic_input;  // defined in dev_pic.h / allocated in kernel_init.c
 


//////////////////////////////////////////////////////////////////////////////////////
//        SOCLIB PIC private functions 
//////////////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////
uint32_t soclib_pic_wti_alloc( void )
{
    uint32_t index;

    // get pointer on cluster extension for SOCLIB PIC (XCU descriptor)
    soclib_pic_cluster_t * ext_ptr = LOCAL_CLUSTER->pic_extend;

    assert( (ext_ptr->first_free_wti < ext_ptr->wti_nr) ,
            "no free WTI found : too much external IRQs\n");

    // update WTI allocator
    index = ext_ptr->first_free_wti;
    ext_ptr->first_free_wti++;

    return index;

}  // end soclib_pic_wti_alloc()

/////////////////////////////////////////////
inline uint32_t * soclib_pic_xcu_base( void )
{
    return ((soclib_pic_cluster_t *)(LOCAL_CLUSTER->pic_extend))->xcu_base;
}

/////////////////////////////////////////////////////////
inline uint32_t * soclib_pic_remote_xcu_base( cxy_t cxy )
{
    soclib_pic_cluster_t * extend;

    // get extended pointer on PIC extension in remote cluster 
    extend = hal_remote_lpt( XPTR( cxy , &cluster_manager.pic_extend ) );

        return (uint32_t *)hal_remote_lpt( XPTR( cxy , &extend->xcu_base ) );
                 
}

///////////////////////////////////////////
void soclib_pic_xcu_status( lid_t      lid,
                            uint32_t * hwi_status,
                            uint32_t * wti_status,
                            uint32_t * pti_status )
{
    // get local XCU segment base 
        uint32_t * base = soclib_pic_xcu_base();

    // read PRIO register
    // in TSAR : XCU output [4*lid] is connected to core [lid]
        uint32_t prio = base[ (XCU_PRIO << 5) | (lid<<2) ];

    *wti_status = (prio & 0x4) ? (((prio >> 24) & 0x1F) + 1) : 0;
    *hwi_status = (prio & 0x2) ? (((prio >> 16) & 0x1F) + 1) : 0;
    *pti_status = (prio & 0x1) ? (((prio >>  8) & 0x1F) + 1) : 0;

}

////////////////////////////////////////////////////
inline uint32_t soclib_pic_xcu_ack( uint32_t * reg )
{
    return *reg;
}

///////////////////////////////////
void soclib_pic_irq_handler( void )
{
    uint32_t   hwi_status;   // HWI index + 1  / no pending HWI if 0
    uint32_t   wti_status;   // WTI index + 1  / no pending WTI if 0
    uint32_t   pti_status;   // PTI index + 1  / no pending PTI if 0
    chdev_t  * src_chdev;    // pointer on source chdev descriptor
    uint32_t   index;        // WTI / HWI / PTI index

    uint32_t * xcu_base = soclib_pic_xcu_base();

    core_t   * core = CURRENT_THREAD->core;

    // get XCU status
    soclib_pic_xcu_status( core->lid,
                           &hwi_status,
                           &wti_status,
                           &pti_status );

#if DEBUG_HAL_IRQS
uint32_t cycle = (uint32_t)hal_get_cycles();
if (DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : core[%x,%d] enter / WTI = %x / HWI = %x / PTI = %x / cycle %d\n",
__FUNCTION__ , local_cxy , core->lid , wti_status , hwi_status , pti_status, cycle );
#endif

    // analyse status and handle up to 3 pending IRQs (one WTI, one HWI, one PTI)

    if( wti_status )          // pending WTI 
        {
        index = wti_status - 1;

        ////////////////////////////////////////////////////////
        if( index < LOCAL_CLUSTER->cores_nr )   // it is an IPI
        {
            assert( (index == core->lid) , "illegal IPI index" );

#if DEBUG_HAL_IRQS
if (DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : core[%x,%d] handling IPI\n", __FUNCTION__ , local_cxy , core->lid );
#endif
            // acknowledge IRQ (this require an XCU read)
            uint32_t   ack  = xcu_base[(XCU_WTI_REG << 5) | core->lid];

            // check RPC FIFO,  and activate or create a RPC thread
            // condition is always true, but we use the ack value 
            // to avoid a GCC warning
            if( ack + 1 ) sched_yield("IPI received");
        }
        ////////////////////////////////////////////////////////////////
        else                                    // it is an external IRQ
        {
            // get pointer on source chdev
            src_chdev = ((soclib_pic_core_t *)core->pic_extend)->wti_vector[index];

                    if( src_chdev == NULL )        // strange, but not fatal
                    {
                printk("\n[WARNING] in %s : no handler for WTI %d on core %d in cluster %x\n",
                       __FUNCTION__ , index , core->lid , local_cxy );

                    core->spurious_irqs ++;

                // disable WTI in local XCU controller 
                xcu_base[(XCU_MSK_WTI_DISABLE << 5) | core->lid] = 1 << core->lid;

                hal_fence();
            }
            else                                 // call relevant ISR
            {

#if DEBUG_HAL_IRQS
if (DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : core[%x,%d] handling external WTI %d\n",
__FUNCTION__ , local_cxy , core->lid , index );
#endif
                // call ISR
                    src_chdev->isr( src_chdev );
            }
        }
        }

    /////////////////////////////////////////////////////////////
        if( hwi_status )                     // It is an Internal IRQ
        {
        index = hwi_status - 1;

        // get pointer on source chdev
        src_chdev = ((soclib_pic_core_t *)core->pic_extend)->hwi_vector[index];

                if( src_chdev == NULL )        // strange, but not fatal
                {
            printk("\n[WARNING] in %s : no handler for HWI %d on core %d in cluster %x\n",
                   __FUNCTION__ , index , core->lid , local_cxy );

                core->spurious_irqs ++;

            // disable HWI in local XCU controller
            xcu_base[(XCU_MSK_HWI_DISABLE << 5) | core->lid] = 1 << core->lid;

            hal_fence();
                }
        else                    // call relevant ISR
        {

#if DEBUG_HAL_IRQS
if (DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : core[%x,%d] handling HWI %d\n",
__FUNCTION__ , local_cxy , core->lid , index );
#endif
            // call ISR
                    src_chdev->isr( src_chdev );
        }
        }
    ///////////////////////////////////////////////////////
    if( pti_status )                   // It is a Timer IRQ
        {
        index = pti_status - 1;

        assert( (index == core->lid) , "unconsistent PTI index\n");

#if DEBUG_HAL_IRQS
if (DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : core[%x,%d] handling PTI %d\n",
__FUNCTION__ , core->lid , local_cxy , index );
#endif
        // acknowledge IRQ (this require a read access to XCU)
        uint32_t   ack  = xcu_base[(XCU_PTI_ACK << 5) | core->lid];

        // execute all actions related to TICK event
        // condition is always true, but we use the ack value
        // to avoid a GCC warning
        if( ack + 1 ) core_clock( core );
        }
}  // end soclib_pic_irq_handler()




//////////////////////////////////////////////////////////////////////////////////////
//         SOCLIC PIC device  generic API
//////////////////////////////////////////////////////////////////////////////////////

/////////////////////////////////////
void soclib_pic_init( chdev_t * pic )
{
    uint32_t    i;      // for loop on IOPIC inputs

    // get IOPIC controller cluster and segment base pointer
    cxy_t      iopic_seg_cxy = GET_CXY( pic->base );
    uint32_t * iopic_seg_ptr = GET_PTR( pic->base );

    // reset the IOPIC component registers : disable all input IRQs
    for( i = 0 ; i < CONFIG_MAX_EXTERNAL_IRQS ; i++ )
    {
        xptr_t iopic_seg_xp = XPTR( iopic_seg_cxy,
                                    iopic_seg_ptr + i*IOPIC_SPAN + IOPIC_MASK ); 
        hal_remote_s32( iopic_seg_xp , 0 ); 
    }

}  // end soclib_pic_init()

//////////////////////////////////////////////////
void soclib_pic_extend_init( uint32_t * xcu_base )
{
    soclib_pic_cluster_t * cluster_ext_ptr;   
    soclib_pic_core_t    * core_ext_ptr;
    kmem_req_t             req;
    uint32_t               lid;
    uint32_t               idx;

    cluster_t            * cluster = LOCAL_CLUSTER;

    // create core extension for all cores in cluster
    for( lid = 0 ; lid < cluster->cores_nr ; lid++ )
    {
        // allocate memory for core extension
        req.type     = KMEM_KCM;
        req.order    = bits_log2( sizeof(soclib_pic_core_t) );
        req.flags    = AF_KERNEL;
        core_ext_ptr = kmem_alloc( &req );

        if( core_ext_ptr == NULL )
        {
            printk("\n[PANIC] in %s : cannot allocate memory for core extension\n",
            __FUNCTION__ );
        }
    
        // reset the HWI / WTI  interrupt vectors
        for( idx = 0 ; idx < SOCLIB_MAX_HWI ; idx++ ) core_ext_ptr->hwi_vector[idx] = NULL;
        for( idx = 0 ; idx < SOCLIB_MAX_WTI ; idx++ ) core_ext_ptr->wti_vector[idx] = NULL;

        // register PIC extension in core descriptor
        cluster->core_tbl[lid].pic_extend = core_ext_ptr;
    }

    // allocate memory for cluster extension
    req.type        = KMEM_KCM;
    req.order       = bits_log2( sizeof(soclib_pic_cluster_t) );
    req.flags       = AF_KERNEL;
    cluster_ext_ptr = kmem_alloc( &req );

    if( cluster_ext_ptr == NULL )
    {
        printk("\n[PANIC] in %s : cannot allocate memory for cluster extension\n",
        __FUNCTION__ );
    }
    
assert( (cluster_ext_ptr != NULL) , "cannot allocate memory for cluster extension");

    // get XCU characteristics from the XCU config register 
    uint32_t  config = xcu_base[XCU_CONFIG<<5];
    uint32_t  wti_nr = (config >> 16) & 0xFF; 
    uint32_t  hwi_nr = (config >> 8 ) & 0xFF; 
    uint32_t  pti_nr = (config      ) & 0xFF; 

    // initialize the cluster extension
    // The first WTI slots are for IPIs (one slot per core)
    cluster_ext_ptr->xcu_base       = xcu_base;
    cluster_ext_ptr->hwi_nr         = hwi_nr;
    cluster_ext_ptr->wti_nr         = wti_nr;
    cluster_ext_ptr->pti_nr         = pti_nr;
    cluster_ext_ptr->first_free_wti = cluster->cores_nr;

    // register PIC extension in cluster manager
    cluster->pic_extend = cluster_ext_ptr;

    // reset the XCU component registers 
    // mask all HWIs, all WTIs, and all PTIs, for all cores in local cluster   
    for( lid = 0 ; lid < cluster->cores_nr ; lid++ )
    {
        xcu_base[XCU_MSK_HWI_DISABLE << 5 | lid] = 0xFFFFFFFF;
        xcu_base[XCU_MSK_WTI_DISABLE << 5 | lid] = 0xFFFFFFFF;
        xcu_base[XCU_MSK_PTI_DISABLE << 5 | lid] = 0xFFFFFFFF;
    }

}  // end soclib_pic_extend_init()

////////////////////////////////////////
void soclib_pic_bind_irq( lid_t     lid,
                          chdev_t * src_chdev )
{

#if DEBUG_HAL_IRQS
uint32_t cycle = (uint32_t)hal_get_cycles();
if( DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : thread %x enter for core[%x,%d] / cycle %d\n",
__FUNCTION__ , CURRENT_THREAD , local_cxy , lid , cycle );
#endif

    // get extended & local pointers on PIC chdev descriptor
    xptr_t     pic_xp  = chdev_dir.pic;
    cxy_t      pic_cxy = GET_CXY( pic_xp );
    chdev_t *  pic_ptr = (chdev_t *)GET_PTR( pic_xp );

    // get extended and local pointers on IOPIC  segment base
    xptr_t     seg_pic_xp  = hal_remote_l64( XPTR( pic_cxy , &pic_ptr->base ) );
    cxy_t      seg_pic_cxy = GET_CXY( seg_pic_xp );
    uint32_t * seg_pic_ptr = (uint32_t *)GET_PTR( seg_pic_xp );

    // get local pointer on XCU segment base
    uint32_t * seg_xcu_ptr = soclib_pic_xcu_base();

    // get the source chdev functionnal type, channel, and direction
    uint32_t func    = src_chdev->func;
    uint32_t impl    = src_chdev->impl;
    uint32_t channel = src_chdev->channel;
    bool_t   is_rx   = src_chdev->is_rx;

    if( (func == DEV_FUNC_IOC && impl == IMPL_IOC_BDV) || (func == DEV_FUNC_NIC) ||
        (func == DEV_FUNC_TXT && impl == IMPL_TXT_TTY) || (func == DEV_FUNC_IOB) )          // external IRQ => WTI
    {
        // get external IRQ index
        uint32_t  hwi_id;   
        if     (  func == DEV_FUNC_IOC            ) hwi_id = iopic_input.ioc[channel];
        else if(  func == DEV_FUNC_TXT  &&  is_rx ) hwi_id = iopic_input.txt_rx[channel];
        else if(  func == DEV_FUNC_TXT  && !is_rx ) hwi_id = iopic_input.txt_tx[channel];
        else if( (func == DEV_FUNC_NIC) &&  is_rx ) hwi_id = iopic_input.nic_rx[channel];
        else if( (func == DEV_FUNC_NIC) && !is_rx ) hwi_id = iopic_input.nic_tx[channel];
        else if(  func == DEV_FUNC_IOB            ) hwi_id = iopic_input.iob;
        else      assert( false , "illegal device functionnal type\n");

        // get a WTI mailbox from local XCU descriptor  
        uint32_t wti_id = soclib_pic_wti_alloc();

        // register IRQ type and index in chdev
        src_chdev->irq_type = SOCLIB_TYPE_WTI;
        src_chdev->irq_id   = wti_id;

        // compute extended pointer on WTI mailbox in local XCU
        xptr_t wti_xp = XPTR( local_cxy , &seg_xcu_ptr[(XCU_WTI_REG << 5) | wti_id] );

            // set the IOPIC_ADDRESS and IOPIC_EXTEND registers in IOPIC
        uint32_t lsb_wdata = (uint32_t)wti_xp;
        uint32_t msb_wdata = (uint32_t)(wti_xp >> 32);
        xptr_t   lsb_xp = XPTR( seg_pic_cxy , seg_pic_ptr+hwi_id*IOPIC_SPAN+IOPIC_ADDRESS );
        xptr_t   msb_xp = XPTR( seg_pic_cxy , seg_pic_ptr+hwi_id*IOPIC_SPAN+IOPIC_EXTEND );
        hal_remote_s32( lsb_xp , lsb_wdata );
        hal_remote_s32( msb_xp , msb_wdata );

        // enable IRQ in IOPIC
        hal_remote_s32( XPTR( seg_pic_cxy , seg_pic_ptr+hwi_id*IOPIC_SPAN+IOPIC_MASK ), 1 );

        // update the WTI interrupt vector for core[lid]
        core_t * core = &LOCAL_CLUSTER->core_tbl[lid];
        ((soclib_pic_core_t *)core->pic_extend)->wti_vector[wti_id] = src_chdev;

#if DEBUG_HAL_IRQS
if( DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : %s / channel = %d / rx = %d / hwi_id = %d / wti_id = %d / cluster = %x\n",
__FUNCTION__ , chdev_func_str( func ) , channel , is_rx , hwi_id , wti_id , local_cxy );
#endif

    }
    else if( (func == DEV_FUNC_DMA) || (func == DEV_FUNC_MMC) ||
             (func == DEV_FUNC_TXT && impl == IMPL_TXT_MTY) ||
             (func == DEV_FUNC_IOC && impl == IMPL_IOC_SPI) )   // internal IRQ => HWI
    {
        // get internal IRQ index
        uint32_t hwi_id;
        if( func == DEV_FUNC_DMA ) hwi_id = lapic_input.dma[channel];
        else if (func == DEV_FUNC_TXT ) hwi_id = lapic_input.mtty;
        else if (func == DEV_FUNC_IOC ) hwi_id = lapic_input.sdcard;
        else                       hwi_id = lapic_input.mmc;

        // register IRQ type and index in chdev
        src_chdev->irq_type = SOCLIB_TYPE_HWI;
        src_chdev->irq_id   = hwi_id;

        // update the HWI interrupt vector for core[lid]
        core_t * core = &LOCAL_CLUSTER->core_tbl[lid];
        ((soclib_pic_core_t *)core->pic_extend)->hwi_vector[hwi_id] = src_chdev;

#if DEBUG_HAL_IRQS
if( DEBUG_HAL_IRQS < cycle )
printk("\n[DBG] %s : %s / channel = %d / hwi_id = %d / cluster = %x\n",
__FUNCTION__ , chdev_func_str( func ) , channel , hwi_id , local_cxy );
#endif

    }
    else
    {
        assert( false , "illegal device functionnal type\n" );
    } 
}  // end soclib_pic_bind_irq();

///////////////////////////////////////
void soclib_pic_enable_irq( lid_t  lid,
                            xptr_t src_chdev_xp )
{
    // get cluster and local pointer on remote src_chdev
    cxy_t     src_chdev_cxy = GET_CXY( src_chdev_xp );
    chdev_t * src_chdev_ptr = (chdev_t *)GET_PTR( src_chdev_xp );

    // get local pointer on remote XCU segment base
    uint32_t * seg_xcu_ptr = soclib_pic_remote_xcu_base( src_chdev_cxy );

    // get the source chdev IRQ type and index
    uint32_t irq_type = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_type ) );
    uint32_t irq_id   = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_id ) );

    if( irq_type == SOCLIB_TYPE_HWI )
    {
        // enable this HWI in remote XCU controller 
        // in TSAR : XCU output [4*lid] is connected to core [lid]
        hal_remote_s32( XPTR( src_chdev_cxy , 
                       &seg_xcu_ptr[ (XCU_MSK_HWI_ENABLE << 5) | (lid<<2) ] ) , (1 << irq_id) );
    }
    else if( irq_type == SOCLIB_TYPE_WTI )
    {
        // enable this WTI in remote XCU controller 
        // in TSAR : XCU output [4*lid] is connected to core [lid]
        hal_remote_s32( XPTR( src_chdev_cxy , 
                       &seg_xcu_ptr[ (XCU_MSK_WTI_ENABLE << 5) | (lid<<2) ] ) , (1 << irq_id) );
    }
    else
    {
        assert( false , "illegal IRQ type\n" );
    }
} // end soclib_pic_enable_irq()

////////////////////////////////////////
void soclib_pic_disable_irq( lid_t  lid,
                             xptr_t src_chdev_xp )
{
    // get cluster and local pointer on remote src_chdev
    cxy_t     src_chdev_cxy = GET_CXY( src_chdev_xp );
    chdev_t * src_chdev_ptr = (chdev_t *)GET_PTR( src_chdev_xp );

    // get local pointer on remote XCU segment base
    uint32_t * seg_xcu_ptr = soclib_pic_remote_xcu_base( src_chdev_cxy );

    // get the source chdev IRQ type and index
    uint32_t irq_type = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_type ) );
    uint32_t irq_id   = hal_remote_l32( XPTR( src_chdev_cxy , &src_chdev_ptr->irq_id ) );

    if( irq_type == SOCLIB_TYPE_HWI )
    {
        // enable this HWI in remote XCU controller
        // in TSAR : XCU output [4*lid] is connected to core [lid]
        hal_remote_s32( XPTR( src_chdev_cxy , 
                       &seg_xcu_ptr[(XCU_MSK_HWI_DISABLE << 5) | (lid<<2) ] ) , (1 << irq_id) );
    }
    else if( irq_type == SOCLIB_TYPE_WTI )
    {
        // enable this WTI in remote XCU controller 
        // in TSAR : XCU output [4*lid] is connected to core [lid]
        hal_remote_s32( XPTR( src_chdev_cxy , 
                       &seg_xcu_ptr[(XCU_MSK_WTI_DISABLE << 5) | (lid<<2) ] ) , (1 << irq_id) );
    }
    else
    {
        assert( false , "illegal IRQ type\n" );
    }
} // end soclib_pic_enable_irq()

///////////////////////////////////////////////
void soclib_pic_enable_timer( uint32_t period )
{
    // calling core local index
    lid_t  lid = CURRENT_THREAD->core->lid;

    // get XCU segment base
    uint32_t * base = soclib_pic_xcu_base();

    // set period value in XCU (in cycles)
    uint32_t cycles = period * SOCLIB_CYCLES_PER_MS;
    base[(XCU_PTI_PER << 5) | lid] = cycles;

    // enable PTI in local XCU controller 
    // In TSAR : XCU output [4*lid] is connected to core [lid]
    base[ (XCU_MSK_PTI_ENABLE << 5) | (lid<<2) ] = 1 << lid;
}

////////////////////////////
void soclib_pic_enable_ipi( void )
{
    // calling core local index
    lid_t  lid = CURRENT_THREAD->core->lid;

    // get XCU segment base
    uint32_t * base = soclib_pic_xcu_base();

    // enable WTI in local XCU controller 
    // In TSAR : XCU output [4*lid] is connected to core [lid]
    base[ (XCU_MSK_WTI_ENABLE << 5) | (lid<<2) ] = 1 << lid;
}

///////////////////////////////////////
void soclib_pic_send_ipi( cxy_t    cxy,
                          lid_t    lid )
{
    // get pointer on local XCU segment base
    uint32_t * base = soclib_pic_xcu_base();

    // write to WTI mailbox[cxy][lid]
    hal_remote_s32( XPTR( cxy , &base[(XCU_WTI_REG << 5) | lid ] ) , 0 );
}

/////////////////////////
void soclib_pic_ack_ipi( void )
{
    // get calling core local index
    lid_t      lid  = CURRENT_THREAD->core->lid;

    // get pointer on local XCU segment base
    uint32_t * base = soclib_pic_xcu_base();

    // acknowlege IPI
    uint32_t   ack  = base[ (XCU_WTI_REG << 5) | lid ];

    // we must make a fake use for ack value to avoid a warning
    if( (ack + 1) == 0 ) asm volatile( "nop" );
}