source: soft/giet_vm/giet_drivers/dma_driver.c @ 412

///////////////////////////////////////////////////////////////////////////////////
// File     : dma_driver.c
// Date     : 23/11/2013
// Author   : alain greiner
// Copyright (c) UPMC-LIP6
///////////////////////////////////////////////////////////////////////////////////
// The dma_driver.c and dma_driver.h files are part ot the GIET-VM nano-kernel.
// This driver supports the SoCLib vci_multi_dma component.
// 
// It can exist several DMA controlers in the architecture (one per cluster),
// and each controller can contain several channels.
// 
// There is  (NB_CLUSTERS * NB_DMA_CHANNELS) channels, indexed by a global index:
//        dma_id = cluster_xy * NB_DMA_CHANNELS + loc_id
//
// A DMA channel is a private ressource allocated to a given processor.
// It is exclusively used by the kernet to speedup data transfers, and
// there is no lock protecting exclusive access to the channel.
// As the kernel uses a polling policy on the DMA_STATUS register to detect
// transfer completion, the DMA IRQ is not used.
//
// The virtual base address of the segment associated to a channel is:
//    SEG_DMA_BASE + cluster_xy * PERI_CLUSTER_INCREMENT + DMA_SPAN * channel_id
//
// The SEG_DMA_BASE virtual address mus be defined in the hard_config.h file.
////////////////////////////////////////////////////////////////////////////////////

#include <giet_config.h>
#include <hard_config.h>
#include <dma_driver.h>
#include <tty_driver.h>
#include <vmem.h>
#include <utils.h>
#include <io.h>

#if !defined(X_SIZE) 
# error: You must define X_SIZE in the hard_config.h file
#endif

#if !defined(Y_SIZE) 
# error: You must define X_SIZE in the hard_config.h file
#endif

#if !defined(X_WIDTH) 
# error: You must define X_WIDTH in the hard_config.h file
#endif

#if !defined(Y_WIDTH) 
# error: You must define X_WIDTH in the hard_config.h file
#endif

#if !defined(NB_DMA_CHANNELS) 
# error: You must define NB_DMA_CHANNELS in the hard_config.h file
#endif

#if !defined(SEG_DMA_BASE) 
# error: You must define SEG_DMA_BASE in the hard_config.h file
#endif

#if !defined(PERI_CLUSTER_INCREMENT) 
# error: You must define PERI_CLUSTER_INCREMENT in the hard_config.h file
#endif

extern volatile unsigned int _ptabs_vaddr[];

///////////////////////////////////////////////////////////////////////////////
// This low level function returns the value contained in register "index"
// in the DMA component contained in cluster "cluster_xy"
///////////////////////////////////////////////////////////////////////////////
static
unsigned int _dma_get_register( unsigned int cluster_xy, // cluster index
                                unsigned int channel_id, // channel index
                                unsigned int index )     // register index
{
    unsigned int vaddr =
        SEG_DMA_BASE + 
        (cluster_xy * PERI_CLUSTER_INCREMENT) +
        (channel_id * DMA_SPAN) +
        (index << 2);

    return ioread32( (void*)vaddr );
}

///////////////////////////////////////////////////////////////////////////////
// This low level function sets a new value in register "index"
// in the DMA component contained in cluster "cluster_xy"
///////////////////////////////////////////////////////////////////////////////
static
void _dma_set_register( unsigned int cluster_xy,       // cluster index
                        unsigned int channel_id,       // channel index
                        unsigned int index,            // register index
                        unsigned int value )           // value to be written
{
    unsigned int vaddr =
        SEG_DMA_BASE + 
        (cluster_xy * PERI_CLUSTER_INCREMENT) +
        (channel_id * DMA_SPAN) +
        (index << 2);

    iowrite32( (void*)vaddr, value );
}

//////////////////////////////////////////////////////////////////////////////////
// AS the GIET-VM uses a polling policy to detect transfer completion,
// The DMA component initialisation must disable interrupts.
// This function disables interrupts for one DMA channel in one cluster.
// Returns 0 if success, returns > 0 if error.
//////////////////////////////////////////////////////////////////////////////////
unsigned int _dma_init( unsigned int cluster_xy,
                        unsigned int channel_id )
{
#if NB_DMA_CHANNELS > 0

    // parameters checking 
    unsigned int x = cluster_xy >> Y_WIDTH;
    unsigned int y = cluster_xy & ((1<<Y_WIDTH)-1);
    if (x >= X_SIZE)                    return 1; 
    if (y >= Y_SIZE)                    return 1; 
    if (channel_id >= NB_DMA_CHANNELS)  return 1; 

    // disable interrupt for selected channel
    _dma_set_register(cluster_xy, channel_id, DMA_IRQ_DISABLE, 1);
    return 0;
#else
    return 1;
#endif
}

//////////////////////////////////////////////////////////////////////////////////
// This function re-initialises one DMA channel in one cluster after a transfer
// completion. It actually forces the channel to return in iDLE state.
//////////////////////////////////////////////////////////////////////////////////
unsigned int _dma_reset( unsigned int cluster_xy, 
                         unsigned int channel_id ) 
{
#if NB_DMA_CHANNELS > 0

    // parameters checking 
    unsigned int x = cluster_xy >> Y_WIDTH;
    unsigned int y = cluster_xy & ((1<<Y_WIDTH)-1);
    if (x >= X_SIZE)                    return 1; 
    if (y >= Y_SIZE)                    return 1; 
    if (channel_id >= NB_DMA_CHANNELS)  return 1; 

    // reset selected channel
    _dma_set_register(cluster_xy, channel_id, DMA_RESET, 0);
    return 0;
#else
    return 1;
#endif
}

//////////////////////////////////////////////////////////////////////////////////
// This function returns the status of a DMA channel in a given cluster
//////////////////////////////////////////////////////////////////////////////////
unsigned int _dma_get_status( unsigned int cluster_xy, 
                              unsigned int channel_id ) 
{
#if NB_DMA_CHANNELS > 0

    // parameters checking 
    unsigned int x = cluster_xy >> Y_WIDTH;
    unsigned int y = cluster_xy & ((1<<Y_WIDTH)-1);
    if (x >= X_SIZE)                    return 1; 
    if (y >= Y_SIZE)                    return 1; 
    if (channel_id >= NB_DMA_CHANNELS)  return 1;

    // get selected channel status
    return _dma_get_register(cluster_xy, channel_id, DMA_LEN);
#else
    return DMA_IDLE;
#endif
}

//////////////////////////////////////////////////////////////////////////////////
// This function sets the physical address (including 64 bits extension)
// for the source and destination buffers in a DMA channel in a given cluster
// and sets the transfer size to lauch the transfer.
//////////////////////////////////////////////////////////////////////////////////
void _dma_start_transfer( unsigned int       cluster_xy,  // DMA cluster
                          unsigned int       channel_id,  // DMA channel
                          unsigned long long dst_paddr,   // physical address
                          unsigned long long src_paddr,   // physical address
                          unsigned int       size )       // bytes
{
#if NB_DMA_CHANNELS > 0

    // selected channel configuration and lauching
    _dma_set_register(cluster_xy, channel_id, DMA_SRC,
            (unsigned int)(src_paddr));
    _dma_set_register(cluster_xy, channel_id, DMA_SRC_EXT,
            (unsigned int)(src_paddr>>32));
    _dma_set_register(cluster_xy, channel_id, DMA_DST,
            (unsigned int)(dst_paddr));
    _dma_set_register(cluster_xy, channel_id, DMA_DST_EXT,
            (unsigned int)(dst_paddr>>32));
    _dma_set_register(cluster_xy, channel_id, DMA_LEN,
            (unsigned int)size);

#endif
}

///////////////////////////////////////////////////////////////////////////////////
// This function copies a source memory buffer to a destination memory buffer,
// using directly physical addresses.
// This blocking function is supposed to be used by the kernel only, 
// and uses a polling policy on DMA_STATUS register to detect completion.
// Therefore, the DMA_IRQ is NOT used.
// The source and destination buffers base addresses must be word aligned, 
// and the buffer's size must be multiple of 4.
// In case of error (buffer unmapped, unaligned, or DMA_STATUS error), an error 
// message is displayed on TTY0, and the system crash.
///////////////////////////////////////////////////////////////////////////////////
void _dma_physical_copy( unsigned int       cluster_xy,  // DMA cluster
                         unsigned int       channel_id,  // DMA channel
                         unsigned long long dst_paddr,   // destination physical address
                         unsigned long long src_paddr,   // source physical address
                         unsigned int       size )       // bytes
{
#if NB_DMA_CHANNELS > 0

    // check DMA channel parameters 
    unsigned int x = cluster_xy >> Y_WIDTH;
    unsigned int y = cluster_xy & ((1<<Y_WIDTH)-1);
    if ( (x >= X_SIZE) || (y >= Y_SIZE) || (channel_id >= NB_DMA_CHANNELS) )
    {
        _printf("\n[GIET ERROR] in _dma_physical_copy() : illegal DMA channel ");
        _exit();
    }

    // check buffers alignment constraints
    if ( (dst_paddr & 0x3)   || (src_paddr & 0x3) || (size & 0x3) )
    {
        _printf("\n[GIET ERROR] in _dma_physical_copy() : buffer unaligned\n");
        _exit();
    }

#if GIET_DEBUG_DMA_DRIVER
_printf("\n[DMA DEBUG] Start a dma_physical_copy on channel[%d,%d,%d] at cycle %d\n"
        " - src_paddr   = %l\n"
        " - dst_paddr   = %l\n"
        " - bytes       = %x\n",
        x, y, channel_id, _get_proctime(), src_paddr, dst_paddr, size );
#endif

    // dma channel configuration & lauching
    _dma_start_transfer( cluster_xy, channel_id, dst_paddr, src_paddr, size ); 

    // scan dma channel status 
    unsigned int status = _dma_get_status( cluster_xy, channel_id );
    while( (status != DMA_SUCCESS) && 
           (status != DMA_READ_ERROR) &&
           (status != DMA_WRITE_ERROR) )
    {
        status = _dma_get_status( cluster_xy, channel_id );

#if GIET_DEBUG_DMA_DRIVER
_printf("\n[DMA DEBUG] _dma_physical_copy() : ... waiting on DMA_STATUS register\n");
#endif

    }
    
    // analyse status
    if( status != DMA_SUCCESS )
    {
        _printf("\n[GIET ERROR] in _dma_physical_copy() : DMA_STATUS = %x\n", status );
        _exit();
    }
    // reset dma channel
    _dma_reset( cluster_xy, channel_id );

#if GIET_DEBUG_DMA_DRIVER
_printf("\n[DMA DEBUG] _dma_physical_copy() completed at cycle %d\n", _get_proctime() );
#endif

#else // NB_DMA_CHANNELS == 0
    _printf("\n[GIET ERROR] in _dma_physical_copy() : NB_DMA_CHANNELS == 0 / cycle %d\n",
            _get_proctime );
    _exit();
#endif
}

///////////////////////////////////////////////////////////////////////////////////
// This function copies a source memory buffer to a destination memory buffer,
// making virtual to physical address translation: the MMU should be activated. 
// This blocking function is supposed to be used by the kernel only, 
// and uses a polling policy on DMA_STATUS register to detect completion.
// Therefore, the DMA_IRQ is NOT used.
// The source and destination buffers base addresses must be word aligned, 
// and the buffer's size must be multiple of 4.
// In case of error (buffer unmapped, unaligned, or DMA_STATUS error), an error 
// message is displayed on TTY0, and the system crash.
///////////////////////////////////////////////////////////////////////////////////
void  _dma_copy( unsigned int cluster_xy,    // DMA cluster
                 unsigned int channel_id,    // DMA channel
                 unsigned int vspace_id,     // vspace index for v2p translation
                 unsigned int dst_vaddr,     // dst_vaddr buffer vbase
                 unsigned int src_vaddr,     // src_vaddr buffer vbase
                 unsigned int size )         // bytes
{
#if NB_DMA_CHANNELS > 0

    // check DMA channel parameters 
    unsigned int x = cluster_xy >> Y_WIDTH;
    unsigned int y = cluster_xy & ((1<<Y_WIDTH)-1);
    if ( (x >= X_SIZE) || (y >= Y_SIZE) || (channel_id >= NB_DMA_CHANNELS) )
    {
        _printf("\n[GIET ERROR] in _dma_copy() : illegal DMA channel ");
        _exit();
    }

    // check buffers alignment constraints
    if ( (dst_vaddr & 0x3)   || (src_vaddr & 0x3) || (size & 0x3) )
    {
        _printf("\n[GIET ERROR] in _dma_copy() : buffer unaligned\n");
        _exit();
    }

    unsigned int ko;
    unsigned int ppn;
    unsigned int flags;

#if GIET_DEBUG_DMA_DRIVER
_printf("\n[DMA DEBUG] Start a dma_copy on channel[%d,%d,%d] at cycle %d\n"
        " - src_vaddr   = %x\n"
        " - dst_vaddr   = %x\n"
        " - bytes       = %x\n",
        x, y, channel_id, _get_proctime(), src_vaddr, dst_vaddr, size );
#endif

    // checking alignment constraints
    if ( (((unsigned int)dst_vaddr) & 0x3) ||
         (((unsigned int)src_vaddr) & 0x3) ||
         (size & 0x3) )
    {
        _printf("\n[GIET ERROR] in _dma_copy() : buffer unaligned\n");
        _exit();
    }

    // get vspace page table pointer 
    unsigned int pt = _ptabs_vaddr[vspace_id];

    // get src_paddr buffer physical addresse
    ko = _v2p_translate( (page_table_t*)pt,              // page table pointer
                         src_vaddr>>12,                  // vpn 
                         &ppn,                           // ppn
                         &flags );                       // flags
    if ( ko ) 
    {
        _printf("\n[GIET ERROR] in _dma_copy() : source buffer unmapped\n");
        _exit();
    }
    unsigned long long src_paddr = (((unsigned long long)ppn) << 12) | 
                                   (unsigned long long)(src_vaddr & 0x00000FFF);

    // get dst_paddr buffer physical addresse
    ko = _v2p_translate( (page_table_t*)pt,              // page table pointer
                         dst_vaddr>>12,                  // vpn 
                         &ppn,                           // ppn
                         &flags );                       // flags
    if ( ko ) 
    {
        _printf("\n[GIET ERROR] in _dma_copy() : dest buffer unmapped\n");
        _exit();
    }
    unsigned long long dst_paddr = (((unsigned long long)ppn) << 12) | 
                                   (unsigned long long)(dst_vaddr & 0x00000FFF);

#if GIET_DEBUG_DMA_DRIVER
_printf(" - src_paddr   = %l\n"
        " - dst_paddr   = %l\n",
        src_paddr, dst_paddr );
#endif

    // dma channel configuration & lauching
    _dma_start_transfer(  cluster_xy, channel_id, dst_paddr, src_paddr, size ); 

    // scan dma channel status 
    unsigned int status = _dma_get_status( cluster_xy, channel_id );
    while( (status != DMA_SUCCESS) && 
           (status != DMA_READ_ERROR) &&
           (status != DMA_WRITE_ERROR) )
    {
        status = _dma_get_status( cluster_xy, channel_id );

#if GIET_DEBUG_DMA_DRIVER
_printf("\n[DMA DEBUG] _dma_copy() : ... waiting on DMA_STATUS register\n");
#endif

    }
    
    // analyse status
    if( status != DMA_SUCCESS )
    {
        _printf("\n[GIET ERROR] in _dma_copy() : DMA_STATUS = %x\n", status );
        _exit();
    }
    // reset dma channel
    _dma_reset( cluster_xy, channel_id );

#if GIET_DEBUG_DMA_DRIVER
_printf("\n[DMA DEBUG] _dma_copy() completed at cycle %d\n", _get_proctime() );
#endif

#else // NB_DMA_CHANNELS == 0
    _printf("\n[GIET ERROR] in _dma_copy() : NB_DMA_CHANNELS == 0 / cycle %d\n",
            _get_proctime );
    _exit();
#endif
} // end _dma_copy

///////////////////////////////////////////////////////////////////////////////
// This ISR handles the IRQ generated by a DMA channel.
///////////////////////////////////////////////////////////////////////////////
void _dma_isr( unsigned int irq_type,
               unsigned int irq_id,
               unsigned int channel )
{
    _printf("\n[GIET ERROR] _dma_isr() not implemented / cycle %d\n",
            _get_proctime() );
    _exit();
}




// tab-width: 4
// c-basic-offset: 4
// c-file-offsets:((innamespace . 0)(inline-open . 0))
// indent-tabs-mode: nil
// End:
// vim: filetype=c:expandtab:shiftwidth=4:tabstop=4:softtabstop=4