Ignore:
Timestamp:
Jan 1, 2015, 7:58:56 PM (10 years ago)
Author:
alain
Message:

1) The chained buffer structure has been modified to have one single buffer descriptor per cache line (64 bytes),
in order to simplify the software cache coherence between L2 and L3 caches when the IO bridge is used.
A new buffer_descriptor_t structure has been defined, and the fbf_chbuf_t and nic_chbuf_t structures have been adapted.
2) The NIC related system call handler _sys_nic_start() and _sys_nic_move() have been modified to support a distributed
kernel chbuf (one 4 Kbytes buffer per cluster), in order to support the one Gbit Ethernet NIC controller throughput.

  • the _sys_nic_start() function initialises the distributed chbuf, using the distributed heap.
  • the _sys_nic_move() function transfer one 4 KBytes container from the local kernel chbuf to an user local buffer.

This approach has been validated on the "classif" application: no packet loss with 16 clusters for average packet
length = 600 bytes and inter-packet gap = 300 cycles.

Location:
soft/giet_vm/giet_kernel
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • soft/giet_vm/giet_kernel/kernel_init.c

    r467 r478  
    1616#include <xcu_driver.h>
    1717#include <ioc_driver.h>
     18#include <mmc_driver.h>
    1819#include <ctx_handler.h>
    1920#include <irq_handler.h>
    … …  
    106107////////////////////////////////////////////////////////////////////////////////////
    107108
    108 sbt_lock_t     _tty_tx_lock[NB_TTY_CHANNELS]  __attribute__((aligned(64)));
    109 
    110109unsigned int   _tty_rx_buf[NB_TTY_CHANNELS];
    111110unsigned int   _tty_rx_full[NB_TTY_CHANNELS];
    112111
    113 
    114 
    115 
     112////////////////////////////////////////////////////////////////////////////////////
     113// Distributed locks protecting TTY terminals       
     114////////////////////////////////////////////////////////////////////////////////////
     115
     116sbt_lock_t     _tty_tx_lock[NB_TTY_CHANNELS]  __attribute__((aligned(64)));
    116117
    117118///////////////////////////////////////////////////////////////////////////////////
    … …  
    142143    while( cpid != kernel_init_barrier ) asm volatile ( "nop" );
    143144
    144     // Step 0 : P[0,0,0] initialises the kernel FAT
    145     //          and the distributed kernel_heap descriptors array.
     145    // Step 0 : P[0,0,0] initialises various complex structures
     146    //          - kernel FAT
     147    //          - distributed kernel heaps
     148    //          - distributed locks protecting TTY channels
     149    //          - distributed locks protecting MMC components
    146150    if ( gpid == 0 )
    147151    {
    … …  
    155159        {
    156160            _sbt_lock_init( &_tty_tx_lock[channel] );
     161
     162#if GIET_DEBUG_INIT
     163_nolock_printf("\n[DEBUG KERNEL_INIT] P[%d,%d,%d] completes TTY[%d] lock init\n",
     164               x , y , p , channel );
     165#endif
    157166        }
    158167
    159 #if GIET_DEBUG_INIT
    160 _nolock_printf("\n[DEBUG KERNEL_INIT] P[%d,%d,%d] completes TTY locks init\n", x, y, p );
    161 #endif
    162 
     168/*
     169        unsigned int cx, cy;
     170        for ( cx = 0 ; cx < X_SIZE ; cx++ )
     171        {
     172            for ( cy = 0 ; cy < X_SIZE ; cy++ )
     173            {
     174                _sbt_lock_init( &_mmc_lock[cx][cy] );
     175
     176#if GIET_DEBUG_INIT
     177_nolock_printf("\n[DEBUG KERNEL_INIT] P[%d,%d,%d] completes MMC[%d][%d] lock init\n",
     178               x , y , p , cx , cy );
     179#endif
     180            }
     181        }
     182*/
    163183        _fat_init( IOC_BOOT_MODE );
    164184

  • soft/giet_vm/giet_kernel/sys_handler.c

    r467 r478  
    1616#include <fat32.h>
    1717#include <utils.h>
     18#include <kernel_malloc.h>
    1819#include <tty0.h>
    1920#include <vmem.h>
    … …  
    4647#endif
    4748
    48 #if !defined(GIET_NIC_NBUFS)
    49 # error: You must define GIET_NIC_NBUFS in the giet_config.h file
    50 #endif
    51 
    52 #if !defined(GIET_NIC_BUFSIZE)
    53 # error: You must define GIET_NIC_BUFSIZE in the giet_config.h file
    54 #endif
    55 
    56 #if !defined(GIET_NIC_TIMEOUT)
    57 # error: You must define GIET_NIC_TIMEOUT in the giet_config.h file
    58 #endif
    59 
    6049#if !defined(GIET_NO_HARD_CC)
    6150# error: You must define GIET_NO_HARD_CC in the giet_config.h file
     51#endif
     52
     53#if !defined ( GIET_NIC_MAC4 )
     54# error: You must define GIET_NIC_MAC4 in the giet_config.h file
     55#endif
     56
     57#if !defined ( GIET_NIC_MAC2 )
     58# error: You must define GIET_NIC_MAC2 in the giet_config.h file
    6259#endif
    6360
    … …  
    6865////////////////////////////////////////////////////////////////////////////
    6966
    70 unsigned int _tty_channel_allocator = 1;
    71 unsigned int _tim_channel_allocator = 0;
    72 unsigned int _cma_channel_allocator = 0;
     67unsigned int _tty_channel_allocator    = 1;
     68unsigned int _tim_channel_allocator    = 0;
     69unsigned int _cma_channel_allocator    = 0;
    7370unsigned int _nic_rx_channel_allocator = 0;
    7471unsigned int _nic_tx_channel_allocator = 0;
    7572
    7673////////////////////////////////////////////////////////////////////////////
    77 //     NIC_RX and NIC_TX chbuf arrays and associated containers arrays
     74//     NIC_RX and NIC_TX chbuf arrays
    7875////////////////////////////////////////////////////////////////////////////
    7976
    80 spin_lock_t  _nic_rx_lock[NB_NIC_CHANNELS]  __attribute__((aligned(64)));
    81 
    82 spin_lock_t  _nic_tx_lock[NB_NIC_CHANNELS]  __attribute__((aligned(64)));
    83 
    8477nic_chbuf_t  _nic_rx_chbuf[NB_NIC_CHANNELS] __attribute__((aligned(64)));
    8578
    8679nic_chbuf_t  _nic_tx_chbuf[NB_NIC_CHANNELS] __attribute__((aligned(64)));
    8780
    88 unsigned char _nic_rx_cont[GIET_NIC_BUFSIZE]
    89                          [GIET_NIC_NBUFS]
    90                          [NB_NIC_CHANNELS] __attribute__((aligned(64)));
    91 
    92 unsigned char _nic_tx_cont[GIET_NIC_BUFSIZE]
    93                          [GIET_NIC_NBUFS]
    94                          [NB_NIC_CHANNELS] __attribute__((aligned(64)));
     81////////////////////////////////////////////////////////////////////////////
     82// FBF related chbuf descriptors array, indexed by the CMA channel index.
     83// Physical addresses of these chbuf descriptors required for L2 cache sync.
     84////////////////////////////////////////////////////////////////////////////
     85
     86fbf_chbuf_t _fbf_chbuf[NB_CMA_CHANNELS] __attribute__((aligned(64)));
     87
     88unsigned long long _fbf_chbuf_paddr[NB_CMA_CHANNELS];
    9589
    9690////////////////////////////////////////////////////////////////////////////
    … …  
    336330//////////////////////////////////////////////////////////////////////////////
    337331
     332#define NIC_CONTAINER_SIZE 4096
     333
    338334///////////////////////////////////////////
    339335int _sys_nic_alloc( unsigned int is_rx )
    … …  
    342338#if GIET_DEBUG_NIC
    343339unsigned int thread  = _get_context_slot( CTX_TRDID_ID );
    344 unsigned int vspace  = _get_context_slot( CTX_VSID_ID );
    345 _printf("\n[GIET DEBUG NIC] Task %d in vspace %d enters sys_nic_alloc()\n",
    346         thread, vspace );
     340_printf("\n[GIET DEBUG NIC] Task %d enters sys_nic_alloc() at cycle %d\n",
     341        thread, _get_proctime() );
    347342#endif
    348343
    … …  
    382377
    383378#if GIET_DEBUG_NIC
    384 _printf("\n[GIET DEBUG NIC] Task %d in vspace %d exit _sys_nic_alloc() : "
     379_printf("\n[GIET DEBUG NIC] Task %d exit _sys_nic_alloc() at cycle %d : "
    385380        "NIC channel = %d / CMA channel = %d\n",
    386         thread, vspace, nic_channel, cma_channel );
     381        thread, _get_proctime(), nic_channel, cma_channel );
    387382#endif
    388383
    … …  
    396391#if GIET_DEBUG_NIC
    397392unsigned int thread  = _get_context_slot( CTX_TRDID_ID );
    398 unsigned int vspace  = _get_context_slot( CTX_VSID_ID );
    399 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d enters _sys_nic_start()\n",
    400         thread, vspace );
     393_printf("\n[GIET DEBUG NIC] Task %d enters _sys_nic_start() at cycle %d\n",
     394        thread , _get_proctime() );
    401395#endif
    402396
    … …  
    417411
    418412#if GIET_DEBUG_NIC
    419 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d in _sys_nic_start()"
     413_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_start() at cycle %d"
    420414        " get NIC channel = %d / CMA channel = %d\n",
    421         thread, vspace, nic_channel, cma_channel );
     415        thread, _get_proctime(), nic_channel, cma_channel );
    422416#endif
    423417
    … …  
    435429    unsigned long long nic_chbuf_pbase;     // NIC chbuf physical address
    436430    unsigned long long ker_chbuf_pbase;     // kernel chbuf physical address
    437     unsigned long long ker_cont_pbase;      // kernel container array physical address
    438 
    439     // These variables are used for V2P translation
     431
     432    // These variables are used for the various V2P translation
    440433    unsigned int       ptab  = _get_context_slot(CTX_PTAB_ID);
    441434    unsigned int       ppn;
    442435    unsigned int       flags;
    443     unsigned int       ko;
    444436    unsigned int       vaddr;
    445437
    446     // get the NIC chbuf descriptor physical address
     438    // allocate two containers per cluster
     439    unsigned int        cx;           // container X coordinate
     440    unsigned int        cy;           // container Y coordinate
     441    unsigned int        index;        // container index in chbuf
     442    unsigned long long  cont_paddr;   // container physical address
     443   
     444    for ( cx = 0 ; cx < X_SIZE ; cx++ )
     445    {
     446        for ( cy = 0 ; cy < Y_SIZE ; cy++ )
     447        {
     448            // compute index in chbuf
     449            index = (cx * Y_SIZE) + cy;
     450
     451            // allocate the container
     452            vaddr = (unsigned int)_remote_malloc( NIC_CONTAINER_SIZE, cx, cy );
     453
     454            // compute container physical address
     455            _v2p_translate( (page_table_t*)ptab,
     456                            vaddr>>12,
     457                            &ppn,
     458                            &flags );
     459            cont_paddr = (((unsigned long long)ppn) << 12) | (vaddr & 0x00000FFF);
     460
     461            // initialize chbuf
     462            if ( is_rx ) _nic_rx_chbuf[nic_channel].buffer[index].desc = cont_paddr;
     463            else         _nic_tx_chbuf[nic_channel].buffer[index].desc = cont_paddr;
     464
     465#if GIET_DEBUG_NIC
     466_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_start()"
     467        " allocates container in cluster[%d,%d] : vaddr = %x / paddr = %l\n",
     468        thread , cx , cy , vaddr , cont_paddr );
     469#endif
     470        }
     471    }
     472
     473    // compute the NIC chbuf descriptor physical address
    447474    unsigned int offset;
    448475    if ( is_rx ) offset = 0x4000;
    … …  
    452479
    453480#if GIET_DEBUG_NIC
    454 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d in _sys_nic_start()"
    455         " get NIC chbuf paddr = %l\n",
    456         thread, vspace, nic_chbuf_pbase );
     481_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_start()"
     482        " get NIC chbuf : paddr = %l\n",
     483        thread , nic_chbuf_pbase );
    457484#endif
    458485
    … …  
    460487    if ( is_rx ) vaddr = (unsigned int)( &_nic_rx_chbuf[nic_channel] );
    461488    else         vaddr = (unsigned int)( &_nic_tx_chbuf[nic_channel] );
    462     ko    = _v2p_translate( (page_table_t*)ptab,
    463                             vaddr>>12,
    464                             &ppn,
    465                             &flags );
    466     if ( ko )
    467     {
    468         _puts("\n[GIET ERROR] in _sys_nic_start() : kernel chbuf unmapped\n");
    469         return -1;
    470     }
     489    _v2p_translate( (page_table_t*)ptab,
     490                     vaddr>>12,
     491                     &ppn,
     492                     &flags );
    471493    ker_chbuf_pbase = (((unsigned long long)ppn) << 12) | (vaddr & 0x00000FFF);
    472494
    473495#if GIET_DEBUG_NIC
    474 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d in _sys_nic_start()"
    475         " get kernel chbuf paddr = %l\n",
    476         thread, vspace, ker_chbuf_pbase );
    477 #endif
    478 
    479     // compute the container array physical address
    480     if ( is_rx ) vaddr = ((unsigned int)&_nic_rx_cont[0][0][nic_channel]);
    481     else         vaddr = ((unsigned int)&_nic_tx_cont[0][0][nic_channel]);
    482     ko = _v2p_translate( (page_table_t*)ptab,
    483                          vaddr>>12,
    484                          &ppn,
    485                          &flags );
    486     if ( ko )
    487     {
    488         _puts("\n[GIET ERROR] in _sys_nic_start() : containers array unmapped\n");
    489         return -1;
    490     }
    491     ker_cont_pbase = (((unsigned long long)ppn) << 12) | (vaddr & 0x00000FFF);
    492 
    493 #if GIET_DEBUG_NIC
    494 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d in _sys_nic_start()"
    495         " get kernel containers array paddr = %l\n",
    496         thread, vspace, ker_cont_pbase );
    497 #endif
    498 
    499     // initializes the kernel chbuf descriptor
    500     unsigned int index;
    501     if ( is_rx )
    502     {
    503         _spin_lock_init( &_nic_rx_lock[nic_channel] );
    504         _nic_rx_chbuf[nic_channel].index = 0;
    505         for( index = 0 ; index < GIET_NIC_NBUFS ; index++ )
    506         {
    507             _nic_rx_chbuf[nic_channel].buffer[index] = ker_cont_pbase + (index<<12);
    508         }
    509     }
    510     else
    511     {
    512         _spin_lock_init( &_nic_tx_lock[nic_channel] );
    513         _nic_tx_chbuf[nic_channel].index = 0;
    514         for( index = 0 ; index < GIET_NIC_NBUFS ; index++ )
    515         {
    516             _nic_tx_chbuf[nic_channel].buffer[index] = ker_cont_pbase + (index<<12);
    517         }
    518     }
     496_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_start()"
     497        " get kernel chbuf : vaddr = %x / paddr = %l\n",
     498        thread , vaddr , ker_chbuf_pbase );
     499#endif
    519500
    520501    // sync the kernel chbuf in L2 after write in L2
    … …  
    529510        _cma_set_register( cma_channel, CHBUF_DST_DESC , (unsigned int)(ker_chbuf_pbase) );
    530511        _cma_set_register( cma_channel, CHBUF_DST_EXT  , (unsigned int)(ker_chbuf_pbase>>32) );
    531         _cma_set_register( cma_channel, CHBUF_DST_NBUFS, GIET_NIC_NBUFS );
     512        _cma_set_register( cma_channel, CHBUF_DST_NBUFS, X_SIZE*Y_SIZE );
    532513    }
    533514    else                      // kernel to NIC
    … …  
    535516        _cma_set_register( cma_channel, CHBUF_SRC_DESC , (unsigned int)(ker_chbuf_pbase) );
    536517        _cma_set_register( cma_channel, CHBUF_SRC_EXT  , (unsigned int)(ker_chbuf_pbase>>32) );
    537         _cma_set_register( cma_channel, CHBUF_SRC_NBUFS, GIET_NIC_NBUFS );
     518        _cma_set_register( cma_channel, CHBUF_SRC_NBUFS, X_SIZE*Y_SIZE );
    538519        _cma_set_register( cma_channel, CHBUF_DST_DESC , (unsigned int)(nic_chbuf_pbase) );
    539520        _cma_set_register( cma_channel, CHBUF_DST_EXT  , (unsigned int)(nic_chbuf_pbase>>32) );
    … …  
    542523
    543524    // start CMA transfer
    544     _cma_set_register( cma_channel, CHBUF_BUF_SIZE , 4096 );
    545     _cma_set_register( cma_channel, CHBUF_PERIOD   , 300 );
     525    _cma_set_register( cma_channel, CHBUF_BUF_SIZE , NIC_CONTAINER_SIZE );
     526    _cma_set_register( cma_channel, CHBUF_PERIOD   , 0 );                   // OUT_OF_ORDER mode
    546527    _cma_set_register( cma_channel, CHBUF_RUN      , 1 );
    547528
    … …  
    550531
    551532#if GIET_DEBUG_NIC
    552 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d exit _sys_nic_start()\n",
    553         thread, vspace );
     533_printf("\n[GIET DEBUG NIC] Task %d exit _sys_nic_start() at cycle %d\n",
     534        thread , _get_proctime() );
    554535#endif
    555536
    … …  
    565546#if GIET_DEBUG_NIC
    566547unsigned int thread  = _get_context_slot( CTX_TRDID_ID );
    567 unsigned int vspace  = _get_context_slot( CTX_VSID_ID );
    568 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d enters _sys_nic_move()\n",
    569         thread, vspace );
    570 #endif
    571 
     548_printf("\n[GIET DEBUG NIC] Task %d enters _sys_nic_move() at cycle %d\n",
     549        thread , _get_proctime() );
     550#endif
     551
     552    // get cluster coordinates for the processor running the calling task
     553    unsigned int  procid = _get_procid();
     554    unsigned int  cx     = procid >> (Y_WIDTH + P_WIDTH);
     555    unsigned int  cy     = (procid >> P_WIDTH) & ((1<<Y_WIDTH)-1);
     556   
    572557    unsigned long long user_buffer_paddr;    // user buffer physical address
    573558    unsigned long long kernel_buffer_paddr;  // kernel buffer physical address
    574559    unsigned long long kernel_chbuf_paddr;   // kernel chbuf physical address
    575     unsigned long long kernel_buffer_desc;   // kernel buffer descriptor
     560    unsigned long long buffer_desc;          // kernel buffer descriptor
     561    unsigned long long buffer_desc_paddr;    // kernel buffer descriptor physical address
     562    unsigned int       index;                // kernel buffer index in chbuf
    576563
    577564    // The following variables are used for V2P translation
    … …  
    580567    unsigned int flags;
    581568    unsigned int vaddr;
    582     unsigned int ko;
    583569
    584570    // Compute user buffer physical address and check access rights
    585571    vaddr = (unsigned int)buffer;
    586     ko = _v2p_translate( (page_table_t*)ptab,
    587                           vaddr>>12,
    588                           &ppn,
    589                           &flags );
    590     if ( ko )
    591     {
    592         _printf("\n[GIET ERROR] in _sys_nic_move() : user buffer unmapped\n");
    593         return -1;
    594     }
     572    _v2p_translate( (page_table_t*)ptab,
     573                     vaddr>>12,
     574                     &ppn,
     575                     &flags );
     576
    595577    if ( (flags & PTE_U) == 0 )
    596578    {
    … …  
    601583
    602584#if GIET_DEBUG_NIC
    603 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d in _sys_nic_move()"
    604         " get user buffer paddr = %l\n",
    605         thread, vspace, user_buffer_paddr );
     585_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_move() get user buffer : paddr = %l\n",
     586        thread, user_buffer_paddr );
    606587#endif
    607588
    … …  
    613594    }
    614595
    615     // get kernel chbuf and lock addresses
     596    // get kernel chbuf virtual address
    616597    nic_chbuf_t* chbuf;
    617     spin_lock_t* lock;
    618 
    619     if ( is_rx )
    620     {
    621         chbuf = &_nic_rx_chbuf[channel];
    622         lock  = &_nic_rx_lock[channel];
    623     }
    624     else
    625     {
    626         chbuf = &_nic_tx_chbuf[channel];
    627         lock  = &_nic_tx_lock[channel];
    628     }
     598    if ( is_rx )  chbuf = &_nic_rx_chbuf[channel];
     599    else          chbuf = &_nic_tx_chbuf[channel];
    629600
    630601    // compute kernel chbuf physical address (required for sync)
    631602    vaddr = (unsigned int)chbuf;
    632     ko = _v2p_translate( (page_table_t*)ptab,
    633                           vaddr>>12,
    634                           &ppn,
    635                           &flags );
    636     if ( ko )
    637     {
    638         _printf("\n[GIET ERROR] in _sys_nic_move() : kernel chbuf unmapped\n");
    639         return -1;
    640     }
     603    _v2p_translate( (page_table_t*)ptab,
     604                     vaddr>>12,
     605                     &ppn,
     606                     &flags );
    641607    kernel_chbuf_paddr = ((unsigned long long)ppn << 12) | (vaddr & 0x00000FFF);
    642608
    643     // get the lock protecting the chbuf
    644     _spin_lock_acquire( lock );
    645 
    646     // polling on the kernel chbuf status
    647     unsigned int full;
    648     unsigned int ok      = 0;
    649     unsigned int index   = chbuf->index;
    650     unsigned int timeout = GIET_NIC_TIMEOUT;
    651     while( ok == 0 )
    652     {
    653         // inval kernel chbuf in L2 before read in L2
    654         _mmc_inval( kernel_chbuf_paddr, sizeof( nic_chbuf_t ) );
    655 
    656         // get kernel buffer descriptor
    657         kernel_buffer_desc = chbuf->buffer[index];
     609    // poll chbuf until success
     610    while ( 1 )
     611    {
     612        // compute buffer index and buffer descriptor paddr
     613        index = (Y_SIZE * cx) + cy;
     614        buffer_desc_paddr = kernel_chbuf_paddr + (index<<6);
     615
     616        // inval buffer descriptor in L2 before read in L2
     617        _mmc_inval( buffer_desc_paddr , 8 );
     618        buffer_desc = chbuf->buffer[index].desc;
    658619
    659620#if GIET_DEBUG_NIC
    660 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d in _sys_nic_move()"
    661         " poll kernel container descriptor = %l\n",
    662         thread, vspace, kernel_buffer_desc );
    663 #endif
    664 
    665         full  =  kernel_buffer_desc >> 63;
    666         if ( (is_rx && full) || ((is_rx == 0) && (full == 0)) )  ok = 1;
    667         else                                                     timeout--;
    668 
    669         if ( timeout == 0 )
    670         {
    671             _printf("\n[GIET_ERROR] in _sys_nic_move() : timeout\n");
    672             return -1;
    673         }
     621_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_move() read buffer descriptor %d\n"
     622        " at cycle = %d / paddr = %l / buffer descriptor = %l\n",
     623        thread, index, _get_proctime(), buffer_desc_paddr, buffer_desc );
     624#endif
     625
     626        // test buffer status and break if found
     627        if ( ( is_rx != 0 ) && (buffer_desc >> 63) == 1 )  break;
     628        if ( ( is_rx == 0 ) && (buffer_desc >> 63) == 0 )  break;
    674629    }
    675630
    676631    // compute kernel buffer physical address
    677     kernel_buffer_paddr = kernel_buffer_desc & 0x0000FFFFFFFFFFFFULL;
     632    kernel_buffer_paddr = buffer_desc & 0x0000FFFFFFFFFFFFULL;
    678633   
    679634    // move one container, using a physical_memcpy
    680     // We must maintain L2/L3 cache coherence
    681635    if ( is_rx )
    682636    {
    683637        // inval kernel buffer in L2 before read in L2
    684         _mmc_inval( kernel_buffer_paddr, GIET_NIC_BUFSIZE );
     638        _mmc_inval( kernel_buffer_paddr, NIC_CONTAINER_SIZE );
    685639
    686640        // transfer data from kernel buffer to user buffer
    687641        _physical_memcpy( user_buffer_paddr,
    688642                          kernel_buffer_paddr,
    689                           GIET_NIC_BUFSIZE );
     643                          NIC_CONTAINER_SIZE );
     644#if GIET_DEBUG_NIC
     645_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_move() transfer "
     646        "kernel buffer %l to user buffer %l at cycle %d\n",
     647        thread , kernel_buffer_paddr , user_buffer_paddr , _get_proctime() );
     648#endif
    690649
    691650    }
    … …  
    695654        _physical_memcpy( kernel_buffer_paddr,
    696655                          user_buffer_paddr,
    697                           GIET_NIC_BUFSIZE );
     656                          NIC_CONTAINER_SIZE );
    698657
    699658        // sync kernel buffer in L2 after write in L2
    700         _mmc_sync( kernel_buffer_paddr, GIET_NIC_BUFSIZE );
    701     }
    702 
    703     // update kernel chbuf status and index
    704     if ( is_rx ) chbuf->buffer[index] = kernel_buffer_paddr & 0x0000FFFFFFFFFFFFULL;
    705     else         chbuf->buffer[index] = kernel_buffer_paddr | 0x8000000000000000ULL;
    706 
    707     if ( index == (GIET_NIC_NBUFS - 1) ) chbuf->index = 0;
    708     else                                 chbuf->index = index + 1;
     659        _mmc_sync( kernel_buffer_paddr, NIC_CONTAINER_SIZE );
     660
     661#if GIET_DEBUG_NIC
     662_printf("\n[GIET DEBUG NIC] Task %d in _sys_nic_move() transfer "
     663        "user buffer %l to kernel buffer %l at cycle %d\n",
     664        thread , user_buffer_paddr , kernel_buffer_paddr , _get_proctime() );
     665#endif
     666
     667    }
     668
     669    // update kernel chbuf status
     670    if ( is_rx ) chbuf->buffer[index].desc = kernel_buffer_paddr & 0x0000FFFFFFFFFFFFULL;
     671    else         chbuf->buffer[index].desc = kernel_buffer_paddr | 0x8000000000000000ULL;
    709672
    710673    // sync kernel chbuf in L2 after write in L2
    711     _mmc_sync( kernel_chbuf_paddr, sizeof( nic_chbuf_t ) );
    712 
    713     // release the lock protecting the chbuf
    714     _spin_lock_release( lock );
    715    
     674    _mmc_sync( kernel_chbuf_paddr + (index<<6) , 8 );
     675
    716676#if GIET_DEBUG_NIC
    717 _printf("\n[GIET DEBUG NIC] Task %d of vspace %d exit _sys_nic_move()\n",
    718         thread, vspace );
     677_printf("\n[GIET DEBUG NIC] Task %d get buffer %d  and exit _sys_nic_move() at cycle %d\n",
     678        thread , index , _get_proctime() );
    719679#endif
    720680
    … …  
    822782        unsigned int fifo_full  = _nic_get_global_register( NIC_G_NPKT_RX_DES_MFIFO_FULL     );
    823783        unsigned int crc_fail   = _nic_get_global_register( NIC_G_NPKT_RX_DES_CRC_FAIL       );
    824         unsigned int transmit   = _nic_get_global_register( NIC_G_NPKT_RX_DISPATCH_RECEIVED  );
    825784        unsigned int broadcast  = _nic_get_global_register( NIC_G_NPKT_RX_DISPATCH_BROADCAST );
    826785        unsigned int dst_fail   = _nic_get_global_register( NIC_G_NPKT_RX_DISPATCH_DST_FAIL  );
    … …  
    829788        _printf("\n### Network Controller RX Statistics ###\n"
    830789                "- packets received : %d\n"
    831                 "- packets transmit : %d\n"
    832790                "- too small        : %d\n"
    833791                "- too big          : %d\n"
    … …  
    838796                "- broadcast        : %d\n",
    839797                received,
    840                 transmit,
    841798                too_small,
    842799                too_big,
    … …  
    850807    {
    851808        unsigned int received   = _nic_get_global_register( NIC_G_NPKT_TX_DISPATCH_RECEIVED  );
    852         unsigned int transmit   = _nic_get_global_register( NIC_G_NPKT_TX_DISPATCH_TRANSMIT  );
    853809        unsigned int too_big    = _nic_get_global_register( NIC_G_NPKT_TX_DISPATCH_TOO_BIG   );
    854810        unsigned int too_small  = _nic_get_global_register( NIC_G_NPKT_TX_DISPATCH_TOO_SMALL );
    … …  
    859815        _printf("\n### Network Controller TX Statistics ###\n"
    860816                "- packets received : %d\n"
    861                 "- packets transmit : %d\n"
    862817                "- too small        : %d\n"
    863818                "- too big          : %d\n"
    … …  
    866821                "- broadcast        : %d\n",
    867822                received,
    868                 transmit,
    869823                too_big,
    870824                too_small,
    … …  
    880834/////////////////////////////////////////////////////////////////////////////////////////
    881835
    882 // Array of fbf_chbuf descriptors, indexed by the CMA channel index.
    883 __attribute__((section (".unckdata")))
    884 volatile fbf_chbuf_t _fbf_chbuf[NB_CMA_CHANNELS] __attribute__((aligned(32)));
    885 
    886 // Physical addresses of these fbf_chbuf descriptors (required for L2 cache sync)
    887 __attribute__((section (".unckdata")))
    888 unsigned long long _fbf_chbuf_paddr[NB_CMA_CHANNELS];
    889 
    890836/////////////////////////////////////////////
    891837int _sys_fbf_sync_write( unsigned int offset,
    … …  
    940886
    941887    unsigned int       ptab;            // page table virtual address
    942     unsigned int       ko;              // unsuccessfull V2P translation
    943888    unsigned int       vaddr;           // virtual address
    944889    unsigned int       flags;           // protection flags
    … …  
    979924    // compute frame buffer physical address and initialize _fbf_chbuf[channel]
    980925    vaddr = ((unsigned int)SEG_FBF_BASE);
    981     ko    = _v2p_translate( (page_table_t*) ptab,
    982                             (vaddr >> 12),
    983                             &ppn,
    984                             &flags );
    985     if (ko)
    986     {
    987         _printf("\n[GIET ERROR] in _fb_cma_start() : frame buffer unmapped\n");
    988         return -1;
    989     }
    990 
    991     _fbf_chbuf[channel].fbf    = ((paddr_t)ppn << 12) | (vaddr & 0x00000FFF);
     926    _v2p_translate( (page_table_t*) ptab,
     927                    (vaddr >> 12),
     928                    &ppn,
     929                    &flags );
     930
     931    _fbf_chbuf[channel].fbf.desc = ((paddr_t)ppn << 12) | (vaddr & 0x00000FFF);
    992932
    993933    // Compute user buffer 0 physical addresses and intialize _fbf_chbuf[channel]
    994934    vaddr = (unsigned int)vbase0;
    995     ko = _v2p_translate( (page_table_t*) ptab,
    996                          (vaddr >> 12),
    997                          &ppn,
    998                          &flags );
    999     if (ko)
    1000     {
    1001         _printf("\n[GIET ERROR] in _fbf_cma_start() : user buffer 0 unmapped\n");
    1002         return -1;
    1003     }
     935    _v2p_translate( (page_table_t*) ptab,
     936                    (vaddr >> 12),
     937                    &ppn,
     938                    &flags );
    1004939    if ((flags & PTE_U) == 0)
    1005940    {
    … …  
    1008943    }
    1009944
    1010     _fbf_chbuf[channel].buf0 = ((paddr_t)ppn << 12) | (vaddr & 0x00000FFF);
     945    _fbf_chbuf[channel].buf0.desc = ((paddr_t)ppn << 12) | (vaddr & 0x00000FFF);
    1011946
    1012947    // Compute user buffer 1 physical addresses and intialize _fbf_chbuf[channel]
    1013948    vaddr = (unsigned int)vbase1;
    1014     ko = _v2p_translate( (page_table_t*) ptab,
    1015                          (vaddr >> 12),
    1016                          &ppn,
    1017                          &flags );
    1018     if (ko)
    1019     {
    1020         _printf("\n[GIET ERROR] in _fbf_cma_start() : user buffer 1 unmapped\n");
    1021         return -1;
    1022     }
     949    _v2p_translate( (page_table_t*) ptab,
     950                    (vaddr >> 12),
     951                    &ppn,
     952                    &flags );
    1023953    if ((flags & PTE_U) == 0)
    1024954    {
    … …  
    1027957    }
    1028958
    1029     _fbf_chbuf[channel].buf1 = ((paddr_t)ppn << 12) | (vaddr & 0x00000FFF);
     959    _fbf_chbuf[channel].buf1.desc = ((paddr_t)ppn << 12) | (vaddr & 0x00000FFF);
    1030960
    1031961    // initializes buffer length
    … …  
    1034964    // Compute and register physical adress of the chbuf descriptor
    1035965    vaddr = (unsigned int)(&_fbf_chbuf[channel]);
    1036     ko = _v2p_translate( (page_table_t*) ptab,
    1037                          (vaddr >> 12),
    1038                          &ppn,
    1039                          &flags );
    1040     if (ko)
    1041     {
    1042         _printf("\n[GIET ERROR] in _fbf_cma_start() : chbuf descriptor unmapped\n");
    1043         return -1;
    1044     }
     966    _v2p_translate( (page_table_t*) ptab,
     967                    (vaddr >> 12),
     968                    &ppn,
     969                    &flags );
    1045970 
    1046971    chbuf_paddr = (((paddr_t)ppn) << 12) | (vaddr & 0x00000FFF);
    … …  
    1060985        " - buf1   pbase = %l\n"
    1061986        " - chbuf  pbase = %l\n",
    1062         _fbf_chbuf[channel].fbf,
    1063         _fbf_chbuf[channel].buf0,
    1064         _fbf_chbuf[channel].buf1,
     987        _fbf_chbuf[channel].fbf.desc,
     988        _fbf_chbuf[channel].buf0.desc,
     989        _fbf_chbuf[channel].buf1.desc,
    1065990        chbuf_paddr );
    1066991#endif
    … …  
    1070995                        chbuf_paddr,
    1071996                        2,
    1072                         chbuf_paddr + 16,
     997                        chbuf_paddr + 128,
    1073998                        1,
    1074999                        length );
    … …  
    10881013#if NB_CMA_CHANNELS > 0
    10891014
    1090     volatile paddr_t buf_paddr;
    1091     unsigned int     full = 1;
     1015    volatile unsigned int long long user_buffer_desc;
     1016    volatile unsigned int           full = 1;
    10921017
    10931018    // get channel index
    … …  
    11071032#endif
    11081033
    1109     // waiting user buffer empty
    1110     while ( full )
    1111     { 
    1112         if ( USE_IOB )
    1113         {
    1114             // INVAL L1 cache for the chbuf descriptor,
    1115             _dcache_buf_invalidate( (unsigned int)&_fbf_chbuf[channel], 32 );
    1116 
    1117             // INVAL L2 cache for the chbuf descriptor,
    1118             _mmc_inval( _fbf_chbuf_paddr[channel], 32 );
     1034    if ( buffer_index == 0 )    // user buffer 0
     1035    {
     1036        // waiting user buffer released by the CMA component)
     1037        while ( full )
     1038        { 
     1039            // INVAL L1 and L2 cache copies of user buffer descriptor, because
     1040            // it has been modified in RAM by the CMA component
     1041            _dcache_buf_invalidate( (unsigned int)&_fbf_chbuf[channel].buf0.desc , 8 );
     1042            _mmc_inval( _fbf_chbuf_paddr[channel] , 8 );
     1043
     1044            // get user buffer descriptor
     1045            user_buffer_desc = _fbf_chbuf[channel].buf0.desc;
     1046
     1047            // get user buffer descriptor status
     1048            full = ( (unsigned int)(user_buffer_desc>>63) );
    11191049        }
    11201050
    1121         // read user buffer descriptor
    1122         if ( buffer_index == 0 ) buf_paddr = _fbf_chbuf[channel].buf0;
    1123         else                     buf_paddr = _fbf_chbuf[channel].buf1;
    1124 
    1125         full = ( (unsigned int)(buf_paddr>>63) );
    1126     }
    1127 
    1128     if ( USE_IOB )
    1129     {
    11301051        // SYNC request for the user buffer, because
    11311052        // it will be read from XRAM by the CMA component
    1132         _mmc_sync( buf_paddr, _fbf_chbuf[channel].length );
    1133     }
    1134 
    1135     // set user buffer status
    1136     if ( buffer_index == 0 ) _fbf_chbuf[channel].buf0 = buf_paddr | 0x8000000000000000ULL;
    1137     else                     _fbf_chbuf[channel].buf1 = buf_paddr | 0x8000000000000000ULL;
    1138 
    1139     // reset fbf buffer status
    1140     _fbf_chbuf[channel].fbf  = _fbf_chbuf[channel].fbf & 0x7FFFFFFFFFFFFFFFULL;
    1141 
    1142     if ( USE_IOB )
    1143     {
    1144         // SYNC request for the channel descriptor, because
     1053        _mmc_sync( user_buffer_desc, _fbf_chbuf[channel].length );
     1054
     1055        // set user buffer status, and SYNC request, because this buffer
     1056        // descriptor will be read from XRAM by the CMA component
     1057        _fbf_chbuf[channel].buf0.desc = user_buffer_desc | 0x8000000000000000ULL;
     1058        _mmc_sync( _fbf_chbuf_paddr[channel] , 8 );
     1059
     1060        // reset fbf buffer status, and SYNC request, because this buffer
     1061        // descriptor will be read from XRAM by the CMA component
     1062        _fbf_chbuf[channel].fbf.desc  = _fbf_chbuf[channel].fbf.desc & 0x7FFFFFFFFFFFFFFFULL;
     1063        _mmc_sync( _fbf_chbuf_paddr[channel] + 128 , 8 );
     1064    }
     1065    else                        // user buffer 1
     1066    {
     1067        // waiting user buffer released by the CMA component)
     1068        while ( full )
     1069        { 
     1070            // INVAL L1 and L2 cache copies of user buffer descriptor, because
     1071            // it has been modified in RAM by the CMA component
     1072            _dcache_buf_invalidate( (unsigned int)&_fbf_chbuf[channel].buf1.desc , 8 );
     1073            _mmc_inval( _fbf_chbuf_paddr[channel] + 64 , 8 );
     1074
     1075            // get user buffer descriptor
     1076            user_buffer_desc = _fbf_chbuf[channel].buf1.desc;
     1077
     1078            // get user buffer descriptor status
     1079            full = ( (unsigned int)(user_buffer_desc>>63) );
     1080        }
     1081
     1082        // SYNC request for the user buffer, because
    11451083        // it will be read from XRAM by the CMA component
    1146         _mmc_sync( _fbf_chbuf_paddr[channel], 32 );
     1084        _mmc_sync( user_buffer_desc, _fbf_chbuf[channel].length );
     1085
     1086        // set user buffer status, and SYNC request, because this buffer
     1087        // descriptor will be read from XRAM by the CMA component
     1088        _fbf_chbuf[channel].buf1.desc = user_buffer_desc | 0x8000000000000000ULL;
     1089        _mmc_sync( _fbf_chbuf_paddr[channel] , 8 );
     1090
     1091        // reset fbf buffer status, and SYNC request, because this buffer
     1092        // descriptor will be read from XRAM by the CMA component
     1093        _fbf_chbuf[channel].fbf.desc  = _fbf_chbuf[channel].fbf.desc & 0x7FFFFFFFFFFFFFFFULL;
     1094        _mmc_sync( _fbf_chbuf_paddr[channel] + 128 , 8 );
    11471095    }
    11481096
    11491097#if GIET_DEBUG_FBF_CMA
    1150 _printf(" - fbf    pbase = %l\n"
    1151         " - buf0   pbase = %l\n"
    1152         " - buf1   pbase = %l\n",
    1153         _fbf_chbuf[channel].fbf,
    1154         _fbf_chbuf[channel].buf0,
    1155         _fbf_chbuf[channel].buf1 );
    1156 #endif
    1157 
     1098_printf(" - fbf    desc = %l\n"
     1099        " - buf0   desc = %l\n"
     1100        " - buf1   desc = %l\n",
     1101        _fbf_chbuf[channel].fbf.desc,
     1102        _fbf_chbuf[channel].buf0.desc,
     1103        _fbf_chbuf[channel].buf1.desc );
     1104#endif
    11581105
    11591106    return 0;
    … …  
    11661113#endif
    11671114} // end _sys_fbf_cma_display()
     1115
    11681116
    11691117///////////////////////
    … …  
    13681316                      unsigned int* y )
    13691317{
    1370     unsigned int ret;
    13711318    unsigned int ppn;
    13721319    unsigned int flags;
    … …  
    13771324
    13781325    // compute the physical address
    1379     if ( (ret = _v2p_translate( pt, vpn, &ppn, &flags )) ) return -1;
     1326    _v2p_translate( pt, vpn, &ppn, &flags );
    13801327
    13811328    *x = (ppn>>24) & 0xF;

  • soft/giet_vm/giet_kernel/sys_handler.h

    r467 r478  
    1616#include "locks.h"
    1717
    18 #if !defined ( GIET_NIC_NBUFS )
    19 # error: You must define GIET_NIC_NBUFS in the giet_config.h file
    20 #endif
    21 
    22 #if !defined ( GIET_NIC_NFAKE )
    23 # error: You must define GIET_NIC_NFAKE in the giet_config.h file
    24 #endif
    25 
    26 #if !defined ( GIET_NIC_BUFSIZE )
    27 # error: You must define GIET_NIC_BUFSIZE in the giet_config.h file
    28 #endif
    29 
    30 #if !defined ( GIET_NIC_TIMEOUT )
    31 # error: You must define GIET_NIC_TIMEOUT in the giet_config.h file
    32 #endif
    33 
    34 #if !defined ( GIET_NIC_MAC4 )
    35 # error: You must define GIET_NIC_MAC4 in the giet_config.h file
    36 #endif
    37 
    38 #if !defined ( GIET_NIC_MAC2 )
    39 # error: You must define GIET_NIC_MAC2 in the giet_config.h file
    40 #endif
    41 
    42 #if ( (GIET_NIC_NBUFS + GIET_NIC_NFAKE) % 8 )
    43 #error: GIET_NIC_NBUFS + GIET_NIC_NFAKE must be multiple of 8 for alignment
    44 #endif
    45 
    4618///////////////////////////////////////////////////////////////////////////////////
    4719//     Syscall Vector Table (indexed by syscall index)
    … …  
    5123
    5224///////////////////////////////////////////////////////////////////////////////////
     25// This structure is used by the nic_chbuf_t and fbf_chbuf_t structures.
     26// It describes a single buffer descriptor. The useful information is contained
     27// in one single 64 bits word (desc field):
     28// - the 48 LSB bits contain the buffer physical address
     29// - the MSB bit 63 indicates the buffer state (empty if 0)
     30// This descriptor must be aligned on a cache line (64 bytes) to simplify
     31// the software L2/L3 cache coherence when the IO bridge is used.
     32///////////////////////////////////////////////////////////////////////////////////
     33
     34typedef struct buffer_descriptor_s
     35{
     36    unsigned long long  desc;
     37    unsigned int        padding[14];
     38} buffer_descriptor_t;
     39 
     40///////////////////////////////////////////////////////////////////////////////////
    5341// This structure is used by the CMA component to move a stream
    54 // of images from two buffers in user space to the frame buffer in kernel space.
     42// of images from two user buffers to the frame buffer in kernel space.
    5543// it must be 64 bytes aligned.
    5644// It contains two chbuf arrays:
    57 // - The SRC chbuf contains two buffers (buf0 & buf1), that can be in user space.
     45// - The SRC chbuf contains two buffers (buf0 & buf1), in user space.
    5846// - The DST cbuf contains one single buffer (fbf), that is the frame buffer.
    5947// - The length field define the buffer size (bytes)
    … …  
    6250typedef struct fbf_chbuf_s
    6351{
    64     unsigned long long  buf0;        // physical address + status for user buffer 0
    65     unsigned long long  buf1;        // physical address + status for user buffer 1
    66     unsigned long long  fbf;         // physical address + status for user buffer 0
    67     unsigned int        length;      // buffer length (bytes)
    68     unsigned int        padding[9];  // padding for 64 bytes alignment
     52    buffer_descriptor_t  buf0;         // first user buffer descriptor
     53    buffer_descriptor_t  buf1;         // second user buffer descriptor
     54    buffer_descriptor_t  fbf;          // frame buffer descriptor
     55    unsigned int         length;       // buffer length (bytes)
     56    unsigned int         padding[15];  // padding for 64 bytes alignment
    6957} fbf_chbuf_t;   
    7058
    7159//////////////////////////////////////////////////////////////////////////////////
    72 // This structure is used by the CMA component to move a stream
    73 // of packet containers between the NIC component an a chbuf containing
    74 // a variable number of buffers in kernel space.
     60// This structure is used by the CMA component to move a stream of containers
     61// between the NIC chbuf containing 2 buffers, and a kernel chbuf
     62// containing (X_SIZE * Y_SIZE) buffers (one buffer per cluster).
    7563// The same structure is used for both TX or RX transfers.
    7664// It must be 64 bytes aligned.
    77 // The single buffer size and the number of buffers must be defined by the
    78 // GIET_NIC_BUFSIZE and GIET_NIC_NBUFS parameters in giet_config.h.
    79 // - The buffer array implements the chbuf, and is concurently accessed
    80 //   by the CMA component and by the kernel code.
    81 // - The lock must be taken by the kernel code, because several user tasks
    82 //   can concurently try to consume buffers in the chbuf.
    83 // - The index is only used by the kernel, and define the currently pointed
    84 //   buffer for read (RX transfer) or write (TX transfer).
    8565//////////////////////////////////////////////////////////////////////////////////
    8666
    8767typedef struct nic_chbuf_s
    8868{
    89     unsigned long long  buffer[GIET_NIC_NBUFS];  // Kernel chbuf
    90     unsigned long long  unused[GIET_NIC_NFAKE];  // padding for 64 bytes alignment
    91     unsigned int        index;                   // current buffer index
    92     unsigned int        padding[15];             // padding for 64 bytes alignment
     69    buffer_descriptor_t  buffer[X_SIZE*Y_SIZE];  // kernel chbuf
    9370} nic_chbuf_t;
    94 
    9571
    9672//////////////////////////////////////////////////////////////////////////////////
Note: See TracChangeset for help on using the changeset viewer.