source: trunk/platforms/tsar_generic_leti/top.cpp @ 690

/////////////////////////////////////////////////////////////////////////
// File: top.cpp (for tsar_generic_leti)
// Author: Alain Greiner 
// Copyright: UPMC/LIP6
// Date : february 2014
// This program is released under the GNU public license
/////////////////////////////////////////////////////////////////////////
// This file define a generic TSAR architecture, fully compatible
// with the VLSI Hardware prototype developped by CEA-LETI and LIP6
// in the framework of the SHARP project.
// 
// The processor is a MIPS32 processor wrapped in a GDB server
// (this is defined in the tsar_xbar_cluster).
// 
// It does not use an external ROM, as the boot code is (pre)loaded
// in cluster (0,0) memory at address 0x0.
//
// The physical address space is 40 bits.
// The 8 address MSB bits define the cluster index.
//
// The main hardware parameters are the mesh size (X_SIZE & Y_SIZE),
// and the number of processors per cluster (NB_PROCS_MAX).
// The number of clusters cannot be larger than 128.
// The number of processors per cluster cannot be larger than 4.
// 
// Each cluster contains:
// - 5 dspin_local_crossbar (local interconnect)
// - 5 dspin_router (global interconnect)
// - up to 4 vci_cc_vcache wrapping a MIPS32 processor
// - 1 vci_mem_cache
// - 1 vci_xicu
// - 1 vci_simple_ram (to model the L3 cache).
//
// Each processor receives 4 consecutive IRQ lines from the local XICU.
//
// In all clusters, the MEMC IRQ line (signaling a late write error)
// is connected to XICU HWI[8]
// The cluster (0,0) contains two "backup" peripherals:
// - one block device controller, whose IRQ is connected to XICU HWI[9].
// - one single channel TTY controller, whose IRQ is connected to XICU HWI[10].
//
// The cluster internal architecture is defined in file tsar_leti_cluster,
// that must be considered as an extension of this top.cpp file.
//
// Besides the hardware components in clusters, "external" peripherals 
// are connected to an external IO bus (implemented as a vci_local_crossbar):
// - one disk controller 
// - one multi-channel ethernet controller 
// - one multi-channel chained buffer dma controller
// - one multi-channel tty controller 
// - one frame buffer controller 
// - one 32 channels iopic controller
//
// This IOBUS is connected to the north  port of the DIR_CMD 
// and DIR_RSP routers, in cluster(X_SIZE-1, Y_SIZE-1).
// For all external peripherals, the hardware interrupts (HWI) are
// translated to write interrupts (WTI) by the iopic component:
// - IOPIC HWI[1:0]     connected to IRQ_NIC_RX[1:0]
// - IOPIC HWI[3:2]     connected to IRQ_NIC_TX[1:0]      
// - IOPIC HWI[7:4]     connected to IRQ_CMA_TX[3:0]]
// - IOPIC HWI[8]       connected to IRQ_BDEV
// - IOPIC HWI[15:9]    unused       (grounded)
// - IOPIC HWI[23:16]   connected to IRQ_TTY_RX[7:0]]
// - IOPIC HWI[31:24]   connected to IRQ_TTY_TX[7:0]]
////////////////////////////////////////////////////////////////////////////
// The following parameters must be defined in the hard_config.h file :
// - X_WIDTH          : number of bits for x coordinate (must be 4)
// - Y_WIDTH          : number of bits for y coordinate (must be 4)
// - X_SIZE           : number of clusters in a row (1,2,4,8,16)
// - Y_SIZE           : number of clusters in a column (1,2,4,8)
// - NB_PROCS_MAX     : number of processors per cluster (1, 2 or 4)
// - NB_CMA_CHANNELS  : number of CMA channels in I/0 cluster (4 max)
// - NB_TTY_CHANNELS  : number of TTY channels in I/O cluster (8 max)
// - NB_NIC_CHANNELS  : number of NIC channels in I/O cluster (2 max)
// 
// Some other hardware parameters are not used when compiling the OS,
// and are only defined in this top.cpp file:
// - XRAM_LATENCY     : external ram latency 
// - MEMC_WAYS        : L2 cache number of ways
// - MEMC_SETS        : L2 cache number of sets
// - L1_IWAYS         : L1 cache instruction number of ways
// - L1_ISETS         : L1 cache instruction number of sets
// - L1_DWAYS         : L1 cache data number of ways
// - L1_DSETS         : L1 cache data number of sets
// - FBUF_X_SIZE      : width of frame buffer (pixels)
// - FBUF_Y_SIZE      : heigth of frame buffer (lines)
// - BDEV_IMAGE_NAME  : file pathname for block device 
// - NIC_RX_NAME      : file pathname for NIC received packets
// - NIC_TX_NAME      : file pathname for NIC transmited packets
// - NIC_MAC4         : MAC address
// - NIC_MAC2         : MAC address
/////////////////////////////////////////////////////////////////////////
// General policy for 40 bits physical address decoding:
// All physical segments base addresses are multiple of 1 Mbytes
// (=> the 24 LSB bits = 0, and the 16 MSB bits define the target) 
// The (X_WIDTH + Y_WIDTH) MSB bits (left aligned) define
// the cluster index, and the LADR bits define the local index:
//      |X_ID|Y_ID|  LADR |     OFFSET          |
//      |  4 |  4 |   8   |       24            |
/////////////////////////////////////////////////////////////////////////
// General policy for 14 bits SRCID decoding:
// Each component is identified by (x_id, y_id, l_id) tuple.
//      |X_ID|Y_ID| L_ID |
//      |  4 |  4 |  6   |
/////////////////////////////////////////////////////////////////////////

#include <systemc>
#include <sys/time.h>
#include <iostream>
#include <sstream>
#include <cstdlib>
#include <cstdarg>
#include <stdint.h>

#include "gdbserver.h"
#include "mapping_table.h"
#include "tsar_leti_cluster.h"
#include "vci_local_crossbar.h"
#include "vci_dspin_initiator_wrapper.h"
#include "vci_dspin_target_wrapper.h"
#include "vci_multi_tty.h"
#include "vci_multi_nic.h"
#include "vci_chbuf_dma.h"
#include "vci_block_device_tsar.h"
#include "vci_framebuffer.h"
#include "vci_iopic.h"
#include "alloc_elems.h"

//////////////////////////////////////////////////////////////////////////////////////////
//    Parameters depending on the OS and software application         
//    - path to hard_config file
//    - path to binary code for the RAM loader
//    - path to disk image for RAMDISK loader
//    - path to disk image for IOC device
//////////////////////////////////////////////////////////////////////////////////////////

#define USE_GIET_VM     1
#define USE_GIET_TSAR   0

#if ( USE_GIET_VM and USE_GIET_TSAR )
#error "Can't use Two different OS"
#endif

#if ( (not USE_GIET_VM) and (not USE_GIET_TSAR) )
#error "You need to specify one OS"
#endif

#if USE_GIET_TSAR
#include                         "../../softs/soft_transpose_giet/hard_config.h"
#define BINARY_PATH_FOR_LOADER   "../../softs/soft_transpose_giet/bin.soft"
#define RAMDISK_PATH_FOR_LOADER  "../../softs/soft_transpose_giet/images.raw@0x00800000:"
#define DISK_IMAGE_PATH_FOR_IOC  "../../softs/soft_transpose_giet/images.raw"
#endif

#if USE_GIET_VM
#include                         "../../../giet_vm/hard_config.h"
#define BINARY_PATH_FOR_LOADER   "../../softs/tsar_boot/preloader.elf"
#define RAMDISK_PATH_FOR_LOADER  "../../../giet_vm/hdd/virt_hdd.dmg@0x02000000:"
#define DISK_IMAGE_PATH_FOR_IOC  "../../../giet_vm/hdd/virt_hdd.dmg"        
#endif

///////////////////////////////////////////////////
//               Parallelisation
///////////////////////////////////////////////////
#define USE_OPENMP 0

#if USE_OPENMP
#include <omp.h>
#endif

///////////////////////////////////////////////////
//  cluster index (from x,y coordinates)
///////////////////////////////////////////////////

#define cluster(x,y)   (y + (x << Y_WIDTH))

#define min(a, b) (a < b ? a : b)

///////////////////////////////////////////////////////////
//          DSPIN parameters           
///////////////////////////////////////////////////////////

#define dspin_cmd_width      39
#define dspin_rsp_width      32

///////////////////////////////////////////////////////////
//          VCI parameters           
///////////////////////////////////////////////////////////

#define vci_cell_width_int    4
#define vci_cell_width_ext    8
#define vci_address_width     40
#define vci_plen_width        8
#define vci_rerror_width      1
#define vci_clen_width        1
#define vci_rflag_width       1
#define vci_srcid_width       14
#define vci_pktid_width       4
#define vci_trdid_width       4
#define vci_wrplen_width      1


/////////////////////////////////////////////////////////////////////////////////////////
//    Secondary Hardware Parameters         
/////////////////////////////////////////////////////////////////////////////////////////

#define RESET_ADDRESS         0x0

#define MAX_TTY_CHANNELS      8
#define MAX_CMA_CHANNELS      4
#define MAX_NIC_CHANNELS      2

#define XRAM_LATENCY          0
#define XRAM_SIZE             0x04000000    // 64 Mbytes per cluster

#define MEMC_WAYS             16
#define MEMC_SETS             256

#define L1_IWAYS              4
#define L1_ISETS              64

#define L1_DWAYS              4
#define L1_DSETS              64

#define FBUF_X_SIZE           128
#define FBUF_Y_SIZE           128

#define NIC_MAC4              0XBABEF00D
#define NIC_MAC2              0xBEEF
#define NIC_RX_NAME           "./fake"
#define NIC_TX_NAME           "./fake"

#define NORTH                 0
#define SOUTH                 1
#define EAST                  2
#define WEST                  3

///////////////////////////////////////////////////////////////////////////////////////
//     DEBUG Parameters default values         
///////////////////////////////////////////////////////////////////////////////////////

#define MAX_FROZEN_CYCLES     500000

///////////////////////////////////////////////////////////////////////////////////////
//     LOCAL TGTID & SRCID definition 
// For all components:  global TGTID = global SRCID = cluster_index
///////////////////////////////////////////////////////////////////////////////////////

#define MEMC_TGTID            0
#define XICU_TGTID            1
#define MTTY_TGTID            2
#define BDEV_TGTID            3
#define FBUF_TGTID            4
#define MNIC_TGTID            5
#define CDMA_TGTID            6
#define IOPI_TGTID            7

#define BDEV_SRCID            NB_PROCS_MAX
#define CDMA_SRCID            NB_PROCS_MAX + 1
#define IOPI_SRCID            NB_PROCS_MAX + 2

//////////////////////////////////////////////////////////////////////////////////////
//    Physical segments definition
//////////////////////////////////////////////////////////////////////////////////////
// - 3 segments are replicated in all clusters
// - 2 segments are only in cluster[0,0] 
// - 4 segments are only in cluster [X_SIZE-1,Y_SIZE] 
// The following values are for segments in cluster 0,
// and these 32 bits values must be concatenate with the cluster 
// index (on 8 bits) to obtain the 40 bits address.
//////////////////////////////////////////////////////////////////////////////////////

   // in cluster [0,0] & [X_SIZE-1,Y_SIZE] 

   #define MTTY_BASE    0xF4000000
   #define MTTY_SIZE    0x00001000   // 4 Kbytes

   #define BDEV_BASE    0xF2000000
   #define BDEV_SIZE    0x00001000   // 4 Kbytes

   // in cluster [X_SIZE-1,Y_SIZE]

   #define FBUF_BASE    0xF3000000
   #define FBUF_SIZE    (FBUF_X_SIZE * FBUF_Y_SIZE * 2)

   #define MNIC_BASE    0xF7000000
   #define MNIC_SIZE    0x00800000   // 512 Kbytes (for 8 channels)

   #define CDMA_BASE    0xF8000000
   #define CDMA_SIZE    0x00004000 * NB_CMA_CHANNELS

   #define IOPI_BASE    0xF9000000
   #define IOPI_SIZE    0x00001000   // 4 Kbytes 
   
   // replicated segments : address is extended to 40 bits by cluster_xy

   #define MEMC_BASE    0x00000000
   #define MEMC_SIZE    XRAM_SIZE

   #define MCFG_BASE    0xE0000000
   #define MCFG_SIZE    0x00001000   // 4 Kbytes

   #define XICU_BASE    0xF0000000
   #define XICU_SIZE    0x00001000   // 4 Kbytes

bool stop_called = false;

/////////////////////////////////
int _main(int argc, char *argv[])
{
   using namespace sc_core;
   using namespace soclib::caba;
   using namespace soclib::common;

   uint32_t ncycles          = 0xFFFFFFFF;         // max simulated cycles
   size_t   threads          = 1;                  // simulator's threads number
   bool     trace_ok         = false;              // trace activated
   uint32_t trace_from       = 0;                  // trace start cycle
   bool     trace_proc_ok    = false;              // detailed proc trace activated
   size_t   trace_memc_ok    = false;              // detailed memc trace activated
   size_t   trace_memc_id    = 0;                  // index of memc to be traced 
   size_t   trace_proc_id    = 0;                  // index of proc to be traced
   uint32_t frozen_cycles    = MAX_FROZEN_CYCLES;  // monitoring frozen processor
   struct   timeval t1,t2;
   uint64_t ms1,ms2;

   ////////////// command line arguments //////////////////////
   if (argc > 1)
   {
      for (int n = 1; n < argc; n = n + 2)
      {
         if ((strcmp(argv[n], "-NCYCLES") == 0) && (n + 1 < argc))
         {
            ncycles = (uint64_t) strtol(argv[n + 1], NULL, 0);
         }
         else if ((strcmp(argv[n],"-DEBUG") == 0) && (n + 1 < argc))
         {
            trace_ok = true;
            trace_from = (uint32_t) strtol(argv[n + 1], NULL, 0);
         }
         else if ((strcmp(argv[n], "-MEMCID") == 0) && (n + 1 < argc))
         {
            trace_memc_ok = true;
            trace_memc_id = (size_t) strtol(argv[n + 1], NULL, 0);
            size_t x = trace_memc_id >> Y_WIDTH;
            size_t y = trace_memc_id & ((1<<Y_WIDTH)-1);

            assert( (x < X_SIZE) and (y < (Y_SIZE)) and
                  "MEMCID parameter refers a not valid memory cache");
         }
         else if ((strcmp(argv[n], "-PROCID") == 0) && (n + 1 < argc))
         {
            trace_proc_ok = true;
            trace_proc_id = (size_t) strtol(argv[n + 1], NULL, 0);
            size_t cluster_xy = trace_proc_id / NB_PROCS_MAX ;
            size_t x          = cluster_xy >> Y_WIDTH;
            size_t y          = cluster_xy & ((1<<Y_WIDTH)-1);
            size_t l          = trace_proc_id % NB_PROCS_MAX ;

            assert( (x < X_SIZE) and (y < (Y_SIZE)) and l < NB_PROCS_MAX and
                  "PROCID parameter refers a not valid processor");
         }
         else if ((strcmp(argv[n], "-THREADS") == 0) && ((n + 1) < argc))
         {
            threads = (size_t) strtol(argv[n + 1], NULL, 0);
            threads = (threads < 1) ? 1 : threads;
         }
         else if ((strcmp(argv[n], "-FROZEN") == 0) && (n + 1 < argc))
         {
            frozen_cycles = (uint32_t) strtol(argv[n + 1], NULL, 0);
         }
         else
         {
            std::cout << "   Arguments are (key,value) couples." << std::endl;
            std::cout << "   The order is not important." << std::endl;
            std::cout << "   Accepted arguments are :" << std::endl << std::endl;
            std::cout << "     -NCYCLES number_of_simulated_cycles" << std::endl;
            std::cout << "     -DEBUG debug_start_cycle" << std::endl;
            std::cout << "     -THREADS simulator's threads number" << std::endl;
            std::cout << "     -FROZEN max_number_of_lines" << std::endl;
            std::cout << "     -PERIOD number_of_cycles between trace" << std::endl;
            std::cout << "     -MEMCID index_memc_to_be_traced" << std::endl;
            std::cout << "     -PROCID index_proc_to_be_traced" << std::endl;
            exit(0);
         }
      }
   }

    // checking hardware parameters
    assert( ((X_SIZE==1) or (X_SIZE==2) or (X_SIZE==4) or (X_SIZE==8) or (X_SIZE==16)) and 
            "Illegal X_SIZE parameter" );

    assert( ((Y_SIZE==1) or (Y_SIZE==2) or (Y_SIZE==4) or (Y_SIZE==8)) and 
            "Illegal Y_SIZE parameter" );

    assert( (NB_PROCS_MAX <= 4) and
            "Illegal NB_PROCS_MAX parameter" );

    assert( (NB_CMA_CHANNELS <= MAX_CMA_CHANNELS) and
            "The NB_CMA_CHANNELS parameter cannot be larger than 4" );

    assert( (NB_TTY_CHANNELS <= MAX_TTY_CHANNELS) and
            "The NB_TTY_CHANNELS parameter cannot be larger than 8" );

    assert( (NB_NIC_CHANNELS <= MAX_NIC_CHANNELS) and
            "The NB_NIC_CHANNELS parameter cannot be larger than 2" );

    assert( (vci_address_width == 40) and
            "VCI address width with the GIET must be 40 bits" );

    assert( (X_WIDTH == 4) and (Y_WIDTH == 4) and
            "ERROR: you must have X_WIDTH == Y_WIDTH == 4");
 
    std::cout << std::endl;

    std::cout << " - X_SIZE           = " << X_SIZE << std::endl;
    std::cout << " - Y_SIZE           = " << Y_SIZE << std::endl;
    std::cout << " - NB_PROCS_MAX     = " << NB_PROCS_MAX <<  std::endl;
    std::cout << " - NB_DMA_CHANNELS  = " << NB_DMA_CHANNELS <<  std::endl;
    std::cout << " - NB_TTY_CHANNELS  = " << NB_TTY_CHANNELS <<  std::endl;
    std::cout << " - NB_NIC_CHANNELS  = " << NB_NIC_CHANNELS <<  std::endl;
    std::cout << " - MEMC_WAYS        = " << MEMC_WAYS << std::endl;
    std::cout << " - MEMC_SETS        = " << MEMC_SETS << std::endl;
    std::cout << " - RAM_LATENCY      = " << XRAM_LATENCY << std::endl;
    std::cout << " - MAX_FROZEN       = " << frozen_cycles << std::endl;
    std::cout << " - MAX_CYCLES       = " << ncycles << std::endl;
    std::cout << " - RESET_ADDRESS    = " << RESET_ADDRESS << std::endl;
    std::cout << " - SOFT_PATH_NAME   = " << BINARY_PATH_FOR_LOADER << std::endl;
    std::cout << " - DISK_IMAGE_PATH  = " << DISK_IMAGE_PATH_FOR_IOC << std::endl;

    std::cout << std::endl;

    // Internal and External VCI parameters definition
    typedef soclib::caba::VciParams<vci_cell_width_int,
                                    vci_plen_width,
                                    vci_address_width,
                                    vci_rerror_width,
                                    vci_clen_width,
                                    vci_rflag_width,
                                    vci_srcid_width,
                                    vci_pktid_width,
                                    vci_trdid_width,
                                    vci_wrplen_width> vci_param_int;

    typedef soclib::caba::VciParams<vci_cell_width_ext,
                                    vci_plen_width,
                                    vci_address_width,
                                    vci_rerror_width,
                                    vci_clen_width,
                                    vci_rflag_width,
                                    vci_srcid_width,
                                    vci_pktid_width,
                                    vci_trdid_width,
                                    vci_wrplen_width> vci_param_ext;

#if USE_OPENMP
   omp_set_dynamic(false);
   omp_set_num_threads(threads);
   std::cerr << "Built with openmp version " << _OPENMP << std::endl;
#endif


   ///////////////////////////////////////
   //  Direct Network Mapping Table
   ///////////////////////////////////////

   MappingTable maptabd(vci_address_width, 
                        IntTab(X_WIDTH + Y_WIDTH, 16 - X_WIDTH - Y_WIDTH), 
                        IntTab(X_WIDTH + Y_WIDTH, vci_srcid_width - X_WIDTH - Y_WIDTH), 
                        0x00FF000000ULL);

   // replicated segments
   for (size_t x = 0; x < X_SIZE; x++)
   {
      for (size_t y = 0; y < (Y_SIZE) ; y++)
      {
         sc_uint<vci_address_width> offset;
         offset = ((sc_uint<vci_address_width>)cluster(x,y)) << 32;

         std::ostringstream    si;
         si << "seg_xicu_" << x << "_" << y;
         maptabd.add(Segment(si.str(), XICU_BASE + offset, XICU_SIZE, 
                  IntTab(cluster(x,y),XICU_TGTID), false));

         std::ostringstream    sd;
         sd << "seg_mcfg_" << x << "_" << y;
         maptabd.add(Segment(sd.str(), MCFG_BASE + offset, MCFG_SIZE, 
                  IntTab(cluster(x,y),MEMC_TGTID), false));

         std::ostringstream    sh;
         sh << "seg_memc_" << x << "_" << y;
         maptabd.add(Segment(sh.str(), MEMC_BASE + offset, MEMC_SIZE, 
                  IntTab(cluster(x,y),MEMC_TGTID), true));
      }
   }

   // segments for peripherals in cluster(0,0)
   maptabd.add(Segment("seg_tty0", MTTY_BASE, MTTY_SIZE, 
               IntTab(cluster(0,0),MTTY_TGTID), false));

   maptabd.add(Segment("seg_ioc0", BDEV_BASE, BDEV_SIZE, 
               IntTab(cluster(0,0),BDEV_TGTID), false));

   // segments for peripherals in cluster_io (X_SIZE-1,Y_SIZE)
   sc_uint<vci_address_width> offset;
   offset = ((sc_uint<vci_address_width>)cluster(X_SIZE-1,Y_SIZE)) << 32;

   maptabd.add(Segment("seg_mtty", MTTY_BASE + offset, MTTY_SIZE, 
               IntTab(cluster(X_SIZE-1, Y_SIZE),MTTY_TGTID), false));

   maptabd.add(Segment("seg_fbuf", FBUF_BASE + offset, FBUF_SIZE, 
               IntTab(cluster(X_SIZE-1, Y_SIZE),FBUF_TGTID), false));

   maptabd.add(Segment("seg_bdev", BDEV_BASE + offset, BDEV_SIZE, 
               IntTab(cluster(X_SIZE-1, Y_SIZE),BDEV_TGTID), false));

   maptabd.add(Segment("seg_mnic", MNIC_BASE + offset, MNIC_SIZE, 
               IntTab(cluster(X_SIZE-1, Y_SIZE),MNIC_TGTID), false));

   maptabd.add(Segment("seg_cdma", CDMA_BASE + offset, CDMA_SIZE, 
               IntTab(cluster(X_SIZE-1, Y_SIZE),CDMA_TGTID), false));

   maptabd.add(Segment("seg_iopi", IOPI_BASE + offset, IOPI_SIZE, 
               IntTab(cluster(X_SIZE-1, Y_SIZE),IOPI_TGTID), false));

   std::cout << maptabd << std::endl;

    /////////////////////////////////////////////////
    // Ram network mapping table
    /////////////////////////////////////////////////

    MappingTable maptabx(vci_address_width, 
                         IntTab(X_WIDTH+Y_WIDTH), 
                         IntTab(X_WIDTH+Y_WIDTH), 
                         0x00FF000000ULL);

    for (size_t x = 0; x < X_SIZE; x++)
    {
        for (size_t y = 0; y < (Y_SIZE) ; y++)
        { 
            sc_uint<vci_address_width> offset;
            offset = (sc_uint<vci_address_width>)cluster(x,y) 
                      << (vci_address_width-X_WIDTH-Y_WIDTH);

            std::ostringstream sh;
            sh << "x_seg_memc_" << x << "_" << y;

            maptabx.add(Segment(sh.str(), MEMC_BASE + offset, 
                     MEMC_SIZE, IntTab(cluster(x,y)), false));
        }
    }
    std::cout << maptabx << std::endl;

    ////////////////////
    // Signals
    ///////////////////

    sc_clock                          signal_clk("clk");
    sc_signal<bool>                   signal_resetn("resetn");

    // IRQs from external peripherals
    sc_signal<bool>                   signal_irq_bdev;
    sc_signal<bool>                   signal_irq_mnic_rx[NB_NIC_CHANNELS];
    sc_signal<bool>                   signal_irq_mnic_tx[NB_NIC_CHANNELS];
    sc_signal<bool>                   signal_irq_mtty_rx[NB_TTY_CHANNELS];
//  sc_signal<bool>                   signal_irq_mtty_tx[NB_TTY_CHANNELS];
    sc_signal<bool>                   signal_irq_cdma[NB_CMA_CHANNELS];
    sc_signal<bool>                   signal_irq_false;

   // Horizontal inter-clusters DSPIN signals
   DspinSignals<dspin_cmd_width>** signal_dspin_h_cmd_inc =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_h_cmd_inc", X_SIZE-1, Y_SIZE);
   DspinSignals<dspin_cmd_width>** signal_dspin_h_cmd_dec =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_h_cmd_dec", X_SIZE-1, Y_SIZE);

   DspinSignals<dspin_rsp_width>** signal_dspin_h_rsp_inc =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_h_rsp_inc", X_SIZE-1, Y_SIZE);
   DspinSignals<dspin_rsp_width>** signal_dspin_h_rsp_dec =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_h_rsp_dec", X_SIZE-1, Y_SIZE);

   DspinSignals<dspin_cmd_width>** signal_dspin_h_m2p_inc =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_h_m2p_inc", X_SIZE-1, Y_SIZE);
   DspinSignals<dspin_cmd_width>** signal_dspin_h_m2p_dec =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_h_m2p_dec", X_SIZE-1, Y_SIZE);

   DspinSignals<dspin_rsp_width>** signal_dspin_h_p2m_inc =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_h_p2m_inc", X_SIZE-1, Y_SIZE);
   DspinSignals<dspin_rsp_width>** signal_dspin_h_p2m_dec =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_h_p2m_dec", X_SIZE-1, Y_SIZE);

   DspinSignals<dspin_cmd_width>** signal_dspin_h_cla_inc =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_h_cla_inc", X_SIZE-1, Y_SIZE);
   DspinSignals<dspin_cmd_width>** signal_dspin_h_cla_dec =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_h_cla_dec", X_SIZE-1, Y_SIZE);

   // Vertical inter-clusters DSPIN signals
   DspinSignals<dspin_cmd_width>** signal_dspin_v_cmd_inc =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_v_cmd_inc", X_SIZE, Y_SIZE-1);
   DspinSignals<dspin_cmd_width>** signal_dspin_v_cmd_dec =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_v_cmd_dec", X_SIZE, Y_SIZE-1);

   DspinSignals<dspin_rsp_width>** signal_dspin_v_rsp_inc =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_v_rsp_inc", X_SIZE, Y_SIZE-1);
   DspinSignals<dspin_rsp_width>** signal_dspin_v_rsp_dec =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_v_rsp_dec", X_SIZE, Y_SIZE-1);

   DspinSignals<dspin_cmd_width>** signal_dspin_v_m2p_inc =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_v_m2p_inc", X_SIZE, Y_SIZE-1);
   DspinSignals<dspin_cmd_width>** signal_dspin_v_m2p_dec =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_v_m2p_dec", X_SIZE, Y_SIZE-1);

   DspinSignals<dspin_rsp_width>** signal_dspin_v_p2m_inc =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_v_p2m_inc", X_SIZE, Y_SIZE-1);
   DspinSignals<dspin_rsp_width>** signal_dspin_v_p2m_dec =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_v_p2m_dec", X_SIZE, Y_SIZE-1);

   DspinSignals<dspin_cmd_width>** signal_dspin_v_cla_inc =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_v_cla_inc", X_SIZE, Y_SIZE-1);
   DspinSignals<dspin_cmd_width>** signal_dspin_v_cla_dec =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_v_cla_dec", X_SIZE, Y_SIZE-1);

   // Mesh boundaries DSPIN signals (Most of those signals are not used...)
   DspinSignals<dspin_cmd_width>*** signal_dspin_bound_cmd_in =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_bound_cmd_in" , X_SIZE, Y_SIZE, 4);
   DspinSignals<dspin_cmd_width>*** signal_dspin_bound_cmd_out =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_bound_cmd_out", X_SIZE, Y_SIZE, 4);

   DspinSignals<dspin_rsp_width>*** signal_dspin_bound_rsp_in =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_bound_rsp_in" , X_SIZE, Y_SIZE, 4);
   DspinSignals<dspin_rsp_width>*** signal_dspin_bound_rsp_out =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_bound_rsp_out", X_SIZE, Y_SIZE, 4);

   DspinSignals<dspin_cmd_width>*** signal_dspin_bound_m2p_in =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_bound_m2p_in" , X_SIZE, Y_SIZE, 4);
   DspinSignals<dspin_cmd_width>*** signal_dspin_bound_m2p_out =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_bound_m2p_out", X_SIZE, Y_SIZE, 4);

   DspinSignals<dspin_rsp_width>*** signal_dspin_bound_p2m_in =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_bound_p2m_in" , X_SIZE, Y_SIZE, 4);
   DspinSignals<dspin_rsp_width>*** signal_dspin_bound_p2m_out =
      alloc_elems<DspinSignals<dspin_rsp_width> >("signal_dspin_bound_p2m_out", X_SIZE, Y_SIZE, 4);

   DspinSignals<dspin_cmd_width>*** signal_dspin_bound_cla_in =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_bound_cla_in" , X_SIZE, Y_SIZE, 4);
   DspinSignals<dspin_cmd_width>*** signal_dspin_bound_cla_out =
      alloc_elems<DspinSignals<dspin_cmd_width> >("signal_dspin_bound_cla_out", X_SIZE, Y_SIZE, 4);

   // VCI signals for iobus and peripherals
   VciSignals<vci_param_int>    signal_vci_ini_bdev("signal_vci_ini_bdev");
   VciSignals<vci_param_int>    signal_vci_ini_cdma("signal_vci_ini_cdma");
   VciSignals<vci_param_int>    signal_vci_ini_iopi("signal_vci_ini_iopi");

   VciSignals<vci_param_int>*   signal_vci_ini_proc = 
       alloc_elems<VciSignals<vci_param_int> >("signal_vci_ini_proc", NB_PROCS_MAX );

   VciSignals<vci_param_int>    signal_vci_tgt_memc("signal_vci_tgt_memc");
   VciSignals<vci_param_int>    signal_vci_tgt_xicu("signal_vci_tgt_xicu");
   VciSignals<vci_param_int>    signal_vci_tgt_bdev("signal_vci_tgt_bdev");
   VciSignals<vci_param_int>    signal_vci_tgt_mtty("signal_vci_tgt_mtty");
   VciSignals<vci_param_int>    signal_vci_tgt_fbuf("signal_vci_tgt_fbuf");
   VciSignals<vci_param_int>    signal_vci_tgt_mnic("signal_vci_tgt_mnic");
   VciSignals<vci_param_int>    signal_vci_tgt_cdma("signal_vci_tgt_cdma");
   VciSignals<vci_param_int>    signal_vci_tgt_iopi("signal_vci_tgt_iopi");

   VciSignals<vci_param_int>    signal_vci_cmd_to_noc("signal_vci_cmd_to_noc");
   VciSignals<vci_param_int>    signal_vci_cmd_from_noc("signal_vci_cmd_from_noc");
   
   ////////////////////////////
   //      Loader    
   ////////////////////////////

#if USE_IOC_RDK
   soclib::common::Loader loader( BINARY_PATH_FOR_LOADER, RAMDISK_PATH_FOR_LOADER );
#else
   soclib::common::Loader loader( BINARY_PATH_FOR_LOADER );
#endif

   ///////////////////////////
   //  processor iss
   ///////////////////////////

   typedef soclib::common::GdbServer<soclib::common::Mips32ElIss> proc_iss;
   proc_iss::set_loader( loader );

   //////////////////////////////////////////////////////////////
   // mesh construction: only (X_SIZE) * (Y_SIZE) clusters
   //////////////////////////////////////////////////////////////

   TsarLetiCluster<dspin_cmd_width,
                   dspin_rsp_width,
                   vci_param_int,
                   vci_param_ext>*          clusters[X_SIZE][Y_SIZE];

#if USE_OPENMP
#pragma omp parallel
    {
#pragma omp for
#endif
        for (size_t i = 0; i  < (X_SIZE * (Y_SIZE)); i++)
        {
            size_t x = i / (Y_SIZE);
            size_t y = i % (Y_SIZE);

#if USE_OPENMP
#pragma omp critical
            {
#endif
            std::cout << std::endl;
            std::cout << "Cluster_" << std::dec << x << "_" << y 
                      << " with cluster_xy = " << std::hex << cluster(x,y) << std::endl;
            std::cout << std::endl;

            std::ostringstream cluster_name;
            cluster_name <<  "cluster_" << std::dec << x << "_" << y;

            clusters[x][y] = new TsarLetiCluster<dspin_cmd_width,
                                                 dspin_rsp_width,
                                                 vci_param_int,
                                                 vci_param_ext>
            (
                cluster_name.str().c_str(),
                NB_PROCS_MAX,
                x,
                y,
                cluster(x,y),
                maptabd,
                maptabx,
                RESET_ADDRESS,
                X_WIDTH,
                Y_WIDTH,
                vci_srcid_width - X_WIDTH - Y_WIDTH,   // l_id width,
                MEMC_TGTID,
                XICU_TGTID,
                MTTY_TGTID,
                BDEV_TGTID,
                DISK_IMAGE_PATH_FOR_IOC,
                MEMC_WAYS,
                MEMC_SETS,
                L1_IWAYS,
                L1_ISETS,
                L1_DWAYS,
                L1_DSETS,
                XRAM_LATENCY,
                loader,
                frozen_cycles,
                trace_from,
                trace_proc_ok, 
                trace_proc_id,
                trace_memc_ok, 
                trace_memc_id
            );

#if USE_OPENMP
            } // end critical
#endif
        } // end for
#if USE_OPENMP
    }
#endif

    //////////////////////////////////////////////////////////////////
    // IO bus and external peripherals in cluster[X_SIZE-1,Y_SIZE]
    // - 6 local targets    : FBF, TTY, CMA, NIC, PIC, IOC
    // - 3 local initiators : IOC, CMA, PIC
    // There is no PROC, no MEMC and no XICU in this cluster,
    // but the crossbar has (NB_PROCS_MAX + 3) intiators and
    // 8 targets, in order to use the same SRCID and TGTID space
    // (same mapping table for the internal components, 
    //  and for the external peripherals) 
    //////////////////////////////////////////////////////////////////

    std::cout << std::endl;
    std::cout << " Building IO cluster (external peripherals)" << std::endl;
    std::cout << std::endl;
  
    size_t cluster_io = cluster(X_SIZE-1, Y_SIZE);

    //////////// vci_local_crossbar  
    VciLocalCrossbar<vci_param_int>*
    iobus = new VciLocalCrossbar<vci_param_int>(
                "iobus",
                maptabd,                      // mapping table
                cluster_io,                   // cluster_xy
                NB_PROCS_MAX + 3,             // number of local initiators
                8,                            // number of local targets
                BDEV_TGTID );                 // default target index

    //////////// vci_framebuffer                        
    VciFrameBuffer<vci_param_int>*
    fbuf = new VciFrameBuffer<vci_param_int>(
                "fbuf",
                IntTab(cluster_io, FBUF_TGTID),
                maptabd,
                FBUF_X_SIZE, FBUF_Y_SIZE );

    ////////////  vci_block_device               
    VciBlockDeviceTsar<vci_param_int>*
    bdev = new VciBlockDeviceTsar<vci_param_int>(
                "bdev",
                maptabd,
                IntTab(cluster_io, BDEV_SRCID),
                IntTab(cluster_io, BDEV_TGTID),
                DISK_IMAGE_PATH_FOR_IOC,
                512,                          // block size
                64 );                         // burst size

    //////////// vci_multi_nic               
    VciMultiNic<vci_param_int>*
    mnic = new VciMultiNic<vci_param_int>(
             "mnic",
                IntTab(cluster_io, MNIC_TGTID),
                maptabd,
                NB_NIC_CHANNELS,
                NIC_MAC4,
                NIC_MAC2,
                NIC_RX_NAME,
                NIC_TX_NAME );

    ///////////// vci_chbuf_dma                   
    VciChbufDma<vci_param_int>*
    cdma = new VciChbufDma<vci_param_int>(
                "cdma",
                maptabd,
                IntTab(cluster_io, CDMA_SRCID),
                IntTab(cluster_io, CDMA_TGTID),
                64,                          // burst size
                NB_CMA_CHANNELS ); 

    ////////////// vci_multi_tty
    std::vector<std::string> vect_names;
    for (size_t id = 0; id < NB_TTY_CHANNELS; id++)
    {
        std::ostringstream term_name;
        term_name <<  "ext_" << id;
        vect_names.push_back(term_name.str().c_str());
    }

    VciMultiTty<vci_param_int>* 
    mtty = new VciMultiTty<vci_param_int>(
                "mtty",
                IntTab(cluster_io, MTTY_TGTID),
                maptabd,
                vect_names );

    ///////////// vci_iopic
    VciIopic<vci_param_int>*
    iopic = new VciIopic<vci_param_int>(
                "iopic",
                maptabd,
                IntTab(cluster_io, IOPI_SRCID),
                IntTab(cluster_io, IOPI_TGTID),
                32,
                5000 );

    ////////////// vci_dspin wrappers 
    VciDspinTargetWrapper<vci_param_int, dspin_cmd_width, dspin_rsp_width>*
    wt_iobus = new VciDspinTargetWrapper<vci_param_int, dspin_cmd_width, dspin_rsp_width>(
                "wt_bdev",
                vci_srcid_width );

    VciDspinInitiatorWrapper<vci_param_int, dspin_cmd_width, dspin_rsp_width>*
    wi_iobus = new VciDspinInitiatorWrapper<vci_param_int, dspin_cmd_width, dspin_rsp_width>(
                "wi_bdev",
                vci_srcid_width );

    ///////////////////////////////////////////////////////////////
    //     Net-list 
    ///////////////////////////////////////////////////////////////

    // iobus 
    iobus->p_clk                       (signal_clk);  
    iobus->p_resetn                    (signal_resetn);

    iobus->p_target_to_up              (signal_vci_cmd_from_noc);
    iobus->p_initiator_to_up           (signal_vci_cmd_to_noc);

    iobus->p_to_target[MEMC_TGTID]     (signal_vci_tgt_memc);
    iobus->p_to_target[XICU_TGTID]     (signal_vci_tgt_xicu);
    iobus->p_to_target[MTTY_TGTID]     (signal_vci_tgt_mtty);
    iobus->p_to_target[FBUF_TGTID]     (signal_vci_tgt_fbuf);
    iobus->p_to_target[MNIC_TGTID]     (signal_vci_tgt_mnic);
    iobus->p_to_target[BDEV_TGTID]     (signal_vci_tgt_bdev);
    iobus->p_to_target[CDMA_TGTID]     (signal_vci_tgt_cdma);
    iobus->p_to_target[IOPI_TGTID]     (signal_vci_tgt_iopi);

    for( size_t p=0 ; p<NB_PROCS_MAX ; p++ )
    {
        iobus->p_to_initiator[p]       (signal_vci_ini_proc[p]);
    }
    iobus->p_to_initiator[BDEV_SRCID]  (signal_vci_ini_bdev);
    iobus->p_to_initiator[CDMA_SRCID]  (signal_vci_ini_cdma);
    iobus->p_to_initiator[IOPI_SRCID]  (signal_vci_ini_iopi);

    std::cout << "  - IOBUS connected" << std::endl;

    // block_device
    bdev->p_clk                        (signal_clk);
    bdev->p_resetn                     (signal_resetn);
    bdev->p_vci_target                 (signal_vci_tgt_bdev);
    bdev->p_vci_initiator              (signal_vci_ini_bdev);
    bdev->p_irq                        (signal_irq_bdev);

    std::cout << "  - BDEV connected" << std::endl;

    // frame_buffer
    fbuf->p_clk                        (signal_clk);
    fbuf->p_resetn                     (signal_resetn);
    fbuf->p_vci                        (signal_vci_tgt_fbuf);

    std::cout << "  - FBUF connected" << std::endl;

    // multi_nic
    mnic->p_clk                        (signal_clk);
    mnic->p_resetn                     (signal_resetn);
    mnic->p_vci                        (signal_vci_tgt_mnic);
    for ( size_t i=0 ; i<NB_NIC_CHANNELS ; i++ )
    {
         mnic->p_rx_irq[i]             (signal_irq_mnic_rx[i]);
         mnic->p_tx_irq[i]             (signal_irq_mnic_tx[i]);
    }

    std::cout << "  - MNIC connected" << std::endl;

    // chbuf_dma
    cdma->p_clk                        (signal_clk);
    cdma->p_resetn                     (signal_resetn);
    cdma->p_vci_target                 (signal_vci_tgt_cdma);
    cdma->p_vci_initiator              (signal_vci_ini_cdma);
    for ( size_t i=0 ; i<NB_CMA_CHANNELS ; i++)
    {
        cdma->p_irq[i]                 (signal_irq_cdma[i]);
    }

    std::cout << "  - CDMA connected" << std::endl;

    // multi_tty
    mtty->p_clk                        (signal_clk);
    mtty->p_resetn                     (signal_resetn);
    mtty->p_vci                        (signal_vci_tgt_mtty);
    for ( size_t i=0 ; i<NB_TTY_CHANNELS ; i++ )
    {
        mtty->p_irq[i]                  (signal_irq_mtty_rx[i]);
    }

    std::cout << "  - MTTY connected" << std::endl;

    // iopic
    // NB_NIC_CHANNELS <= 2
    // NB_CMA_CHANNELS <= 4
    // NB_TTY_CHANNELS <= 8
    iopic->p_clk                       (signal_clk);
    iopic->p_resetn                    (signal_resetn);
    iopic->p_vci_target                (signal_vci_tgt_iopi);
    iopic->p_vci_initiator             (signal_vci_ini_iopi);
    for ( size_t i=0 ; i<32 ; i++)
    {
       if     (i < NB_NIC_CHANNELS)    iopic->p_hwi[i] (signal_irq_mnic_rx[i]);
       else if(i < 2 )                 iopic->p_hwi[i] (signal_irq_false);
       else if(i < 2+NB_NIC_CHANNELS)  iopic->p_hwi[i] (signal_irq_mnic_tx[i-2]);
       else if(i < 4 )                 iopic->p_hwi[i] (signal_irq_false);
       else if(i < 4+NB_CMA_CHANNELS)  iopic->p_hwi[i] (signal_irq_cdma[i-4]);
       else if(i < 8)                  iopic->p_hwi[i] (signal_irq_false);
       else if(i == 8)                 iopic->p_hwi[i] (signal_irq_bdev);
       else if(i < 16)                 iopic->p_hwi[i] (signal_irq_false);
       else if(i < 16+NB_TTY_CHANNELS) iopic->p_hwi[i] (signal_irq_mtty_rx[i-16]);
       else if(i < 24)                 iopic->p_hwi[i] (signal_irq_false);
       else if(i < 24+NB_TTY_CHANNELS) iopic->p_hwi[i] (signal_irq_false);
//     else if(i < 24+NB_TTY_CHANNELS) iopic->p_hwi[i] (signal_irq_mtty_tx[i-24]);
       else                            iopic->p_hwi[i] (signal_irq_false);
    }

    std::cout << "  - IOPIC connected" << std::endl;

    // vci/dspin wrappers
    wi_iobus->p_clk                    (signal_clk);
    wi_iobus->p_resetn                 (signal_resetn);
    wi_iobus->p_vci                    (signal_vci_cmd_to_noc);
    wi_iobus->p_dspin_cmd              (signal_dspin_bound_cmd_in[X_SIZE-1][Y_SIZE-1][NORTH]);
    wi_iobus->p_dspin_rsp              (signal_dspin_bound_rsp_out[X_SIZE-1][Y_SIZE-1][NORTH]);

    // vci/dspin wrappers
    wt_iobus->p_clk                    (signal_clk);
    wt_iobus->p_resetn                 (signal_resetn);
    wt_iobus->p_vci                    (signal_vci_cmd_from_noc);
    wt_iobus->p_dspin_cmd              (signal_dspin_bound_cmd_out[X_SIZE-1][Y_SIZE-1][NORTH]);
    wt_iobus->p_dspin_rsp              (signal_dspin_bound_rsp_in[X_SIZE-1][Y_SIZE-1][NORTH]);

    // Clock & RESET for clusters
    for (size_t x = 0; x < (X_SIZE); x++)
    {
        for (size_t y = 0; y < (Y_SIZE); y++)
        {
            clusters[x][y]->p_clk                    (signal_clk);
            clusters[x][y]->p_resetn                 (signal_resetn);
        }
    }

    // Inter Clusters horizontal connections
    if (X_SIZE > 1)
    {
        for (size_t x = 0; x < (X_SIZE-1); x++)
        {
            for (size_t y = 0; y < (Y_SIZE); y++)
            {
                clusters[x][y]->p_cmd_out[EAST]      (signal_dspin_h_cmd_inc[x][y]);
                clusters[x+1][y]->p_cmd_in[WEST]     (signal_dspin_h_cmd_inc[x][y]);
                clusters[x][y]->p_cmd_in[EAST]       (signal_dspin_h_cmd_dec[x][y]);
                clusters[x+1][y]->p_cmd_out[WEST]    (signal_dspin_h_cmd_dec[x][y]);

                clusters[x][y]->p_rsp_out[EAST]      (signal_dspin_h_rsp_inc[x][y]);
                clusters[x+1][y]->p_rsp_in[WEST]     (signal_dspin_h_rsp_inc[x][y]);
                clusters[x][y]->p_rsp_in[EAST]       (signal_dspin_h_rsp_dec[x][y]);
                clusters[x+1][y]->p_rsp_out[WEST]    (signal_dspin_h_rsp_dec[x][y]);

                clusters[x][y]->p_m2p_out[EAST]      (signal_dspin_h_m2p_inc[x][y]);
                clusters[x+1][y]->p_m2p_in[WEST]     (signal_dspin_h_m2p_inc[x][y]);
                clusters[x][y]->p_m2p_in[EAST]       (signal_dspin_h_m2p_dec[x][y]);
                clusters[x+1][y]->p_m2p_out[WEST]    (signal_dspin_h_m2p_dec[x][y]);

                clusters[x][y]->p_p2m_out[EAST]      (signal_dspin_h_p2m_inc[x][y]);
                clusters[x+1][y]->p_p2m_in[WEST]     (signal_dspin_h_p2m_inc[x][y]);
                clusters[x][y]->p_p2m_in[EAST]       (signal_dspin_h_p2m_dec[x][y]);
                clusters[x+1][y]->p_p2m_out[WEST]    (signal_dspin_h_p2m_dec[x][y]);

                clusters[x][y]->p_cla_out[EAST]      (signal_dspin_h_cla_inc[x][y]);
                clusters[x+1][y]->p_cla_in[WEST]     (signal_dspin_h_cla_inc[x][y]);
                clusters[x][y]->p_cla_in[EAST]       (signal_dspin_h_cla_dec[x][y]);
                clusters[x+1][y]->p_cla_out[WEST]    (signal_dspin_h_cla_dec[x][y]);
            }
        }
    }
    std::cout << std::endl << "Horizontal connections done" << std::endl;   

    // Inter Clusters vertical connections
    if (Y_SIZE > 1) 
    {
        for (size_t y = 0; y < (Y_SIZE-1); y++)
        {
            for (size_t x = 0; x < X_SIZE; x++)
            {
                clusters[x][y]->p_cmd_out[NORTH]     (signal_dspin_v_cmd_inc[x][y]);
                clusters[x][y+1]->p_cmd_in[SOUTH]    (signal_dspin_v_cmd_inc[x][y]);
                clusters[x][y]->p_cmd_in[NORTH]      (signal_dspin_v_cmd_dec[x][y]);
                clusters[x][y+1]->p_cmd_out[SOUTH]   (signal_dspin_v_cmd_dec[x][y]);

                clusters[x][y]->p_rsp_out[NORTH]     (signal_dspin_v_rsp_inc[x][y]);
                clusters[x][y+1]->p_rsp_in[SOUTH]    (signal_dspin_v_rsp_inc[x][y]);
                clusters[x][y]->p_rsp_in[NORTH]      (signal_dspin_v_rsp_dec[x][y]);
                clusters[x][y+1]->p_rsp_out[SOUTH]   (signal_dspin_v_rsp_dec[x][y]);

                clusters[x][y]->p_m2p_out[NORTH]     (signal_dspin_v_m2p_inc[x][y]);
                clusters[x][y+1]->p_m2p_in[SOUTH]    (signal_dspin_v_m2p_inc[x][y]);
                clusters[x][y]->p_m2p_in[NORTH]      (signal_dspin_v_m2p_dec[x][y]);
                clusters[x][y+1]->p_m2p_out[SOUTH]   (signal_dspin_v_m2p_dec[x][y]);

                clusters[x][y]->p_p2m_out[NORTH]     (signal_dspin_v_p2m_inc[x][y]);
                clusters[x][y+1]->p_p2m_in[SOUTH]    (signal_dspin_v_p2m_inc[x][y]);
                clusters[x][y]->p_p2m_in[NORTH]      (signal_dspin_v_p2m_dec[x][y]);
                clusters[x][y+1]->p_p2m_out[SOUTH]   (signal_dspin_v_p2m_dec[x][y]);

                clusters[x][y]->p_cla_out[NORTH]     (signal_dspin_v_cla_inc[x][y]);
                clusters[x][y+1]->p_cla_in[SOUTH]    (signal_dspin_v_cla_inc[x][y]);
                clusters[x][y]->p_cla_in[NORTH]      (signal_dspin_v_cla_dec[x][y]);
                clusters[x][y+1]->p_cla_out[SOUTH]   (signal_dspin_v_cla_dec[x][y]);
            }
        }
    }
    std::cout << std::endl << "Vertical connections done" << std::endl;

    // East & West boundary cluster connections
    for (size_t y = 0; y < (Y_SIZE); y++)
    {
        clusters[0][y]->p_cmd_in[WEST]           (signal_dspin_bound_cmd_in[0][y][WEST]);
        clusters[0][y]->p_cmd_out[WEST]          (signal_dspin_bound_cmd_out[0][y][WEST]);
        clusters[X_SIZE-1][y]->p_cmd_in[EAST]    (signal_dspin_bound_cmd_in[X_SIZE-1][y][EAST]);
        clusters[X_SIZE-1][y]->p_cmd_out[EAST]   (signal_dspin_bound_cmd_out[X_SIZE-1][y][EAST]);

        clusters[0][y]->p_rsp_in[WEST]           (signal_dspin_bound_rsp_in[0][y][WEST]);
        clusters[0][y]->p_rsp_out[WEST]          (signal_dspin_bound_rsp_out[0][y][WEST]);
        clusters[X_SIZE-1][y]->p_rsp_in[EAST]    (signal_dspin_bound_rsp_in[X_SIZE-1][y][EAST]);
        clusters[X_SIZE-1][y]->p_rsp_out[EAST]   (signal_dspin_bound_rsp_out[X_SIZE-1][y][EAST]);

        clusters[0][y]->p_m2p_in[WEST]           (signal_dspin_bound_m2p_in[0][y][WEST]);
        clusters[0][y]->p_m2p_out[WEST]          (signal_dspin_bound_m2p_out[0][y][WEST]);
        clusters[X_SIZE-1][y]->p_m2p_in[EAST]    (signal_dspin_bound_m2p_in[X_SIZE-1][y][EAST]);
        clusters[X_SIZE-1][y]->p_m2p_out[EAST]   (signal_dspin_bound_m2p_out[X_SIZE-1][y][EAST]);

        clusters[0][y]->p_p2m_in[WEST]           (signal_dspin_bound_p2m_in[0][y][WEST]);
        clusters[0][y]->p_p2m_out[WEST]          (signal_dspin_bound_p2m_out[0][y][WEST]);
        clusters[X_SIZE-1][y]->p_p2m_in[EAST]    (signal_dspin_bound_p2m_in[X_SIZE-1][y][EAST]);
        clusters[X_SIZE-1][y]->p_p2m_out[EAST]   (signal_dspin_bound_p2m_out[X_SIZE-1][y][EAST]);

        clusters[0][y]->p_cla_in[WEST]           (signal_dspin_bound_cla_in[0][y][WEST]);
        clusters[0][y]->p_cla_out[WEST]          (signal_dspin_bound_cla_out[0][y][WEST]);
        clusters[X_SIZE-1][y]->p_cla_in[EAST]    (signal_dspin_bound_cla_in[X_SIZE-1][y][EAST]);
        clusters[X_SIZE-1][y]->p_cla_out[EAST]   (signal_dspin_bound_cla_out[X_SIZE-1][y][EAST]);
    }

    std::cout << std::endl << "West & East boundaries connections done" << std::endl;

    // North & South boundary clusters connections
    for (size_t x = 0; x < X_SIZE; x++)
    {
        clusters[x][0]->p_cmd_in[SOUTH]          (signal_dspin_bound_cmd_in[x][0][SOUTH]);
        clusters[x][0]->p_cmd_out[SOUTH]         (signal_dspin_bound_cmd_out[x][0][SOUTH]);
        clusters[x][Y_SIZE-1]->p_cmd_in[NORTH]   (signal_dspin_bound_cmd_in[x][Y_SIZE-1][NORTH]);
        clusters[x][Y_SIZE-1]->p_cmd_out[NORTH]  (signal_dspin_bound_cmd_out[x][Y_SIZE-1][NORTH]);

        clusters[x][0]->p_rsp_in[SOUTH]          (signal_dspin_bound_rsp_in[x][0][SOUTH]);
        clusters[x][0]->p_rsp_out[SOUTH]         (signal_dspin_bound_rsp_out[x][0][SOUTH]);
        clusters[x][Y_SIZE-1]->p_rsp_in[NORTH]   (signal_dspin_bound_rsp_in[x][Y_SIZE-1][NORTH]);
        clusters[x][Y_SIZE-1]->p_rsp_out[NORTH]  (signal_dspin_bound_rsp_out[x][Y_SIZE-1][NORTH]);

        clusters[x][0]->p_m2p_in[SOUTH]          (signal_dspin_bound_m2p_in[x][0][SOUTH]);
        clusters[x][0]->p_m2p_out[SOUTH]         (signal_dspin_bound_m2p_out[x][0][SOUTH]);
        clusters[x][Y_SIZE-1]->p_m2p_in[NORTH]   (signal_dspin_bound_m2p_in[x][Y_SIZE-1][NORTH]);
        clusters[x][Y_SIZE-1]->p_m2p_out[NORTH]  (signal_dspin_bound_m2p_out[x][Y_SIZE-1][NORTH]);

        clusters[x][0]->p_p2m_in[SOUTH]          (signal_dspin_bound_p2m_in[x][0][SOUTH]);
        clusters[x][0]->p_p2m_out[SOUTH]         (signal_dspin_bound_p2m_out[x][0][SOUTH]);
        clusters[x][Y_SIZE-1]->p_p2m_in[NORTH]   (signal_dspin_bound_p2m_in[x][Y_SIZE-1][NORTH]);
        clusters[x][Y_SIZE-1]->p_p2m_out[NORTH]  (signal_dspin_bound_p2m_out[x][Y_SIZE-1][NORTH]);

        clusters[x][0]->p_cla_in[SOUTH]          (signal_dspin_bound_cla_in[x][0][SOUTH]);
        clusters[x][0]->p_cla_out[SOUTH]         (signal_dspin_bound_cla_out[x][0][SOUTH]);
        clusters[x][Y_SIZE-1]->p_cla_in[NORTH]   (signal_dspin_bound_cla_in[x][Y_SIZE-1][NORTH]);
        clusters[x][Y_SIZE-1]->p_cla_out[NORTH]  (signal_dspin_bound_cla_out[x][Y_SIZE-1][NORTH]);
    }

    std::cout << std::endl << "North & South boundaries connections done" << std::endl;

    std::cout << std::endl;

    ////////////////////////////////////////////////////////
    //   Simulation
    ///////////////////////////////////////////////////////

    sc_start(sc_core::sc_time(0, SC_NS));
    signal_resetn    = false;
    signal_irq_false = false;

    // set network boundaries signals default values
    // for all boundary clusters but the IO cluster
    for (size_t x = 0; x < X_SIZE ; x++)
    {
        for (size_t y = 0; y < Y_SIZE ; y++)
        {
            for (size_t face = 0; face < 4; face++)
            {
                if ( (x != X_SIZE-1) or (y != Y_SIZE-1) or (face != NORTH) )
                {
                    signal_dspin_bound_cmd_in [x][y][face].write = false;
                    signal_dspin_bound_cmd_in [x][y][face].read  = true;
                    signal_dspin_bound_cmd_out[x][y][face].write = false;
                    signal_dspin_bound_cmd_out[x][y][face].read  = true;

                    signal_dspin_bound_rsp_in [x][y][face].write = false;
                    signal_dspin_bound_rsp_in [x][y][face].read  = true;
                    signal_dspin_bound_rsp_out[x][y][face].write = false;
                    signal_dspin_bound_rsp_out[x][y][face].read  = true;
                }

                signal_dspin_bound_m2p_in [x][y][face].write = false;
                signal_dspin_bound_m2p_in [x][y][face].read  = true;
                signal_dspin_bound_m2p_out[x][y][face].write = false;
                signal_dspin_bound_m2p_out[x][y][face].read  = true;

                signal_dspin_bound_p2m_in [x][y][face].write = false;
                signal_dspin_bound_p2m_in [x][y][face].read  = true;
                signal_dspin_bound_p2m_out[x][y][face].write = false;
                signal_dspin_bound_p2m_out[x][y][face].read  = true;

                signal_dspin_bound_cla_in [x][y][face].write = false;
                signal_dspin_bound_cla_in [x][y][face].read  = true;
                signal_dspin_bound_cla_out[x][y][face].write = false;
                signal_dspin_bound_cla_out[x][y][face].read  = true;
            }
        }
    }

    // set default values for VCI signals connected to unused ports on iobus 
    signal_vci_tgt_memc.rspval = false;
    signal_vci_tgt_xicu.rspval = false;
    for ( size_t p = 0 ; p < NB_PROCS_MAX ; p++ ) signal_vci_ini_proc[p].cmdval = false;

    sc_start(sc_core::sc_time(1, SC_NS));
    signal_resetn = true;

    if (gettimeofday(&t1, NULL) != 0) 
    {
        perror("gettimeofday");
        return EXIT_FAILURE;
    }

    // variable used for IRQ trace
    bool prev_irq_bdev = false;
    bool prev_irq_mtty_rx[8];
    bool prev_irq_proc[16][16][4];

    for( size_t x = 0 ; x<8  ; x++ ) prev_irq_mtty_rx[x] = false;

    for( size_t x = 0 ; x<16 ; x++ )
    for( size_t y = 0 ; y<16 ; y++ )
    for( size_t i = 0 ; i<4  ; i++ ) prev_irq_proc[x][y][i] = false;

    for (uint64_t n = 1; n < ncycles && !stop_called; n++)
    {
        // Monitor a specific address for L1 & L2 caches
        // clusters[0][0]->proc[0]->cache_monitor(0x110002C078ULL);
        // clusters[1][1]->memc->cache_monitor(0x110002c078ULL);

        // stats display
        if( (n % 5000000) == 0)
        {

            if (gettimeofday(&t2, NULL) != 0) 
            {
                perror("gettimeofday");
                return EXIT_FAILURE;
            }

            ms1 = (uint64_t) t1.tv_sec * 1000ULL + (uint64_t) t1.tv_usec / 1000;
            ms2 = (uint64_t) t2.tv_sec * 1000ULL + (uint64_t) t2.tv_usec / 1000;
            std::cerr << "platform clock frequency " 
                      << (double) 5000000 / (double) (ms2 - ms1) << "Khz" << std::endl;

            if (gettimeofday(&t1, NULL) != 0) 
            {
                perror("gettimeofday");
                return EXIT_FAILURE;
            }
        }

        // trace display
        if ( trace_ok and (n > trace_from) )
        {
            std::cout << "****************** cycle " << std::dec << n ;
            std::cout << " ************************************************" << std::endl;

            size_t l = 0;
            size_t x = 0;
            size_t y = 0;

            if ( trace_proc_ok )
            {
                l = trace_proc_id % NB_PROCS_MAX ;
                x = (trace_proc_id / NB_PROCS_MAX) >> Y_WIDTH ;
                y = (trace_proc_id / NB_PROCS_MAX) & ((1<<Y_WIDTH) - 1);

                std::ostringstream proc_signame;
                proc_signame << "[SIG]PROC_" << x << "_" << y << "_" << l ;
                clusters[x][y]->proc[l]->print_trace(1);
                clusters[x][y]->signal_vci_ini_proc[l].print_trace(proc_signame.str());

                std::ostringstream xicu_signame;
                xicu_signame << "[SIG]XICU_" << x << "_" << y ;
                clusters[x][y]->xicu->print_trace(0);
                clusters[x][y]->signal_vci_tgt_xicu.print_trace(xicu_signame.str());
            }

            if ( trace_memc_ok )
            {
                x = trace_memc_id >> Y_WIDTH;
                y = trace_memc_id & ((1<<Y_WIDTH) - 1);

                std::ostringstream smemc;
                smemc << "[SIG]MEMC_" << x << "_" << y;
                std::ostringstream sxram;
                sxram << "[SIG]XRAM_" << x << "_" << y;

                clusters[x][y]->memc->print_trace();
                clusters[x][y]->signal_vci_tgt_memc.print_trace(smemc.str());
                clusters[x][y]->signal_vci_xram.print_trace(sxram.str());
            }   

            // trace coherence signals
            // clusters[0][0]->signal_dspin_m2p_proc[0].print_trace("[CC_M2P_0_0]");
            // clusters[0][1]->signal_dspin_m2p_proc[0].print_trace("[CC_M2P_0_1]");
            // clusters[1][0]->signal_dspin_m2p_proc[0].print_trace("[CC_M2P_1_0]");
            // clusters[1][1]->signal_dspin_m2p_proc[0].print_trace("[CC_M2P_1_1]");

            // clusters[0][0]->signal_dspin_p2m_proc[0].print_trace("[CC_P2M_0_0]");
            // clusters[0][1]->signal_dspin_p2m_proc[0].print_trace("[CC_P2M_0_1]");
            // clusters[1][0]->signal_dspin_p2m_proc[0].print_trace("[CC_P2M_1_0]");
            // clusters[1][1]->signal_dspin_p2m_proc[0].print_trace("[CC_P2M_1_1]");

            // trace xbar(s) m2p
            // clusters[0][0]->xbar_m2p->print_trace();
            // clusters[1][0]->xbar_m2p->print_trace();
            // clusters[0][1]->xbar_m2p->print_trace();
            // clusters[1][1]->xbar_m2p->print_trace();
        
            // trace router(s) m2p
            // clusters[0][0]->router_m2p->print_trace();
            // clusters[1][0]->router_m2p->print_trace();
            // clusters[0][1]->router_m2p->print_trace();
            // clusters[1][1]->router_m2p->print_trace();
        
            // trace external ioc
            bdev->print_trace();
            signal_vci_tgt_bdev.print_trace("[SIG]BDEV_TGT");
            signal_vci_ini_bdev.print_trace("[SIG]BDEV_INI");

            // trace external iopic
            iopic->print_trace();
            signal_vci_tgt_iopi.print_trace("[SIG]IOPI_TGT");
            signal_vci_ini_iopi.print_trace("[SIG]IOPI_INI");

            // trace internal tty
            // clusters[0][0]->mtty->print_trace();
            // clusters[0][0]->signal_vci_tgt_mtty.print_trace("[SIG]MTTY");

        }  // end trace

        if (0)
        {
            // trace BDV interrupts events
            if ( signal_irq_bdev.read() != prev_irq_bdev ) 
            {
                prev_irq_bdev = signal_irq_bdev.read();
                std::cout << std::dec << "@@@ IRQ_BDEV = " << signal_irq_bdev.read()
                          << " at cycle " << n << std::endl;
            }
        
            // trace TTY interrupts events
            for ( size_t x = 0 ; x < 8 ; x++ )
            {
                if ( signal_irq_mtty_rx[x].read() != prev_irq_mtty_rx[x] ) 
                {
                    prev_irq_mtty_rx[x] = signal_irq_mtty_rx[x].read();
                    std::cout << std::dec << "@@@ IRQ_MTTY["<<x<<"] = " 
                              << signal_irq_mtty_rx[x].read()
                              << " at cycle " << n << std::endl;
                }
            }

            // trace processor interrupts events
            for ( size_t x = 0 ; x < X_SIZE ; x++ )
            for ( size_t y = 0 ; y < Y_SIZE ; y++ )
            for ( size_t i = 0 ; i < NB_PROCS_MAX ; i++ )
            {
                if ( clusters[x][y]->signal_proc_irq[i] != prev_irq_proc[x][y][i] )
                { 
                    prev_irq_proc[x][y][i] = clusters[x][y]->signal_proc_irq[i];
                    std::cout << std::dec << "@@@ IRQ_PROC["<<x<<","<<y<<","<<i<<"] = "
                              << clusters[x][y]->signal_proc_irq[i]
                              << " at cycle " << n << std::endl;
                }
            }

            // trace VCI transactions on IOPIC and XCU(0,0)
            signal_vci_tgt_iopi.print_trace("@@@ IOPI_TGT");
            signal_vci_ini_iopi.print_trace("@@@ IOPI_INI");
            clusters[0][0]->signal_vci_tgt_xicu.print_trace("@@@ XCU_0_0");
        }

        sc_start(sc_core::sc_time(1, SC_NS));
    }
    // Free memory
    for (size_t i = 0 ; i  < (X_SIZE * Y_SIZE) ; i++)
    {
        size_t x = i / (Y_SIZE);
        size_t y = i % (Y_SIZE);
        delete clusters[x][y];
    }

    return EXIT_SUCCESS;
}


void handler(int dummy = 0) 
{
   stop_called = true;
   sc_stop();
}

void voidhandler(int dummy = 0) {}

int sc_main (int argc, char *argv[])
{
   signal(SIGINT, handler);
   signal(SIGPIPE, voidhandler);

   try {
      return _main(argc, argv);
   } catch (std::exception &e) {
      std::cout << e.what() << std::endl;
   } catch (...) {
      std::cout << "Unknown exception occured" << std::endl;
      throw;
   }
   return 1;
}


// Local Variables:
// tab-width: 3
// c-basic-offset: 3
// c-file-offsets:((innamespace . 0)(inline-open . 0))
// indent-tabs-mode: nil
// End:

// vim: filetype=cpp:expandtab:shiftwidth=3:tabstop=3:softtabstop=3