source: trunk/platforms/tsarv4_generic_mmu/top.cpp @ 262

/////////////////////////////////////////////////////////////////////////
// File: top.cpp 
// Author: Alain Greiner 
// Copyright: UPMC/LIP6
// Date : august 2012
// This program is released under the GNU public license
/////////////////////////////////////////////////////////////////////////
// This file define a generic TSAR architecture with virtual memory.
// - It uses vci_local_crossbar as local interconnect 
// - It uses virtual_dspin as global interconnect
// - It uses the vci_cc_vcache_wrapper_v4 
// - It uses the vci_mem_cache_v4
// - It contains one vci_xicu and one vci_multi_dma per cluster.
// The peripherals BDEV, FBUF, MTTY, and the boot BROM
// are in the cluster containing address 0xBFC00000.
//
// It is build with one single component implementing a cluster:
// The Tsarv4ClusterMmu component is defined in files
// tsarv4_cluster_mmu.* (with * = cpp, h, sd)
//
// The IRQs are connected to XICUs as follow:
// - The IRQ_IN[0] to IRQ_IN[7] ports are not used in all clusters.
// - The DMA IRQs are connected to IRQ_IN[8] to IRQ_IN[15] in all clusters.
// - The TTY IRQs are connected to IRQ_IN[16] to IRQ_IN[30] in I/O cluster.
// - The BDEV IRQ is connected to IRQ_IN[31] in I/O cluster.
// 
// The physical address space is 32 bits.
// The number of clusters cannot be larger than 256.
// The number of processors per cluster cannot be larger than 8.
//
// The hardware parameters are :
// - xmax     : number of clusters in a row (power of 2)
// - ymax     : number of clusters in a column (power of 2)
// - nb_procs : number of processors per cluster (power of 2)
// - nb_dmas  : number of DMA channels per cluster (< 9)
// - nb_ttys  : number of TTYs in I/O cluster (< 16)
// 
// General policy for 32 bits physical address decoding:
// All segments base addresses are multiple of 64 Kbytes
// Therefore the 16 address MSB bits completely define the target: 
// The (x_width + y_width) MSB bits (left aligned) define
// the cluster index, and the 8 LSB bits define the local index:
//      | X_ID  | Y_ID  |---| LADR |     OFFSET          |
//      |x_width|y_width|---|  8   |       16            |
/////////////////////////////////////////////////////////////////////////

#include <systemc>
#include <sys/time.h>
#include <iostream>
#include <sstream>
#include <cstdlib>
#include <cstdarg>
#include <stdint.h>

#include "gdbserver.h"
#include "mapping_table.h"
#include "tsarv4_cluster_mmu.h"
#include "alloc_elems.h"

///////////////////////////////////////////////////
//      OS
///////////////////////////////////////////////////
#define USE_ALMOS 0

#define almos_bootloader_pathname "bootloader.bin"
#define almos_kernel_pathname     "kernel-soclib.bin@0xbfc10000:D"
#define almos_archinfo_pathname   "arch-info.bin@0xBFC08000:D"

///////////////////////////////////////////////////
//               Parallelisation
///////////////////////////////////////////////////
#define USE_OPENMP               0

#if USE_OPENMP
#include <omp.h>
#endif
///////////////////////////////////////////////////

//  cluster index (computed from x,y coordinates)
#define cluster(x,y)   (y + ymax*x)

// flit widths for the DSPIN network
#define cmd_width            40
#define rsp_width            33

// VCI format
#define cell_width            4
#define address_width         32
#define plen_width            8
#define error_width           2
#define clen_width            1
#define rflag_width           1
#define srcid_width           14
#define pktid_width           4
#define trdid_width           4
#define wrplen_width          1

///////////////////////////////////////////////////
//     Parameters default values         
///////////////////////////////////////////////////

#define MESH_XMAX             2
#define MESH_YMAX             2

#define NB_PROCS              1
#define NB_TTYS               2
#define NB_DMAS               1

#define XRAM_LATENCY          0

#define MEMC_WAYS             16
#define MEMC_SETS             256

#define L1_IWAYS              4
#define L1_ISETS              64

#define L1_DWAYS              4
#define L1_DSETS              64

#define FBUF_X_SIZE           128
#define FBUF_Y_SIZE           128

#define BDEV_SECTOR_SIZE      512
#define BDEV_IMAGE_NAME       "hdd-img.bin"

#define BOOT_SOFT_NAME        "soft.bin"

#define MAX_FROZEN_CYCLES     10000

#define TRACE_MEMC_ID         1000000
#define TRACE_PROC_ID         1000000

/////////////////////////////////////////////////////////
//    Physical segments definition
/////////////////////////////////////////////////////////
// There is 3 segments replicated in all clusters:
// - seg_memc   -> MEMC / BASE = 0x**000000    (12 M bytes)
// - seg_icu    -> ICU  / BASE = 0x**F00000
// - seg_dma    -> CDMA / BASE = 0x**F30000
//
// There is 4 specific segments in the "IO" cluster 
// (containing address 0xBF000000)
// - seg_reset  -> BROM / BASE = 0xBFC00000   (1 Mbytes)
// - seg_fbuf   -> FBUF / BASE = 0xBFD00000   (2 M bytes)
// - seg_bdev   -> BDEV / BASE = 0xBFF10000
// - seg_tty    -> MTTY / BASE = 0x**F20000
//
// There is one special segment corresponding to
// the processors in the coherence address space
// - seg_proc   -> PROC / BASE = 0x**B0 to 0xBF
///////////////////////////////////////////////////

// specific segments in "IO" cluster : absolute physical address

#define BROM_BASE               0xBFC00000      
#define BROM_SIZE               0x00100000

#define FBUF_BASE               0x80D00000      
#define FBUF_SIZE               0x00200000

#define BDEV_BASE               0x80F10000      
#define BDEV_SIZE               0x00001000

#define MTTY_BASE               0x80F20000      
#define MTTY_SIZE               0x00001000

// replicated segments : physical address is incremented by an offset 
//     offset  = cluster(x,y) << (address_width-x_width-y_width);

#define MEMC_BASE               0x00000000      
#define MEMC_SIZE               0x00C00000

#define XICU_BASE               0x00F00000      
#define XICU_SIZE               0x00001000

#define CDMA_BASE               0x00F30000      
#define CDMA_SIZE               0x00008000

////////////////////////////////////////////////////////////////////
//     TGTID definition in direct space
// For all components:  global TGTID = global SRCID = cluster_index
////////////////////////////////////////////////////////////////////

#define MEMC_TGTID               0
#define XICU_TGTID               1
#define CDMA_TGTID               2
#define MTTY_TGTID               3
#define FBUF_TGTID               4
#define BROM_TGTID               5
#define BDEV_TGTID               6

/////////////////////////////////
int _main(int argc, char *argv[])
{
   using namespace sc_core;
   using namespace soclib::caba;
   using namespace soclib::common;


   char     soft_name[256] = BOOT_SOFT_NAME;     // pathname to binary code
   size_t   ncycles        = 1000000000;         // simulated cycles
   size_t   xmax           = MESH_XMAX;          // number of clusters in a row
   size_t   ymax           = MESH_YMAX;          // number of clusters in a column
   size_t   nb_procs       = NB_PROCS;           // number of processors per cluster
   size_t   nb_dmas        = NB_DMAS;            // number of RDMA channels per cluster
   size_t   nb_ttys        = NB_TTYS;            // number of TTY terminals in I/O cluster
   size_t   xfb            = FBUF_X_SIZE;        // frameBuffer column number
   size_t   yfb            = FBUF_Y_SIZE;        // frameBuffer lines number
   size_t   memc_ways      = MEMC_WAYS;
   size_t   memc_sets      = MEMC_SETS;
   size_t   l1_d_ways      = L1_DWAYS;
   size_t   l1_d_sets      = L1_DSETS;
   size_t   l1_i_ways      = L1_IWAYS;
   size_t   l1_i_sets      = L1_ISETS;
   char     disk_name[256] = BDEV_IMAGE_NAME;    // pathname to the disk image
   size_t   blk_size       = BDEV_SECTOR_SIZE;   // block size (in bytes)
   size_t   xram_latency   = XRAM_LATENCY;       // external RAM latency
   ssize_t  threads_nr     = 1;                  // simulator's threads number
   bool     debug_ok       = false;              // trace activated
   size_t   debug_period   = 1;                  // trace period
   size_t   debug_memc_id  = TRACE_MEMC_ID;      // index of memc to be traced (cluster_id)  
   size_t   debug_proc_id  = TRACE_PROC_ID;      // index of proc to be traced
   uint32_t debug_from     = 0;                  // trace start cycle
   uint32_t frozen_cycles  = MAX_FROZEN_CYCLES;  // monitoring frozen processor

   ////////////// command line arguments //////////////////////
   if (argc > 1)
   {
      for (int n = 1; n < argc; n = n + 2)
      {
         if ((strcmp(argv[n],"-NCYCLES") == 0) && (n+1<argc))
         {
            ncycles = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n],"-NPROCS") == 0) && (n+1<argc))
         {
            nb_procs = atoi(argv[n+1]);
            assert( ((nb_procs == 1) || (nb_procs == 2) || 
                     (nb_procs == 4) || (nb_procs == 8)) &&
                  "NPROCS must be equal to 1, 2, 4, or 8");
         }
         else if ((strcmp(argv[n],"-NTTYS") == 0) && (n+1<argc))
         {
            nb_ttys = atoi(argv[n+1]);
            assert( (nb_ttys < 16) &&
                   "The number of TTY terminals cannot be larger than 15");
         }
         else if ((strcmp(argv[n],"-NDMAS") == 0) && (n+1<argc))
         {
            nb_dmas = atoi(argv[n+1]);
            assert( (nb_dmas < 9) &&
                   "The number of DMA channels per cluster cannot be larger than 8");
         }
         else if ((strcmp(argv[n],"-XMAX") == 0) && (n+1<argc))
         {
            xmax = atoi(argv[n+1]);
            assert( ((xmax == 1) || (xmax == 2) || (xmax == 4) || (xmax == 8) || (xmax == 16)) 
                  && "The XMAX parameter must be 2, 4, 8, or 16" );
         }

         else if ((strcmp(argv[n],"-YMAX") == 0) && (n+1<argc))
         {
            ymax = atoi(argv[n+1]);
            assert( ((ymax == 1) || (ymax == 2) || (ymax == 4) || (ymax == 8) || (ymax == 16)) 
                  && "The YMAX parameter must be 2, 4, 8, or 16" );
         }
         else if ((strcmp(argv[n],"-XFB") == 0) && (n+1<argc))
         {
            xfb = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n],"-YFB") == 0) && (n+1<argc) )
         {
            yfb = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n],"-SOFT") == 0) && (n+1<argc) )
         {
            strcpy(soft_name, argv[n+1]);
         }
         else if ((strcmp(argv[n],"-DISK") == 0) && (n+1<argc) )
         {
            strcpy(disk_name, argv[n+1]);
         }
         else if ((strcmp(argv[n],"-BLKSZ") == 0) && ((n+1) < argc))
         {
            blk_size = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n],"-TRACE") == 0) && (n+1<argc) )
         {
            debug_ok = true;
            debug_from = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n],"-MEMCID") == 0) && (n+1<argc) )
         {
            debug_memc_id = atoi(argv[n+1]);
            assert( (debug_memc_id < (xmax*ymax) ) && 
                   "debug_memc_id larger than XMAX * YMAX" );
         }
         else if ((strcmp(argv[n],"-PROCID") == 0) && (n+1<argc) )
         {
            debug_proc_id = atoi(argv[n+1]);
            assert( (debug_proc_id < (xmax*ymax*nb_procs) ) && 
                   "debug_proc_id larger than XMAX * YMAX * BN_PROCS" );
         }
         else if ((strcmp(argv[n], "-MCWAYS") == 0) && (n+1 < argc))
         {
            memc_ways = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n], "-MCSETS") == 0) && (n+1 < argc))
         {
            memc_sets = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n], "-XLATENCY") == 0) && (n+1 < argc))
         {
            xram_latency = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n], "-THREADS") == 0) && ((n+1) < argc))
         {
            threads_nr = atoi(argv[n+1]);
            threads_nr = (threads_nr < 1) ? 1 : threads_nr;
         }
         else if ((strcmp(argv[n], "-FROZEN") == 0) && (n+1 < argc))
         {
            frozen_cycles = atoi(argv[n+1]);
         }
         else if ((strcmp(argv[n], "-PERIOD") == 0) && (n+1 < argc))
         {
            debug_period = atoi(argv[n+1]);
         }
         else
         {
            std::cout << "   Arguments on the command line are (key,value) couples." << std::endl;
            std::cout << "   The order is not important." << std::endl;
            std::cout << "   Accepted arguments are :" << std::endl << std::endl;
            std::cout << "     -SOFT pathname_for_embedded_soft" << std::endl;
            std::cout << "     -DISK pathname_for_disk_image" << std::endl;
            std::cout << "     -BLKSZ disk sector size" << std::endl;
            std::cout << "     -NCYCLES number_of_simulated_cycles" << std::endl;
            std::cout << "     -NPROCS number_of_processors_per_cluster" << std::endl;
            std::cout << "     -NTTYS total_number_of_TTY_terminals" << std::endl;
            std::cout << "     -NDMAS number_of_DMA_channels_per_cluster" << std::endl;
            std::cout << "     -XMAX number_of_clusters_in_a_row" << std::endl;
            std::cout << "     -YMAX number_of_clusters_in_a_column" << std::endl;
            std::cout << "     -TRACE debug_start_cycle" << std::endl;
            std::cout << "     -MCWAYS memory_cache_number_of_ways" << std::endl;
            std::cout << "     -MCSETS memory_cache_number_of_sets" << std::endl;
            std::cout << "     -XLATENCY external_ram_latency_value" << std::endl;
            std::cout << "     -XFB fram_buffer_number_of_pixels" << std::endl;
            std::cout << "     -YFB fram_buffer_number_of_lines" << std::endl;
            std::cout << "     -THREADS simulator's threads number" << std::endl;
            std::cout << "     -FROZEN max_number_of_lines" << std::endl;
            std::cout << "     -PERIOD number_of_cycles between trace" << std::endl;
            std::cout << "     -MEMCID index_memc_to_be_traced" << std::endl;
            std::cout << "     -PROCID index_proc_to_be_traced" << std::endl;
            exit(0);
         }
      }
   }

   std::cout << std::endl;
   std::cout << " - NB_CLUSTERS = " << xmax*ymax << std::endl;
   std::cout << " - NB_PROCS    = " << nb_procs <<  std::endl;
   std::cout << " - NB_TTYS     = " << nb_ttys <<  std::endl;
   std::cout << " - NB_DMAS     = " << nb_dmas <<  std::endl;
   std::cout << " - MAX_FROZEN  = " << frozen_cycles << std::endl;
   std::cout << " - MEMC_WAYS   = " << memc_ways << std::endl;
   std::cout << " - MEMC_SETS   = " << memc_sets << std::endl;
   std::cout << " - RAM_LATENCY = " << xram_latency << std::endl;

   std::cout << std::endl;

#if USE_OPENMP
   omp_set_dynamic(false);
   omp_set_num_threads(threads_nr);
   std::cerr << "Built with openmp version " << _OPENMP << std::endl;
#endif

   // Define VCI parameters
   typedef soclib::caba::VciParams<cell_width,
           plen_width,
           address_width,
           error_width,                                   
           clen_width,
           rflag_width,
           srcid_width,
           pktid_width,
           trdid_width,
           wrplen_width> vci_param;

   // Define parameters depending on mesh size
   size_t   cluster_io_id;
   size_t   x_width;
   size_t   y_width;

   if      (xmax == 1) x_width = 0;
   else if (xmax == 2) x_width = 1;
   else if (xmax <= 4) x_width = 2;
   else if (xmax <= 8) x_width = 3;
   else                x_width = 4;

   if      (ymax == 1) y_width = 0;
   else if (ymax == 2) y_width = 1;
   else if (ymax <= 4) y_width = 2;
   else if (ymax <= 8) y_width = 3;
   else                y_width = 4;

   cluster_io_id = 0xBF >> (8 - x_width - y_width);

   /////////////////////
   //  Mapping Tables
   /////////////////////

   // direct network
   MappingTable maptabd(address_width, 
         IntTab(x_width + y_width, 16 - x_width - y_width), 
         IntTab(x_width + y_width, srcid_width - x_width - y_width), 
         0x00FF0000);

   for (size_t x = 0; x < xmax; x++)
   {
      for (size_t y = 0; y < ymax; y++)
      {
         sc_uint<address_width> offset  = cluster(x,y) << (address_width-x_width-y_width);

         std::ostringstream    sh;
         sh << "d_seg_memc_" << x << "_" << y;
         maptabd.add(Segment(sh.str(), MEMC_BASE+offset, MEMC_SIZE, IntTab(cluster(x,y),MEMC_TGTID), true));

         std::ostringstream    si;
         si << "d_seg_xicu_" << x << "_" << y;
         maptabd.add(Segment(si.str(), XICU_BASE+offset, XICU_SIZE, IntTab(cluster(x,y),XICU_TGTID), false));

         std::ostringstream    sd;
         sd << "d_seg_mdma_" << x << "_" << y;
         maptabd.add(Segment(sd.str(), CDMA_BASE+offset, CDMA_SIZE, IntTab(cluster(x,y),CDMA_TGTID), false));

         if ( cluster(x,y) == cluster_io_id )
         {
            maptabd.add(Segment("d_seg_mtty", MTTY_BASE, MTTY_SIZE, IntTab(cluster(x,y),MTTY_TGTID), false));
            maptabd.add(Segment("d_seg_fbuf", FBUF_BASE, FBUF_SIZE, IntTab(cluster(x,y),FBUF_TGTID), false));
            maptabd.add(Segment("d_seg_bdev", BDEV_BASE, BDEV_SIZE, IntTab(cluster(x,y),BDEV_TGTID), false));
            maptabd.add(Segment("d_seg_brom", BROM_BASE, BROM_SIZE, IntTab(cluster(x,y),BROM_TGTID), true));
         }
      }
   }
   std::cout << maptabd << std::endl;

   // coherence network
   // - tgtid_c_proc = srcid_c_proc = local procid
   // - tgtid_c_memc = srcid_c_memc = nb_procs
   MappingTable maptabc(address_width, 
         IntTab(x_width + y_width, srcid_width - x_width - y_width), 
         IntTab(x_width + y_width, srcid_width - x_width - y_width), 
         0x00FF0000);

   for (size_t x = 0; x < xmax; x++)
   {
      for (size_t y = 0; y < ymax; y++)
      {
         sc_uint<address_width> offset  = cluster(x,y) << (address_width-x_width-y_width);

         // cleanup requests must be routed to the memory cache
         std::ostringstream sh;
         sh << "c_seg_memc_" << x << "_" << y;
         maptabc.add(Segment(sh.str(), (nb_procs << (address_width - srcid_width)) + offset, 
                     0x10, IntTab(cluster(x,y), nb_procs), false));

         // update & invalidate requests must be routed to the proper processor
         for ( size_t p = 0 ; p < nb_procs ; p++) 
         {
            std::ostringstream sp;
            sp << "c_seg_proc_" << x << "_" << y << "_" << p;
            maptabc.add( Segment( sp.str() , (p << (address_width - srcid_width)) + offset , 
                         0x10 , IntTab(cluster(x,y), p) , false)); 
         }
      }
   }
   std::cout << maptabc << std::endl;

   // external network
   MappingTable maptabx(address_width, IntTab(1), IntTab(x_width+y_width), 0xF0000000);

   for (size_t x = 0; x < xmax; x++)
   {
      for (size_t y = 0; y < ymax ; y++)
      { 
         sc_uint<address_width> offset  = cluster(x,y) << (address_width-x_width-y_width);
         std::ostringstream sh;
         sh << "x_seg_memc_" << x << "_" << y;
         maptabx.add(Segment(sh.str(), MEMC_BASE+offset, 
                     MEMC_SIZE, IntTab(cluster(x,y)), false));
      }
   }
   std::cout << maptabx << std::endl;

   ////////////////////
   // Signals
   ///////////////////

   sc_clock      signal_clk("clk");
   sc_signal<bool>    signal_resetn("resetn");

   // Horizontal inter-clusters DSPIN signals
   DspinSignals<cmd_width>*** signal_dspin_h_cmd_inc =
      alloc_elems<DspinSignals<cmd_width> >("signal_dspin_h_cmd_inc", xmax-1, ymax, 2);
   DspinSignals<cmd_width>*** signal_dspin_h_cmd_dec =
      alloc_elems<DspinSignals<cmd_width> >("signal_dspin_h_cmd_dec", xmax-1, ymax, 2);
   DspinSignals<rsp_width>*** signal_dspin_h_rsp_inc =
      alloc_elems<DspinSignals<rsp_width> >("signal_dspin_h_rsp_inc", xmax-1, ymax, 2);
   DspinSignals<rsp_width>*** signal_dspin_h_rsp_dec =
      alloc_elems<DspinSignals<rsp_width> >("signal_dspin_h_rsp_dec", xmax-1, ymax, 2);

   // Vertical inter-clusters DSPIN signals
   DspinSignals<cmd_width>*** signal_dspin_v_cmd_inc =
      alloc_elems<DspinSignals<cmd_width> >("signal_dspin_v_cmd_inc", xmax, ymax-1, 2);
   DspinSignals<cmd_width>*** signal_dspin_v_cmd_dec =
      alloc_elems<DspinSignals<cmd_width> >("signal_dspin_v_cmd_dec", xmax, ymax-1, 2);
   DspinSignals<rsp_width>*** signal_dspin_v_rsp_inc =
      alloc_elems<DspinSignals<rsp_width> >("signal_dspin_v_rsp_inc", xmax, ymax-1, 2);
   DspinSignals<rsp_width>*** signal_dspin_v_rsp_dec =
      alloc_elems<DspinSignals<rsp_width> >("signal_dspin_v_rsp_dec", xmax, ymax-1, 2);

   // Mesh boundaries DSPIN signals
   DspinSignals<cmd_width>**** signal_dspin_false_cmd_in =
      alloc_elems<DspinSignals<cmd_width> >("signal_dspin_false_cmd_in", xmax, ymax, 2, 4);
   DspinSignals<cmd_width>**** signal_dspin_false_cmd_out =
      alloc_elems<DspinSignals<cmd_width> >("signal_dspin_false_cmd_out", xmax, ymax, 2, 4);
   DspinSignals<rsp_width>**** signal_dspin_false_rsp_in =
      alloc_elems<DspinSignals<rsp_width> >("signal_dspin_false_rsp_in", xmax, ymax, 2, 4);
   DspinSignals<rsp_width>**** signal_dspin_false_rsp_out =
      alloc_elems<DspinSignals<rsp_width> >("signal_dspin_false_rsp_out", xmax, ymax, 2, 4);


   ////////////////////////////
   //      Components
   ////////////////////////////

#if USE_ALMOS
   soclib::common::Loader loader(almos_bootloader_pathname,
         almos_archinfo_pathname,
         almos_kernel_pathname);
#else
   soclib::common::Loader loader(soft_name);
#endif

   typedef soclib::common::GdbServer<soclib::common::Mips32ElIss> proc_iss;
   proc_iss::set_loader(loader);

   TsarV4ClusterMmu<vci_param, proc_iss, cmd_width, rsp_width>* clusters[xmax][ymax];

#if USE_OPENMP

#pragma omp parallel
   {
#pragma omp for
      for(size_t i = 0; i  < (xmax * ymax); i++)
      {
          size_t x = i / ymax;
          size_t y = i % ymax;
#pragma omp critical

          std::cout << "building cluster_" << x << "_" << y << std::endl;

          std::ostringstream sc;
          sc << "cluster_" << x << "_" << y;
          clusters[x][y] = new TsarV4ClusterMmu<vci_param, proc_iss, cmd_width, rsp_width>
            (sc.str().c_str(),
             nb_procs,
             nb_ttys,  
             nb_dmas, 
             x,
             y,
             cluster(x,y),
             maptabd,
             maptabc,
             maptabx,
             x_width,
             y_width,
             MEMC_TGTID,
             XICU_TGTID,
             FBUF_TGTID,
             MTTY_TGTID,
             BROM_TGTID,
             BDEV_TGTID,
             CDMA_TGTID,
             memc_ways,
             memc_sets,
             l1_i_ways,
             l1_i_sets,
             l1_d_ways,
             l1_d_sets,
             xram_latency,
             (cluster(x,y) == cluster_io_id),
             xfb,
             yfb,
             disk_name,
             blk_size,
             loader,
             frozen_cycles,
             debug_from,
             debug_ok and (cluster(x,y) == debug_memc_id),
             debug_ok and (cluster(x,y) == debug_proc_id) );

         std::cout << "cluster_" << x << "_" << y << " constructed" << std::endl;

       }
   }

#else  // NO OPENMP

   for (size_t x = 0; x  < xmax; x++)
   {
       for (size_t y = 0; y < ymax; y++)
       {

         std::cout << "building cluster_" << x << "_" << y << std::endl;

         std::ostringstream sc;
         sc << "cluster_" << x << "_" << y;
         clusters[x][y] = new TsarV4ClusterMmu<vci_param, proc_iss, cmd_width, rsp_width>
            (sc.str().c_str(),
             nb_procs,
             nb_ttys,
             nb_dmas,
             x,
             y,
             cluster(x,y),
             maptabd,
             maptabc,
             maptabx,
             x_width,
             y_width,
             MEMC_TGTID,
             XICU_TGTID,
             FBUF_TGTID,
             MTTY_TGTID,
             BROM_TGTID,
             BDEV_TGTID,
             CDMA_TGTID,
             memc_ways,
             memc_sets,
             l1_i_ways,
             l1_i_sets,
             l1_d_ways,
             l1_d_sets,
             xram_latency,
             (cluster(x,y) == cluster_io_id),
             xfb,
             yfb,
             disk_name,
             blk_size,
             loader,
             frozen_cycles,
             debug_from,
             debug_ok and ( cluster(x,y) == debug_memc_id ),
             debug_ok and ( cluster(x,y) == debug_proc_id ) );

         std::cout << "cluster_" << x << "_" << y << " constructed" << std::endl;

      }
   }

#endif   // USE_OPENMP

   ///////////////////////////////////////////////////////////////
   //     Net-list 
   ///////////////////////////////////////////////////////////////

   // Clock & RESET
   for (size_t x = 0; x < (xmax); x++){
      for (size_t y = 0; y < ymax; y++){
         clusters[x][y]->p_clk     (signal_clk);
         clusters[x][y]->p_resetn  (signal_resetn);
      }
   }

   // Inter Clusters horizontal connections
   if (xmax > 1){
      for (size_t x = 0; x < (xmax-1); x++){
         for (size_t y = 0; y < ymax; y++){
            for (size_t k = 0; k < 2; k++){
               clusters[x][y]->p_cmd_out[k][EAST]      (signal_dspin_h_cmd_inc[x][y][k]);
               clusters[x+1][y]->p_cmd_in[k][WEST]     (signal_dspin_h_cmd_inc[x][y][k]);
               clusters[x][y]->p_cmd_in[k][EAST]       (signal_dspin_h_cmd_dec[x][y][k]);
               clusters[x+1][y]->p_cmd_out[k][WEST]    (signal_dspin_h_cmd_dec[x][y][k]);
               clusters[x][y]->p_rsp_out[k][EAST]      (signal_dspin_h_rsp_inc[x][y][k]);
               clusters[x+1][y]->p_rsp_in[k][WEST]     (signal_dspin_h_rsp_inc[x][y][k]);
               clusters[x][y]->p_rsp_in[k][EAST]       (signal_dspin_h_rsp_dec[x][y][k]);
               clusters[x+1][y]->p_rsp_out[k][WEST]    (signal_dspin_h_rsp_dec[x][y][k]);
            }
         }
      }
   }
   std::cout << "Horizontal connections established" << std::endl;   

   // Inter Clusters vertical connections
   if (ymax > 1) {
      for (size_t y = 0; y < (ymax-1); y++){
         for (size_t x = 0; x < xmax; x++){
            for (size_t k = 0; k < 2; k++){
               clusters[x][y]->p_cmd_out[k][NORTH]     (signal_dspin_v_cmd_inc[x][y][k]);
               clusters[x][y+1]->p_cmd_in[k][SOUTH]    (signal_dspin_v_cmd_inc[x][y][k]);
               clusters[x][y]->p_cmd_in[k][NORTH]      (signal_dspin_v_cmd_dec[x][y][k]);
               clusters[x][y+1]->p_cmd_out[k][SOUTH]   (signal_dspin_v_cmd_dec[x][y][k]);
               clusters[x][y]->p_rsp_out[k][NORTH]     (signal_dspin_v_rsp_inc[x][y][k]);
               clusters[x][y+1]->p_rsp_in[k][SOUTH]    (signal_dspin_v_rsp_inc[x][y][k]);
               clusters[x][y]->p_rsp_in[k][NORTH]      (signal_dspin_v_rsp_dec[x][y][k]);
               clusters[x][y+1]->p_rsp_out[k][SOUTH]   (signal_dspin_v_rsp_dec[x][y][k]);
            }
         }
      }
   }
   std::cout << "Vertical connections established" << std::endl;

   // East & West boundary cluster connections
   for (size_t y = 0; y < ymax; y++)
   {
      for (size_t k = 0; k < 2; k++)
      {
         clusters[0][y]->p_cmd_in[k][WEST]          (signal_dspin_false_cmd_in[0][y][k][WEST]);
         clusters[0][y]->p_cmd_out[k][WEST]         (signal_dspin_false_cmd_out[0][y][k][WEST]);
         clusters[0][y]->p_rsp_in[k][WEST]          (signal_dspin_false_rsp_in[0][y][k][WEST]);
         clusters[0][y]->p_rsp_out[k][WEST]         (signal_dspin_false_rsp_out[0][y][k][WEST]);

         clusters[xmax-1][y]->p_cmd_in[k][EAST]     (signal_dspin_false_cmd_in[xmax-1][y][k][EAST]);
         clusters[xmax-1][y]->p_cmd_out[k][EAST]    (signal_dspin_false_cmd_out[xmax-1][y][k][EAST]);
         clusters[xmax-1][y]->p_rsp_in[k][EAST]     (signal_dspin_false_rsp_in[xmax-1][y][k][EAST]);
         clusters[xmax-1][y]->p_rsp_out[k][EAST]    (signal_dspin_false_rsp_out[xmax-1][y][k][EAST]);
      }
   }

   // North & South boundary clusters connections
   for (size_t x = 0; x < xmax; x++)
   {
      for (size_t k = 0; k < 2; k++)
      {
         clusters[x][0]->p_cmd_in[k][SOUTH]         (signal_dspin_false_cmd_in[x][0][k][SOUTH]);
         clusters[x][0]->p_cmd_out[k][SOUTH]        (signal_dspin_false_cmd_out[x][0][k][SOUTH]);
         clusters[x][0]->p_rsp_in[k][SOUTH]         (signal_dspin_false_rsp_in[x][0][k][SOUTH]);
         clusters[x][0]->p_rsp_out[k][SOUTH]        (signal_dspin_false_rsp_out[x][0][k][SOUTH]);

         clusters[x][ymax-1]->p_cmd_in[k][NORTH]    (signal_dspin_false_cmd_in[x][ymax-1][k][NORTH]);
         clusters[x][ymax-1]->p_cmd_out[k][NORTH]   (signal_dspin_false_cmd_out[x][ymax-1][k][NORTH]);
         clusters[x][ymax-1]->p_rsp_in[k][NORTH]    (signal_dspin_false_rsp_in[x][ymax-1][k][NORTH]);
         clusters[x][ymax-1]->p_rsp_out[k][NORTH]   (signal_dspin_false_rsp_out[x][ymax-1][k][NORTH]);
      }
   }


   ////////////////////////////////////////////////////////
   //   Simulation
   ///////////////////////////////////////////////////////

   sc_start(sc_core::sc_time(0, SC_NS));
   signal_resetn = false;

   // network boundaries signals
   for (size_t x = 0; x < xmax ; x++){
      for (size_t y = 0; y < ymax ; y++){
         for (size_t k = 0; k < 2; k++){
            for (size_t a = 0; a < 4; a++){
               signal_dspin_false_cmd_in[x][y][k][a].write = false;
               signal_dspin_false_cmd_in[x][y][k][a].read = true;
               signal_dspin_false_cmd_out[x][y][k][a].write = false;
               signal_dspin_false_cmd_out[x][y][k][a].read = true;

               signal_dspin_false_rsp_in[x][y][k][a].write = false;
               signal_dspin_false_rsp_in[x][y][k][a].read = true;
               signal_dspin_false_rsp_out[x][y][k][a].write = false;
               signal_dspin_false_rsp_out[x][y][k][a].read = true;
            }
         }
      }
   }

   sc_start(sc_core::sc_time(1, SC_NS));
   signal_resetn = true;

   for (size_t n = 1; n < ncycles; n++)
   {

      if (debug_ok and (n > debug_from) and (n % debug_period == 0))
      {
         std::cout << "****************** cycle " << std::dec << n ;
         std::cout << " ************************************************" << std::endl;

         // trace proc[debug_proc_id] 
         if ( debug_proc_id < (xmax * ymax * nb_procs) )
         {
             size_t proc_x = debug_proc_id / ymax;
             size_t proc_y = debug_proc_id % ymax;

             clusters[proc_x][proc_y]->proc[0]->print_trace();

             clusters[proc_x][proc_y]->signal_vci_ini_d_proc[0].print_trace("proc_ini_d");
             clusters[proc_x][proc_y]->signal_vci_ini_c_proc[0].print_trace("proc_ini_c");
             clusters[proc_x][proc_y]->signal_vci_tgt_c_proc[0].print_trace("proc_tgt_c");
         }

         // trace memc[debug_memc_id] 
         if ( debug_memc_id < (xmax * ymax) )
         {
             size_t memc_x = debug_memc_id / ymax;
             size_t memc_y = debug_memc_id % ymax;

             clusters[memc_x][memc_y]->memc->print_trace();

             clusters[memc_x][memc_y]->signal_vci_tgt_d_memc.print_trace("memc_tgt_d");
             clusters[memc_x][memc_y]->signal_vci_ini_c_memc.print_trace("memc_ini_c");
             clusters[memc_x][memc_y]->signal_vci_tgt_c_memc.print_trace("memc_tgt_c");
         }

         // clusters[0][0]->signal_vci_tgt_d_xicu.print_trace("xicu_0_0");
         // clusters[0][1]->signal_vci_tgt_d_xicu.print_trace("xicu_0_1");
         // clusters[1][0]->signal_vci_tgt_d_xicu.print_trace("xicu_1_0");
         // clusters[1][1]->signal_vci_tgt_d_xicu.print_trace("xicu_1_1");

         // if ( clusters[1][1]->signal_irq_mdma[0].read() ) 
         //    std::cout << std::endl << " IRQ_DMA_1_1 activated" << std::endl;
         // if ( clusters[1][1]->signal_proc_it[0].read() )
         //    std::cout <<  " IRQ_PROC_1_1 activated" << std::endl << std::endl;

         // trace ioc component 
         size_t io_x   = cluster_io_id / ymax;
         size_t io_y   = cluster_io_id % ymax;
//         clusters[io_x][io_y]->bdev->print_trace();
//         clusters[io_x][io_y]->signal_vci_tgt_d_bdev.print_trace("bdev_1_0_tgt_d  ");
//         clusters[io_x][io_y]->signal_vci_ini_d_bdev.print_trace("bdev_1_0_ini_d  ");

         clusters[1][1]->mdma->print_trace();
         clusters[1][1]->signal_vci_tgt_d_mdma.print_trace("mdma_1_1_tgt_d  ");
         clusters[1][1]->signal_vci_ini_d_mdma.print_trace("mdma_1_1_ini_d  ");
      }

      sc_start(sc_core::sc_time(1, SC_NS));
   }
   return EXIT_SUCCESS;
}

int sc_main (int argc, char *argv[])
{
   try {
      return _main(argc, argv);
   } catch (std::exception &e) {
      std::cout << e.what() << std::endl;
   } catch (...) {
      std::cout << "Unknown exception occured" << std::endl;
      throw;
   }
   return 1;
}


// Local Variables:
// tab-width: 3
// c-basic-offset: 3
// c-file-offsets:((innamespace . 0)(inline-open . 0))
// indent-tabs-mode: nil
// End:

// vim: filetype=cpp:expandtab:shiftwidth=3:tabstop=3:softtabstop=3