source: trunk/tools/bootloader_tsar/boot.c @ 9

/*
 * boot.c - TSAR bootloader implementation.
 * 
 * Authors :   Alain Greiner / Vu Son  (2016)
 *
 * Copyright (c) UPMC Sorbonne Universites
 *
 * This file is part of ALMOS-MKH.
 *
 * ALMOS-MKH is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.0 of the License.
 *
 * ALMOS-MKH is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with ALMOS-MKH; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

/****************************************************************************
 * This file contains the ALMOS-MKH. boot-loader for the TSAR architecture. *
 *                                                                          *
 * It supports clusterised shared memory multi-processor architectures,     *
 * where each processor is identified by a composite index [cxy,lid],       *
 * with one physical memory bank per cluster.                               *
 *                                                                          *
 * The 'boot.elf' file (containing the boot-loader binary code) is stored   *
 * on disk and is loaded into memory by bscpu (whose index is [0,0]),       *
 * executing the generic preloader.                                         *
 *                                                                          *
 * 1) The boot-loader first phase is executed by bscpu only, while          *
 *    all other cores are waiting in the preloader.                         *
 *    It does the following tasks:                                          *
 *      - load into the memory bank of cluster (0,0) the 'arch_info.bin'    *
 *        file (containing the hardware architecture description) and the   *
 *        'kernel.elf' file, at temporary locations,                        *    
 *      - initializes the 'boot_info_t' structure in cluster(0,0)           *
 *        (there is 1 'boot_info_t' per cluster), which contains both       *
 *        global and cluster specific information that will be used for     *
 *        kernel initialisation.                                            *
 *      - activate CP0s in all other clusters, using IPIs.                  *
 *      - wait completion reports from CP0s on a global barrier.            * 
 *                                                                          *
 * 2) The boot-loader second phase is then executed in parallel by all      *
 *    CP0s (other than bscpu). Each CP0 performs the following tasks:       *
 *      - copies into the memory bank of the local cluster the 'boot.elf',  *
 *        the 'arch_info.bin' (at the same addresses as the 'boot.elf' and  *
 *        the 'arch_info.bin' in the memory bank of the cluster(0,0), and   *
 *        the kernel image (at address 0x0),                                *
 *      - initializes the 'boot_info_t' structure of the local cluster,     *
 *      - activate all other cores in the same cluster (CPi).               *
 *      - wait local CPi completion reports on a local barrier.             *
 *      - report completion to bscpu on the global barrier.                 *
 *                                                                          *
 * 3) The boot-loader third phase is executed in parallel by all cores.     *
 *    After passing the global barrier the bscpu:                           *
 *      - activates the CPi of cluster(0,0),                                *
 *      - blocks on the local barrier waiting for all local CPi to report   *
 *        completion on the local barrier,                                  *
 *      - moves the local kernel image from the temporary location to the   *
 *        address 0x0, (erasing the preloader code).                        *
 *                                                                          *
 * 4) All cores have finished the boot phase, they jump to the kern_init()  *
 *    function (maybe not at the same time).                                *
 ****************************************************************************/

#include <elf-types.h>
#include <hal_types.h>

#include <almos_config.h>
#include <boot_config.h>

#include <arch_info.h>
#include <boot_info.h>

#include <boot_utils.h>
#include <boot_fat32.h>
#include <boot_bdv_driver.h>
#include <boot_hba_driver.h>
#include <boot_tty_driver.h>

/*****************************************************************************
 *                                 Macros.                              
 ****************************************************************************/

#define PAGE_ROUND_DOWN(x)  ((x) & (~PPM_PAGE_SIZE -1))
#define PAGE_ROUND_UP(x)    (((x) + PPM_PAGE_SIZE-1) &   \
                            (~(PPM_PAGE_SIZE-1)))

/*****************************************************************************
 *                             Global variables.                            
 ****************************************************************************/

// synchronization variables. 

volatile boot_remote_spinlock_t tty0_lock;       // protect TTY0 access
volatile boot_remote_barrier_t  global_barrier;  // synchronize CP0 cores
volatile boot_remote_barrier_t  local_barrier;   // synchronize cores in one cluster
uint32_t                        active_cp0s_nr;  // number of expected CP0s
 
// kernel segments layout variables

uint32_t                        seg_kcode_base;  // kcode segment base address
uint32_t                        seg_kcode_size;  // kcode segment size (bytes) 
uint32_t                        seg_kdata_base;  // kdata segment base address
uint32_t                        seg_kdata_size;  // kdata segment size (bytes) 
uint32_t                        kernel_entry;    // kernel entry point

// address used by the WTI to activate remote CP0s

extern void                     boot_entry();    // boot_loader entry point

/*********************************************************************************
 * This function returns the printable string for each device type
 ********************************************************************************/
char * device_type_str( uint32_t dev_type )
{
    if     ( dev_type == DEV_TYPE_RAM_SCL ) return "RAM_SCL";
    else if( dev_type == DEV_TYPE_ROM_SCL ) return "ROM_SCL";
    else if( dev_type == DEV_TYPE_FBF_SCL ) return "FBF_SCL";
    else if( dev_type == DEV_TYPE_IOB_TSR ) return "IOB_TSR";
    else if( dev_type == DEV_TYPE_IOC_BDV ) return "IOC_BDV";
    else if( dev_type == DEV_TYPE_IOC_HBA ) return "IOC_HBA";
    else if( dev_type == DEV_TYPE_IOC_SDC ) return "IOC_SDC";
    else if( dev_type == DEV_TYPE_IOC_SPI ) return "IOC_SPI";
    else if( dev_type == DEV_TYPE_IOC_RDK ) return "IOC_RDK";
    else if( dev_type == DEV_TYPE_MMC_TSR ) return "MMC_TSR";
    else if( dev_type == DEV_TYPE_DMA_SCL ) return "DMA_SCL";
    else if( dev_type == DEV_TYPE_NIC_CBF ) return "NIC_CBF";
    else if( dev_type == DEV_TYPE_TIM_SCL ) return "TIM_SCL";
    else if( dev_type == DEV_TYPE_TXT_TTY ) return "TXT_TTY";
    else if( dev_type == DEV_TYPE_ICU_XCU ) return "ICU_XCU";
    else if( dev_type == DEV_TYPE_PIC_TSR ) return "PIC_TSR";
    else                                    return "undefined";
}

/************************************************************************************
 * This function loads the arch_info.bin file into the boot cluster memory.
 ***********************************************************************************/
static void boot_archinfo_load()
{
    archinfo_header_t* header = (archinfo_header_t*)ARCHINFO_BASE;  
    
    // Load file into memory
    if (boot_fat32_load(ARCHINFO_PATHNAME, ARCHINFO_BASE, ARCHINFO_MAX_SIZE))
    {
        boot_printf("\n[BOOT ERROR]: boot_archinfo_load(): "
                    "<%s> file not found\n",
                    ARCHINFO_PATHNAME);
        boot_exit();
    }

    if (header->signature != ARCHINFO_SIGNATURE)
    {
        boot_printf("\n[BOOT_ERROR]: boot_archinfo_load(): "
                    "<%s> file signature should be %x\n",
                    ARCHINFO_PATHNAME, ARCHINFO_SIGNATURE);
        boot_exit();
    }

#if DEBUG_BOOT_INFO
boot_printf("\n[BOOT INFO] in %s : file %s loaded at address = %x\n",
            __FUNCTION__ , ARCHINFO_PATHNAME , ARCHINFO_BASE );
#endif

} // boot_archinfo_load()

/**************************************************************************************
 * This function loads the 'kernel.elf' file into the boot cluster memory buffer,
 * analyzes it, and places the the two seg_kcode & seg_kdata segments at their final
 * physical adresses (just after the preloader zone).        
 * It set the global variables defining the kernel layout.
 *************************************************************************************/
static void boot_kernel_load()
{
    Elf32_Ehdr * elf_header;      // pointer on kernel.elf header.  
    Elf32_Phdr * program_header;  // pointer on kernel.elf program header. 
    uint32_t     phdr_offset;     // program header offset in kernel.elf file. 
    uint32_t     segments_nb;     // number of segments in kernel.elf file. 
    uint32_t     seg_src_addr;    // segment address in kernel.elf file (source).
    uint32_t     seg_paddr;       // segment local physical address of segment 
    uint32_t     seg_offset;      // segment offset in kernel.elf file
    uint32_t     seg_filesz;      // segment size (bytes) in kernel.elf file 
    uint32_t     seg_memsz;       // segment size (bytes) in memory image.
    bool_t       kcode_found;     // kcode segment found.
    bool_t       kdata_found;     // kdata segment found.
    uint32_t     seg_id;          // iterator for segments loop.

#if DEBUG_BOOT_ELF 
boot_printf("\n[BOOT INFO] %s enters for file %s at cycle %d\n",
            __FUNCTION__ , KERNEL_PATHNAME , boot_get_proctime() );
#endif

    // Load kernel.elf file into memory buffer
    if ( boot_fat32_load(KERNEL_PATHNAME, KERN_BASE, KERN_MAX_SIZE) )
    {
        boot_printf("\n[BOOT ERROR] in %s : <%s> file not found\n",
                    KERNEL_PATHNAME);
        boot_exit();
    }

    // get pointer to kernel.elf header  
    elf_header = (Elf32_Ehdr*)KERN_BASE;

    // check signature 
    if ((elf_header->e_ident[EI_MAG0] != ELFMAG0)   ||
        (elf_header->e_ident[EI_MAG1] != ELFMAG1)   ||
        (elf_header->e_ident[EI_MAG2] != ELFMAG2)   ||
        (elf_header->e_ident[EI_MAG3] != ELFMAG3))
    {
        boot_printf("\n[BOOT_ERROR]: boot_kernel_load(): "
                    "<%s> is not an ELF file\n",
                    KERNEL_PATHNAME);
        boot_exit();
    }

    // Get program header table offset and number of segments
    phdr_offset     = elf_header->e_phoff;
    segments_nb     = elf_header->e_phnum;

    // Get program header table pointer
    program_header  = (Elf32_Phdr*)(KERN_BASE + phdr_offset);

    // loop on segments
    kcode_found = false;
    kdata_found = false;
    for (seg_id = 0; seg_id < segments_nb; seg_id++) 
    {
        if (program_header[seg_id].p_type == PT_LOAD)   // Found one loadable segment
        {
            // Get segment attributes.
            seg_paddr    = program_header[seg_id].p_paddr;   
            seg_offset   = program_header[seg_id].p_offset;
            seg_filesz   = program_header[seg_id].p_filesz;
            seg_memsz    = program_header[seg_id].p_memsz;

            // get segment base address in buffer
            seg_src_addr = (uint32_t)KERN_BASE + seg_offset;

            // Load segment to its final physical memory address
            boot_memcpy( (void*)seg_paddr, 
                         (void*)seg_src_addr, 
                         seg_filesz );

#if DEBUG_BOOT_ELF 
boot_printf("\n[BOOT INFO] in %s for file %s : found loadable segment\n"
            "   base = %x / size = %x\n",
            __FUNCTION__ , KERNEL_PATHNAME , seg_paddr , seg_memsz );
#endif

            // Fill remaining memory with zero if (filesz < memsz).
            if( seg_memsz < seg_filesz )
            {
                boot_memset( (void*)(seg_paddr + seg_filesz), 0, seg_memsz - seg_filesz);
            }

            // Note: we suppose that the 'kernel.elf' file contains only 2 
            // loadable segments ktext & kdata and that the main 
            // difference between these two is the WRITE permission: ktext 
            // contains read-only instructions and read_only data,
            // while kdata contains writable data.

            if ((program_header[seg_id].p_flags & PF_W) == 0)  // kcode segment
            {
                if( kcode_found )
                {
                    boot_printf("\n[BOOT_ERROR] in %s for file %s :\n"
                                "   two loadable kcode segments found\n",
                                __FUNCTION__ , KERNEL_PATHNAME );
                    boot_exit();
                } 

                kcode_found     = true;
                seg_kcode_base = seg_paddr;
                seg_kcode_size = seg_memsz;
            }
            else                                               // kdata segment
            {
                if( kdata_found )
                {
                    boot_printf("\n[BOOT_ERROR] in %s for file %s :\n"
                                "   two loadable kdata segments found\n",
                                __FUNCTION__ , KERNEL_PATHNAME );
                    boot_exit();
                } 

                kdata_found     = true;
                seg_kdata_base = seg_paddr;
                seg_kdata_size = seg_memsz;
            }
        }
    }

    // check kcode & kdata segments found
    if( kcode_found == false )
    {
        boot_printf("\n[BOOT_ERROR] in %s for file %s :\n"
                    "   kcode segment not found\n",
                    __FUNCTION__ , KERNEL_PATHNAME );
        boot_exit();
    }
    if( kdata_found == false )
    {
        boot_printf("\n[BOOT_ERROR] in %s for file %s :\n"
                    "   kdata segment not found\n",
                    __FUNCTION__ , KERNEL_PATHNAME );
        boot_exit();
    }

    // set entry point 
    kernel_entry = (uint32_t)elf_header->e_entry;

#if DEBUG_BOOT_ELF 
boot_printf("\n[BOOT INFO] %s successfully completed for file %s at cycle %d\n",
            __FUNCTION__ , KERNEL_PATHNAME , boot_get_proctime() );
#endif

} // boot_kernel_load()

/*************************************************************************************
 * This function initializes the  boot_info_t structure for a given cluster. 
 * @ boot_info  : pointer to local boot_info_t structure  
 * @ cxy        : cluster identifier                    
 ************************************************************************************/
static void boot_info_init( boot_info_t * boot_info,
                            cxy_t         cxy )
{
    archinfo_header_t  * header;
    archinfo_core_t    * core_base;     
    archinfo_cluster_t * cluster_base; 
    archinfo_device_t  * device_base;
    archinfo_irq_t     * irq_base;  

    archinfo_cluster_t * cluster; 
    archinfo_cluster_t * my_cluster = NULL;   // target cluster
    archinfo_cluster_t * io_cluster = NULL;   // cluster containing ext. peripherals

    archinfo_core_t    * core;
    uint32_t             core_id; 
    archinfo_device_t  * device;
    uint32_t             device_id;
    archinfo_irq_t     * irq; 
    uint32_t             irq_id;
 
    boot_device_t      * boot_dev; 

#if DEBUG_BOOT_INFO
boot_printf("\n[BOOT INFO] %s : enter for cluster %x at cycle %d\n",
            __FUNCTION__ , cxy , boot_get_proctime() );
#endif

    // get pointer on ARCHINFO header  and on the four arch_info arrays
    header       = (archinfo_header_t*)ARCHINFO_BASE;
    core_base    = archinfo_get_core_base   (header);
    cluster_base = archinfo_get_cluster_base(header);
    device_base  = archinfo_get_device_base (header);
    irq_base     = archinfo_get_irq_base    (header);

    // Initialize global platform parameters
    boot_info->x_size       = header->x_size;
    boot_info->y_size       = header->y_size;
    boot_info->x_width      = header->x_width;
    boot_info->y_width      = header->y_width;
    boot_info->paddr_width  = header->paddr_width;
    boot_info->io_cxy       = header->io_cxy;

    // Initialize kernel segments
    boot_info->kernel_code_start = seg_kcode_base;
    boot_info->kernel_code_end   = seg_kcode_base + seg_kcode_size;
    boot_info->kernel_data_start = seg_kdata_base;
    boot_info->kernel_data_end   = seg_kdata_base + seg_kdata_size;

    // loop on arch_info clusters to get relevant pointers
    for (cluster =  cluster_base;
         cluster < &cluster_base[header->x_size * header->y_size];
         cluster++)
    {
        if( cluster->cxy  == cxy )            my_cluster = cluster;
        if( cluster->cxy  == header->io_cxy ) io_cluster = cluster;
    }

    if( my_cluster == NULL ) 
    {
        boot_printf("\n[ERROR] in %s : cannot found cluster %x in arch_info\n",
                    __FUNCTION__ , cxy );
        boot_exit();
    }

    if( io_cluster == NULL ) 
    {
        boot_printf("\n[ERROR] in %s : cannot found io_cluster %x in arch_info\n",
                    __FUNCTION__ , header->io_cxy );
        boot_exit();
    }

    // loop on all arch-info peripherals in IO_cluster, 
    // to initialize the boot_info array of external peripherals

#if DEBUG_BOOT_INFO
boot_printf("\n[BOOT INFO] %s : External peripherals\n", __FUNCTION__ );
#endif

    device_id = 0;
    for (device = &device_base[io_cluster->device_offset];
         device < &device_base[io_cluster->device_offset + io_cluster->devices];
         device++ )
    {
        // initialise one entry for each external peripheral
        if( (device->type != DEV_TYPE_RAM_SCL) &&
            (device->type != DEV_TYPE_ICU_XCU) &&
            (device->type != DEV_TYPE_MMC_TSR) &&
            (device->type != DEV_TYPE_DMA_SCL) ) 
        {
            boot_dev = &boot_info->ext_dev[device_id];

            boot_dev->type     = device->type;
            boot_dev->base     = device->base;
            boot_dev->size     = device->size;
            boot_dev->channels = device->channels;
            boot_dev->param0   = device->arg0;    
            boot_dev->param1   = device->arg1;    
            boot_dev->param2   = device->arg2;    
            boot_dev->param3   = device->arg3;    
            boot_dev->irqs     = device->irqs;    

            device_id++;
        }

#if DEBUG_BOOT_INFO
boot_printf("  - %s : base = %l / size = %l / channels = %d / irqs = %d\n",
            device_type_str( device->type ) , device->base , device->size ,
            device->channels , device->irqs );   
#endif
    
        // Initialize array of irq descriptors for PIC
        if (device->type == DEV_TYPE_PIC_TSR) 
        {
            for (irq_id = 0; irq_id < CONFIG_MAX_IRQS_PER_PIC; irq_id++)
            {
                boot_dev->irq[irq_id].valid  = 0;
            }

            for (irq = &irq_base[device->irq_offset];
                 irq < &irq_base[device->irq_offset + device->irqs];
                 irq++)
            {
                boot_dev->irq[irq->port].valid    = 1;
                boot_dev->irq[irq->port].dev_type = irq->dev_type;
                boot_dev->irq[irq->port].channel  = irq->channel;
                boot_dev->irq[irq->port].is_rx    = irq->is_rx;

#if DEBUG_BOOT_INFO
boot_printf("    . irq_port = %d / source = %s / channel = %d / is_rx = %d\n",
            irq->port , device_type_str( irq->dev_type ) , irq->channel , irq->is_rx );
#endif
            }
        }
    }

    // initialize number of external peripherals
    boot_info->ext_dev_nr = device_id;

    // Initialize cluster specific resources
    boot_info->cxy  = my_cluster->cxy;

#if DEBUG_BOOT_INFO
boot_printf("\n[BOOT INFO] %s : cores in cluster %x\n", __FUNCTION__ );
#endif

    // Initialize array of core descriptors
    core_id = 0;
    for (core = &core_base[my_cluster->core_offset];
         core < &core_base[my_cluster->core_offset + my_cluster->cores];
         core++ )
    {
        boot_info->core[core_id].gid = (gid_t)core->gid;
        boot_info->core[core_id].lid = (lid_t)core->lid; 
        boot_info->core[core_id].cxy = (cxy_t)core->cxy;

#if DEBUG_BOOT_INFO
boot_printf("  - core_gid = %x : cxy = %x / lid = %d\n", 
            core->gid , core->cxy , core->lid );
#endif
        core_id++;
    }

    // Initialize number of cores in my_cluster
    boot_info->cores_nr = core_id;

    // loop on all peripherals in my_cluster to initialise
    // boot_info array of internal peripherals in my_cluster

#if DEBUG_BOOT_INFO
boot_printf("\n[BOOT INFO] %s : internal peripherals in cluster %x\n", __FUNCTION__ );
#endif

    device_id = 0;
    for (device = &device_base[my_cluster->device_offset];
         device < &device_base[my_cluster->device_offset + my_cluster->devices];
         device++ )
    {
        // initialise one entry for each internal peripheral
        if( (device->type == DEV_TYPE_RAM_SCL) ||
            (device->type == DEV_TYPE_ICU_XCU) ||
            (device->type == DEV_TYPE_MMC_TSR) ||
            (device->type == DEV_TYPE_DMA_SCL) ) 
        {
            boot_dev = &boot_info->int_dev[device_id];

            boot_dev->type     = device->type;
            boot_dev->base     = device->base;
            boot_dev->size     = device->size;
            boot_dev->channels = device->channels;
            boot_dev->param0   = device->arg0;    
            boot_dev->param1   = device->arg1;    
            boot_dev->param2   = device->arg2;    
            boot_dev->param3   = device->arg3;    
            boot_dev->irqs     = device->irqs; 
   
            device_id++;
        }

#if DEBUG_BOOT_INFO
boot_printf("  - %s : base = %l / size = %l / channels = %d / irqs = %d\n",
            device_type_str( device->type ) , device->base , device->size ,
            device->channels , device->irqs );   
#endif

        // Initialize information about physical memory in cluster
        if (device->type == DEV_TYPE_RAM_SCL)
        {
            // Compute total number of physical memory pages in cluster
            boot_info->pages_nr = device->size >> CONFIG_PPM_PAGE_SHIFT;

            // Get the last address allocated for the kernel segments
            uint32_t end;
            if( boot_info->kernel_code_end > boot_info->kernel_data_end )
            {
                end = boot_info->kernel_code_end;
            }
            else
            {
                end = boot_info->kernel_data_end;
            }
                
            // Compute the number of pages allocated for the kernel.
            if( (end & CONFIG_PPM_PAGE_MASK) == 0 )
            {
                boot_info->pages_offset = end >> CONFIG_PPM_PAGE_SHIFT;
            }
            else
            {
                boot_info->pages_offset = (end >> CONFIG_PPM_PAGE_SHIFT) + 1;
            }

#if DEBUG_BOOT_INFO
boot_printf("    . physical memory : %x pages / first free page = %x\n",
            boot_info->pages_nr , boot_info->pages_offset );
#endif
        }
            
        // Initialize array of irq descriptors for XCU 
        if (device->type == DEV_TYPE_ICU_XCU) 
        {
            for (irq_id = 0; irq_id < CONFIG_MAX_HWIS_PER_ICU; irq_id++)
            {
                boot_dev->irq[irq_id].valid  = 0;
            }

            for (irq = &irq_base[device->irq_offset];
                 irq < &irq_base[device->irq_offset + device->irqs];
                 irq++)
            {
                boot_dev->irq[irq->port].valid    = 1;
                boot_dev->irq[irq->port].dev_type = irq->dev_type;
                boot_dev->irq[irq->port].channel  = irq->channel;
                boot_dev->irq[irq->port].is_rx    = irq->is_rx;

#if DEBUG_BOOT_INFO
boot_printf("    . irq_port = %d / source = %s / channel = %d / is_rx = %d\n",
            irq->port , device_type_str( irq->dev_type ) , irq->channel , irq->is_rx );
#endif

            }
        }
    }

    // initialize number of internal peripherals in my_cluster
    boot_info->int_dev_nr = device_id;

    // set boot_info signature
    boot_info->signature = BOOT_INFO_SIGNATURE;

} // boot_info_init()

/***********************************************************************************
 * This function check the local boot_info_t structure for a given core.
 * @ boot_info  : pointer to local 'boot_info_t' structure to be checked.
 * @ lid        : core local identifier, index the core descriptor table.
 **********************************************************************************/
static void boot_check_core( boot_info_t * boot_info, 
                             lid_t         lid)
{
    gid_t         gid;        // global hardware identifier of this core
    boot_core_t * this;       // BOOT_INFO core descriptor of this core.  

    // Get core hardware identifier 
    gid = (gid_t)boot_get_procid();

    // get pointer on core descriptor
    this = &boot_info->core[lid];

    if ( (this->gid != gid) ||  (this->cxy != boot_info->cxy) )
    {
        boot_printf("\n[BOOT ERROR] in boot_check_core() :\n"
                    " - boot_info cxy = %x\n"
                    " - boot_info lid = %d\n"
                    " - boot_info gid = %x\n"
                    " - actual    gid = %x\n",
                    this->cxy , this->lid , this->gid , gid );
        boot_exit();
    }

} // boot_check_core()

/*********************************************************************************
 * This function is called by CP0 in cluster(0,0) to activate all other CP0s.
 * It returns the number of CP0s actually activated.
 ********************************************************************************/
static uint32_t boot_wake_all_cp0s()
{
    archinfo_header_t*  header;         // Pointer on ARCHINFO header
    archinfo_cluster_t* cluster_base;   // Pointer on ARCHINFO clusters base
    archinfo_cluster_t* cluster;        // Iterator for loop on clusters
    archinfo_device_t*  device_base;    // Pointer on ARCHINFO devices base
    archinfo_device_t*  device;         // Iterator for loop on devices
    uint32_t            cp0_nb = 0;     // CP0s counter

    header       = (archinfo_header_t*)ARCHINFO_BASE;
    cluster_base = archinfo_get_cluster_base(header);
    device_base  = archinfo_get_device_base (header); 

    // loop on all clusters 
    for (cluster = cluster_base;
         cluster < &cluster_base[header->x_size * header->y_size];
         cluster++)
    {
        // Skip boot cluster.
        if (cluster->cxy == BOOT_CORE_CXY)
            continue;
            
        // Skip clusters without core (thus without CP0).
        if (cluster->cores == 0)
            continue;

        // Skip clusters without device (thus without XICU).
        if (cluster->devices == 0)
            continue;

        // search XICU device associated to CP0, and send a WTI to activate it 
        for (device = &device_base[cluster->device_offset];
             device < &device_base[cluster->device_offset + cluster->devices];
             device++)
        {
            if (device->type == DEV_TYPE_ICU_XCU)
            {

#if DEBUG_BOOT_WAKUP
boot_printf("\n[BOOT] core[%x][0] activated at cycle %d\n",
            cluster->cxy , boot_get_proctime );
#endif

                boot_remote_sw((xptr_t)device->base, (uint32_t)boot_entry);
                cp0_nb++;
            }
        }
    }
    return cp0_nb;

} // boot_wake_cp0()

/*********************************************************************************
 * This function is called by all CP0 to activate all local CPi cores. 
 * @ boot_info  : pointer to local 'boot_info_t' structure, used to find
 *                the XICU device associated with local CPi base addresses.
 *********************************************************************************/
static void boot_wake_local_cores(boot_info_t * boot_info)
{
    boot_device_t *  device;         // Iterator on devices
    unsigned int     core_id;        // Iterator on cores

    // loop on devices to find XCU
    for (device = &boot_info->int_dev[0];
         device < &boot_info->int_dev[boot_info->int_dev_nr];
         device++)
    {
        if (device->type == DEV_TYPE_ICU_XCU)
        {
            // loop on cores
            for (core_id = 1; core_id < boot_info->cores_nr; core_id++)
            {

#if DEBUG_BOOT_WAKUP
boot_printf("\n[BOOT] core[%x][%d] activated at cycle %d\n",
             boot_info->cxy , core_id , boot_get_proctime() );
#endif
                boot_remote_sw((xptr_t) (device->base + (core_id << 2)),
                               (uint32_t)boot_entry); 
            }
        }
    }
} // boot_wake_local_cores()


/*********************************************************************************
 * This main function of the boot-loader is called by the  boot_entry()  
 * function, and executed by all cores. 
 * The arguments values are computed by the boot_entry code.
 * @ lid    : core local identifier,
 * @ cxy    : cluster identifier,
 *********************************************************************************/
void boot_loader( lid_t lid, 
                  cxy_t cxy )
{
    boot_info_t * boot_info;       // pointer on local boot_info_t structure

    if (lid == 0) 
    {
        /****************************************************
         * PHASE A : only CP0 in boot cluster executes it 
         ***************************************************/
        if (cxy == BOOT_CORE_CXY)
        {
            boot_printf("\n[BOOT] core[%x][%d] enters at cycle %d\n",
                        cxy , lid , boot_get_proctime() );

            // Initialize IOC driver 
            if      (USE_IOC_BDV) boot_bdv_init();
            else if (USE_IOC_HBA) boot_hba_init();
            // else if (USE_IOC_SDC) boot_sdc_init();
            // else if (USE_IOC_SPI) boot_spi_init();
            else if (!USE_IOC_RDK)
            {
                boot_printf("\n[BOOT ERROR] in %s : no IOC driver\n");
                boot_exit();
            }

            // Initialize FAT32. 
            boot_fat32_init();

            // Load the 'kernel.elf' file into memory from IOC, and set   
            // the global variables defining the kernel layout      
            boot_kernel_load();

            boot_printf("\n[BOOT] core[%x][%d] loaded kernel at cycle %d\n",
                        cxy , lid , boot_get_proctime() );

            // Load the arch_info.bin file into memory.
            boot_archinfo_load();

            // Get local boot_info_t structure base address.
            // It is the first structure in the .kdata segment. 
            boot_info = (boot_info_t *)seg_kdata_base;

            // Initialize local boot_info_t structure.
            boot_info_init( boot_info , cxy );

            // check boot_info signature
            if (boot_info->signature != BOOT_INFO_SIGNATURE)
            {
                boot_printf("\n[BOOT ERROR] in %s reported by core[%x][%d]\n"
                            "  illegal boot_info signature / should be %x\n",
                            __FUNCTION__ , cxy , lid , BOOT_INFO_SIGNATURE );
                boot_exit();
            }

            boot_printf("\n[BOOT] core[%x][%d] loaded boot_info at cycle %d\n",
                        cxy , lid , boot_get_proctime() );

            // Check core information.
            boot_check_core(boot_info, lid);

            // Activate other CP0s / get number of active CP0s
            active_cp0s_nr = boot_wake_all_cp0s() + 1;

            // Wait until all clusters (i.e all CP0s) ready to enter kernel.
            boot_remote_barrier( XPTR( BOOT_CORE_CXY , &global_barrier ) ,
                                 active_cp0s_nr );

            // activate other local cores
            boot_wake_local_cores( boot_info );

            // Wait until all local cores in cluster ready
            boot_remote_barrier( XPTR( cxy , &local_barrier ) , 
                                 boot_info->cores_nr );
        }
        /******************************************************************
         * PHASE B : all CP0s other than CP0 in boot cluster execute it
         *****************************************************************/
        else
        {
            // at this point, all INSTRUCTION address extension registers
            // point on cluster(0,0), but the DATA extension registers point
            // already on the local cluster to use the local stack. 
            // To access the bootloader global variables we must first copy
            // the boot code (data and instructions) in the local cluster.
            boot_remote_memcpy( XPTR( cxy           , BOOT_BASE ),
                                XPTR( BOOT_CORE_CXY , BOOT_BASE ),
                                BOOT_MAX_SIZE );

            // from now, it is safe to refer to the boot code global variables 
            boot_printf("\n[BOOT] core[%x][%d] replicated boot code at cycle %d\n",
                        cxy , lid , boot_get_proctime() );

            // switch to the INSTRUCTION local memory space, to avoid contention.
            asm volatile("mtc2  %0, $25" :: "r"(cxy));

            // Copy the arch_info.bin file into the local memory.
            boot_remote_memcpy(XPTR(cxy,           ARCHINFO_BASE),
                               XPTR(BOOT_CORE_CXY, ARCHINFO_BASE),
                               ARCHINFO_MAX_SIZE );

            boot_printf("\n[BOOT] core[%x][%d] replicated arch_info at cycle %d\n",
                        cxy , lid , boot_get_proctime() );

            // Copy the kcode segment into local memory 
            boot_remote_memcpy( XPTR( cxy           , seg_kcode_base ),
                                XPTR( BOOT_CORE_CXY , seg_kcode_base ),
                                seg_kcode_size );

            // Copy the kdata segment into local memory
            boot_remote_memcpy( XPTR( cxy           , seg_kdata_base ),
                                XPTR( BOOT_CORE_CXY , seg_kdata_base ),
                                seg_kdata_size );

            boot_printf("\n[BOOT] core[%x][%d] replicated kernel code at cycle %d\n",
                        cxy , lid , boot_get_proctime() );

            // Get local boot_info_t structure base address.
            boot_info = (boot_info_t*)seg_kdata_base;

            // Initialize local boot_info_t structure.
            boot_info_init( boot_info , cxy );

            // Check core information.
            boot_check_core( boot_info , lid );

            // get number of active clusters from BOOT_CORE cluster
            uint32_t count = boot_remote_lw( XPTR( BOOT_CORE_CXY , &active_cp0s_nr ) );

            // Wait until all clusters (i.e all CP0s) ready to enter kernel
            boot_remote_barrier( XPTR( BOOT_CORE_CXY , &global_barrier ) , count );

            // activate other local cores
            boot_wake_local_cores( boot_info );

            // Wait until all local cores in cluster ready
            boot_remote_barrier( XPTR( cxy , &local_barrier ) , 
                                 boot_info->cores_nr );
        }
    }
    else
    {
        /***************************************************************
         * PHASE C: all non CP0 cores in all clusters execute it
         **************************************************************/

        // Switch to the INSTRUCTIONS local memory space 
        // to avoid contention at the boot cluster.
        asm volatile("mtc2  %0, $25" :: "r"(cxy));

        // Get local boot_info_t structure base address.
        boot_info = (boot_info_t *)seg_kdata_base;

        // Check core information
        boot_check_core(boot_info, lid);

        // Wait until all local cores in cluster ready
        boot_remote_barrier( XPTR( cxy , &local_barrier ) , boot_info->cores_nr );
    }

    // Each core compute address of a temporary kernel stack 
    // in the upper part of the local cluster memory...
    uint32_t stack_ptr = ((boot_info->pages_nr - lid) << 12) - 16;

    // All cores initialise stack pointer,
    // reset the BEV bit in status register,
    // register "boot_info" argument in a0,
    // and jump to kernel_entry. 
    asm volatile( "mfc0  $27,  $12           \n"
                  "lui   $26,  0xFFBF        \n"
                  "ori   $26,  $26,  0xFFFF  \n"
                  "and   $27,  $27,  $26     \n"
                  "mtc0  $27,  $12           \n" 
                  "move  $4,   %0            \n"
                  "move  $29,  %1            \n"
                  "jr    %2                  \n"
                  :: "r"(boot_info) , "r"(stack_ptr) , "r"(kernel_entry) );

} // boot_loader()