source: trunk/hal/x86_64/core/hal_init.c @ 571

/*
 * hal_init.c - C initialization procedure for x86.
 *
 * Copyright (c) 2017 Maxime Villard
 *
 * This file is part of ALMOS-MKH.
 *
 * ALMOS-MKH is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by
 * the Free Software Foundation; version 2.0 of the License.
 *
 * ALMOS-MKH is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with ALMOS-MKH; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include <hal_kernel_types.h>
#include <hal_boot.h>
#include <hal_multiboot.h>
#include <hal_segmentation.h>
#include <hal_acpi.h>
#include <hal_apic.h>
#include <hal_kentry.h>
#include <hal_internal.h>
#include <hal_register.h>

#include <hal_remote.h>
#include <hal_irqmask.h>

#include <memcpy.h>
#include <thread.h>
#include <string.h>
#include <process.h>
#include <printk.h>
#include <vmm.h>
#include <core.h>
#include <cluster.h>
#include <chdev.h>

#include <boot_info.h>

void kernel_init(boot_info_t *info);

static void gdt_create( void );
static void idt_create( void );
void cpu_tls_init(size_t lid);
void cpu_identify( void );
void cpu_attach(size_t lid);

size_t mytest __in_kdata = 0;

struct multiboot_info mb_info __in_kdata;
char mb_loader_name[PAGE_SIZE] __in_kdata;
uint8_t mb_mmap[PAGE_SIZE] __in_kdata;
char mb_cmdline[PAGE_SIZE] __in_kdata;

bool_t has_smp __in_kdata = true;
size_t ncpu __in_kdata = 0;
static boot_info_t btinfo __in_kdata;

/* x86-specific per-cluster structures */
uint8_t gdtstore[PAGE_SIZE] __in_kdata;
uint8_t idtstore[PAGE_SIZE] __in_kdata;

/* x86-specific per-cpu structures */
typedef struct {
        bool_t valid;
        struct tss tss;
        struct tls tls;
        uint8_t boot_stack[STKSIZE];
        uint8_t intr_stack[STKSIZE];
        uint8_t dbfl_stack[STKSIZE];
        uint8_t nmfl_stack[STKSIZE];
} percpu_archdata_t;
percpu_archdata_t cpudata[CONFIG_MAX_LOCAL_CORES] __in_kdata;

/* -------------------------------------------------------------------------- */

/*
 * Configure the features of the system depending on the multiboot info.
 */
static void multiboot_init( void )
{
        size_t mmap_length = mb_info.mi_mmap_length;
        uint8_t *mmap_addr = (uint8_t *)&mb_mmap;
        size_t i;

        if (!(mb_info.mi_flags & MULTIBOOT_INFO_HAS_MMAP))
                x86_panic("No mmap");

        i = 0;
        while (i < mmap_length) {
                struct multiboot_mmap *mm;

                mm = (struct multiboot_mmap *)(mmap_addr + i);

                x86_printf("-> [%Z, %Z] %s\n", mm->mm_base_addr,
                    mm->mm_base_addr + mm->mm_length,
                    (mm->mm_type == 1) ? "ram" : "rsv" );

                i += mm->mm_size + 4;
        }

        if (mb_info.mi_flags & MULTIBOOT_INFO_HAS_CMDLINE) {
                mb_cmdline[PAGE_SIZE-1] = '\0';
                if (strstr(mb_cmdline, "--no-smp")) {
                        has_smp = false;
                        x86_printf("[+] SMP disabled\n");
                }
        }
}

/* -------------------------------------------------------------------------- */

static size_t init_bootinfo_pages_nr( void )
{
        size_t mmap_length = mb_info.mi_mmap_length;
        uint8_t *mmap_addr = (uint8_t *)&mb_mmap;
        paddr_t maxpa, pa;
        size_t i;

        i = 0;
        maxpa = 0;
        while (i < mmap_length) {
                struct multiboot_mmap *mm;

                mm = (struct multiboot_mmap *)(mmap_addr + i);

                if (mm->mm_type == 1) {
                        pa = mm->mm_base_addr + mm->mm_length;
                        if (pa > maxpa)
                                maxpa = pa;
                }

                i += mm->mm_size + 4;
        }

        return (maxpa / PAGE_SIZE);
}

static size_t init_bootinfo_rsvd(boot_rsvd_t *rsvd)
{
        size_t mmap_length = mb_info.mi_mmap_length;
        uint8_t *mmap_addr = (uint8_t *)&mb_mmap;
        size_t i, rsvd_nr;

        memset(rsvd, 0, sizeof(boot_rsvd_t) * CONFIG_PPM_MAX_RSVD);

        i = 0, rsvd_nr = 0;
        while (i < mmap_length) {
                struct multiboot_mmap *mm;

                mm = (struct multiboot_mmap *)(mmap_addr + i);

                if (mm->mm_type != 1) {
                        rsvd[rsvd_nr].first_page =
                            rounddown(mm->mm_base_addr, PAGE_SIZE) / PAGE_SIZE;
                        rsvd[rsvd_nr].npages =
                            roundup(mm->mm_length, PAGE_SIZE) / PAGE_SIZE;
                        rsvd_nr++;
                        if (rsvd_nr == CONFIG_PPM_MAX_RSVD)
                                x86_panic("too many memory holes");
                }

                i += mm->mm_size + 4;
        }

        return rsvd_nr;
}

static void init_bootinfo_core(boot_core_t *core, size_t n)
{
        size_t i;

        // XXX: not necessarily contiguous
        for (i = 0; i < n; i++) {
                memset(&core[i], 0, sizeof(boot_core_t));

                core[i].gid = i;
                core[i].lid = i;
                core[i].cxy = 0;
        }
}

static void init_bootinfo_ioc(boot_device_t *dev)
{
        memset(dev, 0, sizeof(boot_device_t));

        dev->base = 0;
        dev->type = (DEV_FUNC_IOC << 16) | IMPL_IOC_BDV;
        dev->channels = 1;
}

static void init_bootinfo_pic(boot_device_t *dev)
{
        memset(dev, 0, sizeof(boot_device_t));

        dev->base = 0;
        dev->type = (DEV_FUNC_PIC << 16) | IMPL_PIC_I86;
        dev->channels = 1;
        dev->param0 = 0;
        dev->param1 = 0;
        dev->param2 = 0;
        dev->param3 = 0;

        dev->irqs = 16;

        /* ATA */
        dev->irq[IRQ_ATA0].dev_type = (DEV_FUNC_IOC << 16) | IMPL_IOC_BDV;
        dev->irq[IRQ_ATA0].channel = 0;
        dev->irq[IRQ_ATA0].is_rx = 0;
        dev->irq[IRQ_ATA0].valid = 1;
}

static void init_bootinfo_txt(boot_device_t *dev)
{
        memset(dev, 0, sizeof(boot_device_t));

        dev->base = 0;
        dev->type = (DEV_FUNC_TXT << 16) | IMPL_TXT_RS2;
        dev->channels = 4;
        dev->param0 = 0;
        dev->param1 = 0;
        dev->param2 = 0;
        dev->param3 = 0;
}

static void init_bootinfo(boot_info_t *info)
{
        size_t offset;

        memset(info, 0, sizeof(boot_info_t));

        info->signature = 0;

        info->paddr_width = 0;
        info->x_width = 1;
        info->y_width = 1;
        info->x_size = 1;
        info->y_size = 1;
        info->io_cxy = 0;

        info->ext_dev_nr = 3;
        init_bootinfo_txt(&info->ext_dev[0]);
        init_bootinfo_pic(&info->ext_dev[1]);
        init_bootinfo_ioc(&info->ext_dev[2]);

        info->cxy = 0;
        if (has_smp) {
                info->cores_nr = ncpu;
        } else {
                info->cores_nr = 1;
        }
        init_bootinfo_core((boot_core_t *)&info->core, info->cores_nr);

        info->rsvd_nr = init_bootinfo_rsvd((boot_rsvd_t *)&info->rsvd);

        /* TODO: dev_mmc */
        /* TODO: dev_dma */

        offset = hal_gpt_bootstrap_uniformize();
        info->pages_offset = offset / PAGE_SIZE;
        info->pages_nr = init_bootinfo_pages_nr();
}

/* -------------------------------------------------------------------------- */

static uint32_t cpuN_booted __in_kdata;

void start_secondary_cpus( void )
{
        pt_entry_t flags = PG_V | PG_KW;
        extern vaddr_t cpuN_boot_trampoline;
        extern vaddr_t cpuN_boot_trampoline_end;
        extern paddr_t smp_L4pa;
        extern vaddr_t smp_stkva;
        extern paddr_t L4paddr;
        size_t i, sz;

        smp_L4pa = L4paddr;

        /* map the SMP trampoline (identity) */
        vaddr_t trampva = (vaddr_t)SMP_TRAMPOLINE_PA;
        hal_gpt_maptree_area(trampva, trampva + PAGE_SIZE);
        hal_gpt_enter(trampva, SMP_TRAMPOLINE_PA, flags);

        /* copy it */
        sz = (size_t)&cpuN_boot_trampoline_end - (size_t)&cpuN_boot_trampoline;
        memcpy((void *)trampva, (void *)&cpuN_boot_trampoline, sz);

        for (i = 0; i < CONFIG_MAX_LOCAL_CORES; i++) {
                if (i == 0 || !cpudata[i].valid) {
                        continue;
                }

                smp_stkva = ((vaddr_t)&cpudata[i].boot_stack + STKSIZE) & ~0xF;

                cpuN_booted = 0;
                boot_cpuN(i, SMP_TRAMPOLINE_PA);
                while (!hal_atomic_cas(&cpuN_booted, 1, 0)) {
                        pause();
                }
        }

        // XXX: unmap the trampoline
}

void init_x86_64_cpuN( void )
{
        lid_t lid;

        cli();

        lid = hal_lapic_gid();

        cpu_attach(lid);
        x86_printf("[cpu%z] cpu_attach called\n", (uint64_t)lid);

        cpu_tls_init(lid);
        x86_printf("[cpu%z] cput_tls_init called\n", (uint64_t)lid);

        cpu_lapic_init();
        x86_printf("[cpu%z] cpu_lapic_init called\n", (uint64_t)lid);

        cpuN_booted = 1;

        if (lid == 1) {
                hal_ioapic_disable_irq(IRQ_KEYBOARD);
                hal_ioapic_bind_irq(IRQ_KEYBOARD, IOAPIC_KEYBOARD_VECTOR, 1);
                hal_ioapic_enable_irq(IRQ_KEYBOARD);
        }

        kernel_init(&btinfo);

        reg_t dummy;
        hal_enable_irq(&dummy);

        while (1);
}

/* -------------------------------------------------------------------------- */

static void apic_map( void )
{
        extern vaddr_t lapic_va, ioapic_va;
        extern paddr_t lapic_pa, ioapic_pa;

        lapic_va = hal_gpt_bootstrap_valloc(1); // XXX: should be shared
        hal_gpt_enter(lapic_va, lapic_pa, PG_V|PG_KW|PG_NX|PG_N);

        ioapic_va = hal_gpt_bootstrap_valloc(1); // XXX: should be shared
        hal_gpt_enter(ioapic_va, ioapic_pa, PG_V|PG_KW|PG_NX|PG_N);
}

void init_x86_64(paddr_t firstpa)
{
        cli();

        /* Initialize the serial port */
        hal_com_init_early();

        x86_printf("[+] init_x86_64 called\n");

        /* Create the global structures */
        gdt_create();
        idt_create();

        /* Identify the features of the cpu */
        cpu_identify();

        /* Attach cpu0 */
        cpu_attach(0);
        x86_printf("[+] cpu_attach called\n");

        x86_printf("[+] bootloader: '%s'\n", mb_loader_name);

        multiboot_init();
        x86_printf("[+] multiboot_init called\n");

        hal_gpt_init(firstpa);
        x86_printf("[+] hal_gpt_init called\n");

        hal_acpi_init();
        x86_printf("[+] hal_acpi_init called\n");

        hal_gpt_bootstrap_reset();
        x86_printf("[+] hal_gpt_bootstrap_reset called\n");

        apic_map();
        x86_printf("[+] apic_map called\n");

        hal_apic_init();
        cpu_lapic_init();
        x86_printf("[+] hal_apic_init called\n");

        cpu_tls_init(0);
        x86_printf("[+] cput_tls_init called\n");

        mytest = 0;
        x86_printf("-> mytest = %z\n", mytest);
        void *hoho = &init_x86_64;
        xptr_t myptr = XPTR(0, &mytest);

        hal_remote_spt(myptr, hoho);
        x86_printf("-> mytest = %Z\n", hal_remote_lpt(myptr));

        init_bootinfo(&btinfo);

        if (has_smp) {
                start_secondary_cpus();
        }

        kernel_init(&btinfo);

        x86_printf("[+] kernel_init called\n");

        reg_t dummy;
        hal_enable_irq(&dummy);

        while (1);

/*
        void *ptr;

        khm_t *khm = &LOCAL_CLUSTER->khm;
        ptr = khm_alloc(khm, 10);
        memset(ptr, 0, 10);
        khm_free(ptr);


        kcm_t *kcm = &LOCAL_CLUSTER->kcm;
        ptr = kcm_alloc(kcm);
        memset(ptr, 0, 1);
        kcm_free(ptr);

        ptr = ppm_alloc_pages(1);
        ppm_free_pages(ptr);
*/
}

/* -------------------------------------------------------------------------- */

void cpu_activate(uint32_t gid)
{
        cpudata[gid].valid = true;
}

static void setregion(struct region_descriptor *rd, void *base, uint16_t limit)
{
        rd->rd_limit = limit;
        rd->rd_base = (uint64_t)base;
}

/* -------------------------------------------------------------------------- */

static void gdt_set_memseg(struct gdt_memseg *sd, void *base, size_t limit,
        int type, int dpl, int gran, bool_t is64)
{
        sd->sd_lolimit = (unsigned)limit;
        sd->sd_lobase = (unsigned long)base;
        sd->sd_type = type;
        sd->sd_dpl = dpl;
        sd->sd_p = 1;
        sd->sd_hilimit = (unsigned)limit >> 16;
        sd->sd_avl = 0;
        sd->sd_long = is64;
        sd->sd_def32 = !is64;
        sd->sd_gran = gran;
        sd->sd_hibase = (unsigned long)base >> 24;
}

static void gdt_set_sysseg(struct gdt_sysseg *sd, void *base, size_t limit,
        int type, int dpl, int gran)
{
        memset(sd, 0, sizeof *sd);
        sd->sd_lolimit = (unsigned)limit;
        sd->sd_lobase = (uint64_t)base;
        sd->sd_type = type;
        sd->sd_dpl = dpl;
        sd->sd_p = 1;
        sd->sd_hilimit = (unsigned)limit >> 16;
        sd->sd_gran = gran;
        sd->sd_hibase = (uint64_t)base >> 24;
}

static void gdt_create( void )
{
        memset(&gdtstore, 0, PAGE_SIZE);

        /* Flat segments */
        gdt_set_memseg(GDT_ADDR_MEM(gdtstore, GDT_KCODE_SEL), 0,
            0xfffff, SDT_MEMERA, SEL_KPL, 1, true);
        gdt_set_memseg(GDT_ADDR_MEM(gdtstore, GDT_KDATA_SEL), 0,
            0xfffff, SDT_MEMRWA, SEL_KPL, 1, true);
        gdt_set_memseg(GDT_ADDR_MEM(gdtstore, GDT_UCODE32_SEL), 0,
            0xfffff, SDT_MEMERA, SEL_UPL, 1, false);
        gdt_set_memseg(GDT_ADDR_MEM(gdtstore, GDT_UCODE_SEL), 0,
            0xfffff, SDT_MEMERA, SEL_UPL, 1, true);
        gdt_set_memseg(GDT_ADDR_MEM(gdtstore, GDT_UDATA_SEL), 0,
            0xfffff, SDT_MEMRWA, SEL_UPL, 1, true);
}

void cpu_load_gdt( void )
{
        struct region_descriptor region;
        setregion(&region, &gdtstore, PAGE_SIZE - 1);
        lgdt(&region);
}

/* -------------------------------------------------------------------------- */

struct {
        bool_t busy[256];
} idt_bitmap __in_kdata;

int idt_slot_alloc( void )
{
        size_t i;

        for (i = 0; i < 256; i++) {
                if (!idt_bitmap.busy[i])
                        break;
        }
        if (i == 256) {
                return -1;
        }

        idt_bitmap.busy[i] = true;
        return (int)i;
}

void idt_slot_free(int slot)
{
        idt_bitmap.busy[slot] = false;
}

static void idt_set_seg(struct idt_seg *seg, void *func, int ist, int type,
    int dpl, int sel)
{
        seg->gd_looffset = (uint64_t)func & 0xffff;
        seg->gd_selector = sel;
        seg->gd_ist = ist;
        seg->gd_type = type;
        seg->gd_dpl = dpl;
        seg->gd_p = 1;
        seg->gd_hioffset = (uint64_t)func >> 16;
        seg->gd_zero = 0;
        seg->gd_xx1 = 0;
        seg->gd_xx2 = 0;
        seg->gd_xx3 = 0;
}

static void idt_create( void )
{
        extern uint64_t x86_traps[], x86_intrs[], x86_rsvd;
        struct idt_seg *idt;
        size_t i;
        int ist;

        memset(&idt_bitmap, 0, sizeof(idt_bitmap));
        idt = (struct idt_seg *)&idtstore;

        /* First, put a dead entry */
        for (i = 0; i < NIDT; i++) {
                idt_set_seg(&idt[i], (void *)&x86_rsvd, 0,
                    SDT_SYS386IGT, SEL_KPL, GDT_FIXED_SEL(GDT_KCODE_SEL, SEL_KPL));
        }

        /* General exceptions */
        for (i = CPUVEC_MIN; i < CPUVEC_MAX; i++) {
                if (i == 2) { /* NMI */
                        ist = 3;
                } else if (i == 8) { /* Double Fault */
                        ist = 2;
                } else {
                        ist = 0;
                }
                idt_set_seg(&idt[i], (void *)x86_traps[i - CPUVEC_MIN], ist,
                    SDT_SYS386IGT, SEL_KPL, GDT_FIXED_SEL(GDT_KCODE_SEL, SEL_KPL));
                idt_bitmap.busy[i] = true;
        }

        /* Dynamically configured interrupts */
        for (i = DYNVEC_MIN; i < DYNVEC_MAX; i++) {
                idt_set_seg(&idt[i], (void *)x86_intrs[i - DYNVEC_MIN], 0,
                    SDT_SYS386IGT, SEL_KPL, GDT_FIXED_SEL(GDT_KCODE_SEL, SEL_KPL));
                idt_bitmap.busy[i] = true;
        }
}

void cpu_load_idt( void )
{
        struct region_descriptor region;
        setregion(&region, &idtstore, PAGE_SIZE - 1);
        lidt(&region);
}

/* -------------------------------------------------------------------------- */

int tss_alloc(struct tss *tss, size_t lid)
{
        int slot;

        slot = GDT_CPUTSS_SEL + lid;

        gdt_set_sysseg(GDT_ADDR_SYS(gdtstore, slot), tss,
            sizeof(*tss) - 1, SDT_SYS386TSS, SEL_KPL, 0);

        return GDT_DYNAM_SEL(slot, SEL_KPL);
}

void cpu_create_tss(size_t lid)
{
        percpu_archdata_t *data = &cpudata[lid];
        struct tss *tss = &data->tss;
        int sel;

        /* Create the tss */
        memset(tss, 0, sizeof(*tss));

        tss->tss_rsp0 = ((uint64_t)&data->boot_stack + STKSIZE) & ~0xF;
        tss->tss_ist[0] = ((uint64_t)&data->intr_stack + STKSIZE) & ~0xF;
        tss->tss_ist[1] = ((uint64_t)&data->dbfl_stack + STKSIZE) & ~0xF;
        tss->tss_ist[2] = ((uint64_t)&data->nmfl_stack + STKSIZE) & ~0xF;
        tss->tss_iobase = IOMAP_INVALOFF << 16;
        sel = tss_alloc(tss, lid);

        /* Load it */
        ltr(sel);
}

/* -------------------------------------------------------------------------- */

void cpu_tls_init(size_t lid)
{
        extern uint64_t x86_syscall;
        extern uint64_t x86_syscall32;

        percpu_archdata_t *data = &cpudata[lid];
        tls_t *cputls = &data->tls;

        memset(cputls, 0, sizeof(tls_t));

        cputls->tls_self = cputls;
        cputls->tls_gid = hal_lapic_gid();
        cputls->tls_lid = lid;
        cputls->tls_rsp0 = (uint64_t)&data->tss.tss_rsp0;
        cputls->tls_intr = INTRS_DISABLED;

        /* syscall */
        wrmsr(MSR_STAR,
            ((uint64_t)GDT_FIXED_SEL(GDT_KCODE_SEL, SEL_KPL) << 32) |
            ((uint64_t)GDT_FIXED_SEL(GDT_UCODE32_SEL, SEL_UPL) << 48));
        wrmsr(MSR_LSTAR, (uint64_t)&x86_syscall);
        wrmsr(MSR_CSTAR, (uint64_t)&x86_syscall32);
        wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);

        /* TLS */
        wrmsr(MSR_FSBASE, 0);
        wrmsr(MSR_GSBASE, (uint64_t)cputls);
        wrmsr(MSR_KERNELGSBASE, 0);
}

/* -------------------------------------------------------------------------- */

uint64_t cpu_features[4] __in_kdata;

void cpu_identify( void )
{
        /*
         * desc[0] = eax
         * desc[1] = ebx
         * desc[2] = ecx
         * desc[3] = edx
         */
        uint32_t desc[4];
        char vendor[13];
        size_t lvl;

        /*
         * Get information from the standard cpuid leafs
         */
        cpuid(0, 0, (uint32_t *)&desc);

        lvl = (uint64_t)desc[0];
        x86_printf("-> cpuid standard level: %z\n", lvl);

        memcpy(vendor + 0, &desc[1], sizeof(uint32_t));
        memcpy(vendor + 8, &desc[2], sizeof(uint32_t));
        memcpy(vendor + 4, &desc[3], sizeof(uint32_t));
        vendor[12] = '\0';
        x86_printf("-> CPU vendor: '%s'\n", vendor);

        if (lvl >= 1) {
                cpuid(1, 0, (uint32_t *)&desc);
                cpu_features[0] = desc[3];
                cpu_features[1] = desc[2];
        }

        /*
         * Get information from the extended cpuid leafs
         */
        cpuid(0x80000000, 0, desc);

        lvl = (uint64_t)desc[0];
        x86_printf("-> cpuid extended level: %Z\n", lvl);
}

/* -------------------------------------------------------------------------- */

void cpu_attach(size_t lid)
{
        /* Per-cluster structures */
        cpu_load_gdt();
        cpu_load_idt();

        /* Per-cpu structures */
        cpu_create_tss(lid);

        if (cpu_features[0] & CPUID_PSE) {
                lcr4(rcr4() | CR4_PSE);
                tlbflushg();
        } else {
                /*
                 * amd64 supports PSE by default, if it's not here we have a
                 * problem
                 */
                x86_panic("PSE not supported");
        }
}