From 54233b448dca3c917bf83fed3529b589145bf9e3 Mon Sep 17 00:00:00 2001 From: Eyck Jentzsch Date: Sat, 15 Mar 2025 09:34:19 +0100 Subject: [PATCH] moves mmu related code into mmu unit --- CMakeLists.txt | 2 +- src/iss/arch/riscv_hart_common.h | 65 +-- src/iss/arch/riscv_hart_m_p.h | 37 +- src/iss/arch/riscv_hart_msu_vp.h | 545 +++++++---------------- src/iss/arch/riscv_hart_mu_p.h | 112 +++-- src/iss/arch/tgc5c.h | 1 - src/iss/arch/tgc_mapper.h | 15 +- src/iss/{mmio => mem}/clic.h | 18 +- src/iss/mem/memory_if.cpp | 67 +++ src/iss/{mmio => mem}/memory_if.h | 19 +- src/iss/{mmio => mem}/memory_with_htif.h | 6 +- src/iss/mem/mmu.h | 325 ++++++++++++++ src/iss/{mmio => mem}/pmp.h | 13 +- src/iss/mmio/memory_if.cpp | 26 -- src/sysc/sc_core_adapter.h | 2 +- src/vm/asmjit/vm_tgc5c.cpp | 4 +- src/vm/interp/vm_tgc5c.cpp | 18 +- src/vm/llvm/vm_tgc5c.cpp | 2 - src/vm/tcc/vm_tgc5c.cpp | 4 +- 19 files changed, 683 insertions(+), 598 deletions(-) rename src/iss/{mmio => mem}/clic.h (96%) create mode 100644 src/iss/mem/memory_if.cpp rename src/iss/{mmio => mem}/memory_if.h (84%) rename src/iss/{mmio => mem}/memory_with_htif.h (96%) create mode 100644 src/iss/mem/mmu.h rename src/iss/{mmio => mem}/pmp.h (97%) delete mode 100644 src/iss/mmio/memory_if.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b9300bb..a07684b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ add_subdirectory(softfloat) set(LIB_SOURCES src/iss/plugin/instruction_count.cpp src/iss/arch/tgc5c.cpp - src/iss/mmio/memory_if.cpp + src/iss/mem/memory_if.cpp src/vm/interp/vm_tgc5c.cpp src/vm/fp_functions.cpp src/iss/debugger/csr_names.cpp diff --git a/src/iss/arch/riscv_hart_common.h b/src/iss/arch/riscv_hart_common.h index 74776c5..a4035d2 100644 --- a/src/iss/arch/riscv_hart_common.h +++ b/src/iss/arch/riscv_hart_common.h @@ -37,7 +37,6 @@ #include "iss/arch/traits.h" #include "iss/log_categories.h" -#include "iss/mmio/memory_if.h" #include "iss/vm_types.h" #include "mstatus.h" #include "util/delegate.h" @@ -54,6 +53,7 @@ #include #include #include +#include "../mem/memory_if.h" #if defined(__GNUC__) #define likely(x) ::__builtin_expect(!!(x), 1) @@ -198,23 +198,6 @@ enum riscv_csr { dscratch1 = 0x7B3 }; -enum { - PGSHIFT = 12, - PTE_PPN_SHIFT = 10, - // page table entry (PTE) fields - PTE_V = 0x001, // Valid - PTE_R = 0x002, // Read - PTE_W = 0x004, // Write - PTE_X = 0x008, // Execute - PTE_U = 0x010, // User - PTE_G = 0x020, // Global - PTE_A = 0x040, // Accessed - PTE_D = 0x080, // Dirty - PTE_SOFT = 0x300 // Reserved for Software -}; - -template inline bool PTE_TABLE(T PTE) { return (((PTE) & (PTE_V | PTE_R | PTE_W | PTE_X)) == PTE_V); } - enum { PRIV_U = 0, PRIV_S = 1, PRIV_M = 3, PRIV_D = 4 }; enum { @@ -233,21 +216,6 @@ enum { ISA_U = 1 << 20 }; -struct vm_info { - int levels; - int idxbits; - int ptesize; - uint64_t ptbase; - bool is_active() { return levels; } -}; - -struct feature_config { - uint64_t tcm_base{0x10000000}; - uint64_t tcm_size{0x8000}; - uint64_t io_address{0xf0000000}; - uint64_t io_addr_mask{0xf0000000}; -}; - class trap_load_access_fault : public trap_access { public: trap_load_access_fault(uint64_t badaddr) @@ -281,15 +249,18 @@ template struct priv_if { std::function read_csr; std::function write_csr; std::function exec_htif; + std::function raise_trap; // trap_id, cause, fault_data std::unordered_map& csr_rd_cb; std::unordered_map& csr_wr_cb; - hart_state& mstatus; + hart_state& state; + uint8_t& PRIV; + WORD_TYPE& PC; uint64_t& tohost; uint64_t& fromhost; - unsigned& mcause_max_irq; + unsigned& max_irq; }; -template struct riscv_hart_common : public BASE, public mmio::memory_elem { +template struct riscv_hart_common : public BASE, public mem::memory_elem { const std::array lvl = {{'U', 'S', 'H', 'M'}}; const std::array trap_str = {{"" "Instruction address misaligned", // 0 @@ -718,12 +689,19 @@ template struct riscv_hart_co return priv_if{.read_csr = [this](unsigned addr, reg_t& val) -> iss::status { return read_csr(addr, val); }, .write_csr = [this](unsigned addr, reg_t val) -> iss::status { return write_csr(addr, val); }, .exec_htif = [this](uint8_t const* data) -> iss::status { return execute_htif(data); }, + .raise_trap = [this](uint16_t trap_id, uint16_t cause, reg_t fault_data){ + this->reg.trap_state = 0x80ULL << 24 | (cause << 16) | trap_id; + this->fault_data = fault_data; + + }, .csr_rd_cb{this->csr_rd_cb}, .csr_wr_cb{csr_wr_cb}, - .mstatus{this->state}, + .state{this->state}, + .PRIV{this->reg.PRIV}, + .PC{this->reg.PC}, .tohost{this->tohost}, .fromhost{this->fromhost}, - .mcause_max_irq{mcause_max_irq}}; + .max_irq{mcause_max_irq}}; } iss::status execute_htif(uint8_t const* data) { @@ -763,14 +741,14 @@ template struct riscv_hart_co } } - mmio::memory_hierarchy memories; + mem::memory_hierarchy memories; - virtual mmio::memory_if get_mem_if() override { + virtual mem::memory_if get_mem_if() override { assert(false || "This function should nevver be called"); - return mmio::memory_if{}; + return mem::memory_if{}; } - virtual void set_next(mmio::memory_if mem_if) { memory = mem_if; }; + virtual void set_next(mem::memory_if mem_if) { memory = mem_if; }; void set_irq_num(unsigned i) { mcause_max_irq = 1 << util::ilog2(i); } @@ -789,7 +767,7 @@ protected: } } - mmio::memory_if memory; + mem::memory_if memory; struct riscv_instrumentation_if : public iss::instrumentation_if { riscv_instrumentation_if(riscv_hart_common& arch) @@ -846,7 +824,6 @@ protected: int64_t cycle_offset{0}; int64_t instret_offset{0}; semihosting_cb_t semihosting_cb; - std::array vm; unsigned mcause_max_irq{16U}; }; diff --git a/src/iss/arch/riscv_hart_m_p.h b/src/iss/arch/riscv_hart_m_p.h index 6980c2e..96f9e36 100644 --- a/src/iss/arch/riscv_hart_m_p.h +++ b/src/iss/arch/riscv_hart_m_p.h @@ -48,7 +48,7 @@ #include #include #include -#include +#include "../mem/memory_with_htif.h" #include namespace iss { @@ -111,7 +111,7 @@ public: return 0b100010001000; // only machine mode is supported } - riscv_hart_m_p(feature_config cfg = feature_config{}); + riscv_hart_m_p(); virtual ~riscv_hart_m_p() = default; @@ -128,15 +128,12 @@ public: void set_csr(unsigned addr, reg_t val) { this->csr[addr & this->csr.page_addr_mask] = val; } - void set_num_of_irq(unsigned i) { this->mcause_max_irq = 1 << util::ilog2(i); } - protected: using mem_read_f = iss::status(phys_addr_t addr, unsigned, uint8_t* const); using mem_write_f = iss::status(phys_addr_t addr, unsigned, uint8_t const* const); hart_state state; - std::unordered_map ptw; std::unordered_map atomic_reservation; iss::status read_status(unsigned addr, reg_t& val); @@ -154,18 +151,20 @@ protected: iss::status write_dpc(unsigned addr, reg_t val); void check_interrupt(); - feature_config cfg; - mmio::memory_with_htif default_mem; + mem::memory_with_htif default_mem; }; template -riscv_hart_m_p::riscv_hart_m_p(feature_config cfg) -: cfg(cfg) -, default_mem(base::get_priv_if()) { - const std::array rwaddrs{{mepc, mtvec, mscratch, mtval}}; +riscv_hart_m_p::riscv_hart_m_p() +: default_mem(base::get_priv_if()) { + const std::array rwaddrs{{ + mepc, + mtvec, + mscratch, + mtval + }}; for(auto addr : rwaddrs) { this->csr_rd_cb[addr] = MK_CSR_RD_CB(read_plain); - // MK_CSR_RD_CB(read_plain(a,r);}; this->csr_wr_cb[addr] = MK_CSR_WR_CB(write_plain); } this->csr_rd_cb[mstatus] = MK_CSR_RD_CB(read_status); @@ -195,7 +194,7 @@ riscv_hart_m_p::riscv_hart_m_p(feature_config cfg) } this->rd_func = util::delegate::from(this); this->wr_func = util::delegate::from(this); - this->memories.prepend(*this); + this->memories.root(*this); this->memories.append(default_mem); } @@ -300,7 +299,7 @@ iss::status riscv_hart_m_p::write(const address_type type, c try { switch(space) { case traits::MEM: { - if(unlikely((access && iss::access_type::FETCH) && (addr & 0x1) == 1)) { + if(unlikely(is_fetch(access) && (addr & 0x1) == 1)) { this->fault_data = addr; if(access && iss::access_type::DEBUG) throw trap_access(0, addr); @@ -403,15 +402,13 @@ template inline void riscv_har } template void riscv_hart_m_p::check_interrupt() { - // TODO: Implement CLIC functionality - // auto ideleg = csr[mideleg]; // Multiple simultaneous interrupts and traps at the same privilege level are // handled in the following decreasing priority order: // external interrupts, software interrupts, timer interrupts, then finally // any synchronous traps. auto ena_irq = this->csr[mip] & this->csr[mie]; - bool mstatus_mie = this->state.mstatus.MIE; + bool mstatus_mie = state.mstatus.MIE; auto m_enabled = this->reg.PRIV < PRIV_M || mstatus_mie; auto enabled_interrupts = m_enabled ? ena_irq : 0; @@ -525,9 +522,6 @@ uint64_t riscv_hart_m_p::enter_trap(uint64_t flags, uint64_t if((xtvec & 0x1) == 1 && trap_id != 0) this->reg.NEXT_PC += 4 * cause; } - // reset trap state - this->reg.PRIV = new_priv; - this->reg.trap_state = 0; std::array buffer; #if defined(_MSC_VER) sprintf(buffer.data(), "0x%016llx", addr); @@ -538,6 +532,9 @@ uint64_t riscv_hart_m_p::enter_trap(uint64_t flags, uint64_t NSCLOG(INFO, LOGCAT) << (trap_id ? "Interrupt" : "Trap") << " with cause '" << (trap_id ? this->irq_str[cause] : this->trap_str[cause]) << "' (" << cause << ")" << " at address " << buffer.data() << " occurred"; + // reset trap state + this->reg.PRIV = new_priv; + this->reg.trap_state = 0; return this->reg.NEXT_PC; } diff --git a/src/iss/arch/riscv_hart_msu_vp.h b/src/iss/arch/riscv_hart_msu_vp.h index f975377..23877d7 100644 --- a/src/iss/arch/riscv_hart_msu_vp.h +++ b/src/iss/arch/riscv_hart_msu_vp.h @@ -41,22 +41,23 @@ #include "util/logging.h" #include #include +#include #include +#include #include #ifndef FMT_HEADER_ONLY #define FMT_HEADER_ONLY #endif -#include -#include #include #include #include -#include +#include "../mem/mmu.h" +#include "../mem/memory_with_htif.h" namespace iss { namespace arch { -template class riscv_hart_msu_vp : public riscv_hart_common { +template class riscv_hart_msu_vp : public riscv_hart_common { public: using core = BASE; using base = riscv_hart_common; @@ -65,10 +66,6 @@ public: using reg_t = typename core::reg_t; using addr_t = typename core::addr_t; - static constexpr reg_t get_misa() { - return (sizeof(reg_t) == 4 ? (1UL << 30) : (2ULL << 62)) | ISA_I | ISA_M | ISA_A | ISA_U | ISA_S | ISA_M; - } - static constexpr reg_t get_mstatus_mask(unsigned priv_lvl) { if(sizeof(reg_t) == 4) { #if __cplusplus < 201402L @@ -76,21 +73,21 @@ public: #else switch(priv_lvl) { case PRIV_U: - return 0x80000011UL; // 0b1000 0000 0000 0000 0000 0000 0001 0001 + return 0x80000000UL; // 0b1000 0000 0000 0000 0000 0000 0001 0001 case PRIV_S: - return 0x800de133UL; // 0b1000 0000 0000 1101 1110 0001 0011 0011 + return 0x800de122UL; // 0b1000 0000 0000 1101 1110 0001 0011 0011 default: - return 0x807ff9ddUL; // 0b1000 0000 0111 1111 1111 1001 1011 1011 + return 0x807ff9aaUL; // 0b1000 0000 0111 1111 1111 1001 1011 1011 } #endif } else if(sizeof(reg_t) == 8) { switch(priv_lvl) { case PRIV_U: - return 0x8000000f00000011ULL; // 0b1...0 1111 0000 0000 0111 1111 1111 1001 1011 1011 + return 0x8000000f00000000ULL; // 0b1...0 1111 0000 0000 0111 1111 1111 1001 1011 1011 case PRIV_S: - return 0x8000000f000de133ULL; // 0b1...0 0011 0000 0000 0000 1101 1110 0001 0011 0011 + return 0x8000000f000de122ULL; // 0b1...0 0011 0000 0000 0000 1101 1110 0001 0011 0011 default: - return 0x8000000f007ff9ddULL; // 0b1...0 1111 0000 0000 0111 1111 1111 1001 1011 1011 + return 0x8000000f007ff9aaULL; // 0b1...0 1111 0000 0000 0111 1111 1111 1001 1011 1011 } } else assert(false && "Unsupported XLEN value"); @@ -117,48 +114,6 @@ public: } this->state.mstatus = new_val; } - reg_t satp; - - static inline vm_info decode_vm_info(uint32_t state, uint32_t sptbr) { - if(state == PRIV_M) - return {0, 0, 0, 0}; - if(state <= PRIV_S) - switch(bit_sub<31, 1>(sptbr)) { - case 0: - return {0, 0, 0, 0}; // off - case 1: - return {2, 10, 4, bit_sub<0, 22>(sptbr) << PGSHIFT}; // SV32 - default: - abort(); - } - abort(); - return {0, 0, 0, 0}; // dummy - } - - static inline vm_info decode_vm_info(uint32_t state, uint64_t sptbr) { - if(state == PRIV_M) - return {0, 0, 0, 0}; - if(state <= PRIV_S) - switch(bit_sub<60, 4>(sptbr)) { - case 0: - return {0, 0, 0, 0}; // off - case 8: - return {3, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV39 - case 9: - return {4, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV48 - case 10: - return {5, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV57 - case 11: - return {6, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV64 - default: - abort(); - } - abort(); - return {0, 0, 0, 0}; // dummy - } - - const typename core::reg_t PGSIZE = 1 << PGSHIFT; - const typename core::reg_t PGMASK = PGSIZE - 1; constexpr reg_t get_irq_mask(size_t mode) { std::array m = {{ @@ -176,8 +131,6 @@ public: void reset(uint64_t address) override; - phys_addr_t virt2phys(const iss::addr_t& addr) override; - iss::status read(const address_type type, const access_type access, const uint32_t space, const uint64_t addr, const unsigned length, uint8_t* const data); iss::status write(const address_type type, const access_type access, const uint32_t space, const uint64_t addr, const unsigned length, @@ -190,22 +143,14 @@ public: void set_csr(unsigned addr, reg_t val) { this->csr[addr & this->csr.page_addr_mask] = val; } - void set_irq_num(unsigned i) { this->mcause_max_irq = 1 << util::ilog2(i); } - protected: - virtual iss::status read_mem(phys_addr_t addr, unsigned length, uint8_t* const data); - virtual iss::status write_mem(phys_addr_t addr, unsigned length, const uint8_t* const data); + using mem_read_f = iss::status(phys_addr_t addr, unsigned, uint8_t* const); + using mem_write_f = iss::status(phys_addr_t addr, unsigned, uint8_t const* const); hart_state state; - using mem_type = util::sparse_array; - mem_type mem; - void update_vm_info(); - std::unordered_map ptw; std::unordered_map atomic_reservation; - std::vector tcm; - iss::status read_status(unsigned addr, reg_t& val); iss::status write_status(unsigned addr, reg_t val); iss::status write_cause(unsigned addr, reg_t val); @@ -214,29 +159,37 @@ protected: iss::status read_ip(unsigned addr, reg_t& val); iss::status write_ideleg(unsigned addr, reg_t val); iss::status write_edeleg(unsigned addr, reg_t val); - iss::status read_satp(unsigned addr, reg_t& val); - iss::status write_satp(unsigned addr, reg_t val); - - virtual iss::status read_custom_csr_reg(unsigned addr, reg_t& val) { return iss::status::Err; }; - virtual iss::status write_custom_csr_reg(unsigned addr, reg_t val) { return iss::status::Err; }; - - void register_custom_csr_rd(unsigned addr) { this->csr_rd_cb[addr] = MK_CSR_RD_CB(read_custom_csr_reg); } - void register_custom_csr_wr(unsigned addr) { this->csr_wr_cb[addr] = MK_CSR_WR_CB(write_custom_csr_reg); } - - reg_t mhartid_reg{0x0}; void check_interrupt(); + mem::mmu mmu; + mem::memory_with_htif default_mem; }; -template -riscv_hart_msu_vp::riscv_hart_msu_vp() -: state() { - this->mmu = true; - this->rd_func = util::delegate::from(this); - this->rd_func = util::delegate::from(this); +template +riscv_hart_msu_vp::riscv_hart_msu_vp() +: state() +, mmu(base::get_priv_if()) +, default_mem(base::get_priv_if()) { // common regs - const std::array rwaddrs{{mepc, mtvec, mscratch, mtval, mscratch, sepc, stvec, sscratch, scause, stval, sscratch, uepc, - utvec, uscratch, ucause, utval, uscratch}}; + const std::array rwaddrs{{ + mepc, + mtvec, + mscratch, + mtval, + mscratch, + sepc, + stvec, + sscratch, + scause, + stval, + sscratch, + uepc, + utvec, + uscratch, + ucause, + utval, + uscratch + }}; for(auto addr : rwaddrs) { this->csr_rd_cb[addr] = MK_CSR_RD_CB(read_plain); this->csr_wr_cb[addr] = MK_CSR_WR_CB(write_plain); @@ -272,17 +225,32 @@ riscv_hart_msu_vp::riscv_hart_msu_vp() this->csr_wr_cb[mvendorid] = MK_CSR_WR_CB(write_null); this->csr_wr_cb[marchid] = MK_CSR_WR_CB(write_null); this->csr_wr_cb[mimpid] = MK_CSR_WR_CB(write_null); - this->csr_rd_cb[satp] = MK_CSR_RD_CB(read_satp); - this->csr_wr_cb[satp] = MK_CSR_WR_CB(write_satp); + this->rd_func = util::delegate::from(this); + this->wr_func = util::delegate::from(this); + if(FEAT & FEAT_DEBUG) { + this->csr_wr_cb[dscratch0] = MK_CSR_WR_CB(write_dscratch); + this->csr_rd_cb[dscratch0] = MK_CSR_RD_CB(read_debug); + this->csr_wr_cb[dscratch1] = MK_CSR_WR_CB(write_dscratch); + this->csr_rd_cb[dscratch1] = MK_CSR_RD_CB(read_debug); + this->csr_wr_cb[dpc] = MK_CSR_WR_CB(write_dpc); + this->csr_rd_cb[dpc] = MK_CSR_RD_CB(read_dpc); + this->csr_wr_cb[dcsr] = MK_CSR_WR_CB(write_dcsr); + this->csr_rd_cb[dcsr] = MK_CSR_RD_CB(read_debug); + } + this->rd_func = util::delegate::from(this); + this->wr_func = util::delegate::from(this); + this->set_next(mmu.get_mem_if()); + this->memories.root(mmu); + this->memories.append(default_mem); } -template -iss::status riscv_hart_msu_vp::read(const address_type type, const access_type access, const uint32_t space, const uint64_t addr, - const unsigned length, uint8_t* const data) { +template +iss::status riscv_hart_msu_vp::read(const address_type type, const access_type access, const uint32_t space, + const uint64_t addr, const unsigned length, uint8_t* const data) { #ifndef NDEBUG if(access && iss::access_type::DEBUG) { CPPLOG(TRACEALL) << "debug read of " << length << " bytes @addr 0x" << std::hex << addr; - } else if(access && iss::access_type::FETCH) { + } else if(is_fetch(access)) { CPPLOG(TRACEALL) << "fetch of " << length << " bytes @addr 0x" << std::hex << addr; } else { CPPLOG(TRACE) << "read of " << length << " bytes @addr 0x" << std::hex << addr; @@ -294,31 +262,20 @@ iss::status riscv_hart_msu_vp::read(const address_type type, const access_ auto alignment = is_fetch(access) ? (this->has_compressed() ? 2 : 4) : std::min(length, sizeof(reg_t)); if(unlikely(is_fetch(access) && (addr & (alignment - 1)))) { this->fault_data = addr; - if(access && iss::access_type::DEBUG) + if(is_debug(access)) throw trap_access(0, addr); - this->reg.trap_state = (1 << 31); // issue trap 0 + this->reg.trap_state = (1UL<< 31); // issue trap 0 return iss::Err; } try { if(!is_debug(access) && (addr & (alignment - 1))) { - this->reg.trap_state = 1 << 31 | 4 << 16; + this->reg.trap_state = (1UL << 31) | 4 << 16; this->fault_data = addr; return iss::Err; } - if(unlikely((addr & ~PGMASK) != ((addr + length - 1) & ~PGMASK))) { // we may cross a page boundary - vm_info vm = decode_vm_info(this->reg.PRIV, satp); - if(vm.levels != 0) { // VM is active - auto split_addr = (addr + length) & ~PGMASK; - auto len1 = split_addr - addr; - auto res = read(type, access, space, addr, len1, data); - if(res == iss::Ok) - res = read(type, access, space, split_addr, length - len1, data + len1); - return res; - } - } - auto res = read_mem(virt2phys(iss::addr_t{access, type, space, addr}), length, data); + auto res = this->memory.rd_mem(access, addr, length, data); if(unlikely(res != iss::Ok && (access & access_type::DEBUG) == 0)) { - this->reg.trap_state = (1 << 31) | (5 << 16); // issue trap 5 (load access fault + this->reg.trap_state = (1UL << 31) | (5 << 16); // issue trap 5 (load access fault this->fault_data = addr; } return res; @@ -336,8 +293,6 @@ iss::status riscv_hart_msu_vp::read(const address_type type, const access_ return this->read_csr(addr, *reinterpret_cast(data)); } break; case traits::FENCE: { - if((addr + length) > mem.size()) - return iss::Err; switch(addr) { case 2: // SFENCE:VMA lower case 3: { // SFENCE:VMA upper @@ -372,8 +327,8 @@ iss::status riscv_hart_msu_vp::read(const address_type type, const access_ } } -template -iss::status riscv_hart_msu_vp::write(const address_type type, const access_type access, const uint32_t space, const uint64_t addr, +template +iss::status riscv_hart_msu_vp::write(const address_type type, const access_type access, const uint32_t space, const uint64_t addr, const unsigned length, const uint8_t* const data) { #ifndef NDEBUG const char* prefix = (access && iss::access_type::DEBUG) ? "debug " : ""; @@ -401,29 +356,22 @@ iss::status riscv_hart_msu_vp::write(const address_type type, const access try { switch(space) { case traits::MEM: { - if(unlikely((access && iss::access_type::FETCH) && (addr & 0x1) == 1)) { + if(unlikely(is_fetch(access) && (addr & 0x1) == 1)) { this->fault_data = addr; if(access && iss::access_type::DEBUG) throw trap_access(0, addr); - this->reg.trap_state = (1 << 31); // issue trap 0 + this->reg.trap_state = (1UL << 31); // issue trap 0 return iss::Err; } - phys_addr_t paddr = virt2phys(iss::addr_t{access, type, space, addr}); try { - // TODO: There is no check for alignment - if(unlikely((addr & ~PGMASK) != ((addr + length - 1) & ~PGMASK))) { // we may cross a page boundary - vm_info vm = decode_vm_info(this->reg.PRIV, satp); - if(vm.levels != 0) { // VM is active - auto split_addr = (addr + length) & ~PGMASK; - auto len1 = split_addr - addr; - auto res = write(type, access, space, addr, len1, data); - if(res == iss::Ok) - res = write(type, access, space, split_addr, length - len1, data + len1); - return res; - } + auto alignment = std::min(length, sizeof(reg_t)); + if(length > 1 && (addr & (alignment - 1)) && !is_debug(access)) { + this->reg.trap_state = (1UL << 31) | 6 << 16; + this->fault_data = addr; + return iss::Err; } - auto res = write_mem(paddr, length, data); - if(unlikely(res != iss::Ok && (access & access_type::DEBUG) == 0)) { + auto res = this->memory.wr_mem(access, addr, length, data); + if(unlikely(res != iss::Ok && !is_debug(access))) { this->reg.trap_state = (1UL << 31) | (7UL << 16); // issue trap 7 (Store/AMO access fault) this->fault_data = addr; } @@ -440,12 +388,9 @@ iss::status riscv_hart_msu_vp::write(const address_type type, const access return this->write_csr(addr, *reinterpret_cast(data)); } break; case traits::FENCE: { - if((addr + length) > mem.size()) - return iss::Err; switch(addr) { case 2: case 3: { - ptw.clear(); auto tvm = state.mstatus.TVM; if(this->reg.PRIV == PRIV_S & tvm != 0) { this->reg.trap_state = (1 << 31) | (2 << 16); @@ -472,26 +417,25 @@ iss::status riscv_hart_msu_vp::write(const address_type type, const access } } -template iss::status riscv_hart_msu_vp::read_status(unsigned addr, reg_t& val) { +template iss::status riscv_hart_msu_vp::read_status(unsigned addr, reg_t& val) { auto req_priv_lvl = (addr >> 8) & 0x3; val = state.mstatus & get_mstatus_mask(req_priv_lvl); return iss::Ok; } -template iss::status riscv_hart_msu_vp::write_status(unsigned addr, reg_t val) { +template iss::status riscv_hart_msu_vp::write_status(unsigned addr, reg_t val) { auto req_priv_lvl = (addr >> 8) & 0x3; write_mstatus(val, req_priv_lvl); check_interrupt(); - update_vm_info(); return iss::Ok; } -template iss::status riscv_hart_msu_vp::write_cause(unsigned addr, reg_t val) { +template iss::status riscv_hart_msu_vp::write_cause(unsigned addr, reg_t val) { this->csr[addr] = val & ((1UL << (traits::XLEN - 1)) | 0xf); // TODO: make exception code size configurable return iss::Ok; } -template iss::status riscv_hart_msu_vp::read_ie(unsigned addr, reg_t& val) { +template iss::status riscv_hart_msu_vp::read_ie(unsigned addr, reg_t& val) { val = this->csr[mie]; if(addr < mie) val &= this->csr[mideleg]; @@ -500,7 +444,7 @@ template iss::status riscv_hart_msu_vp::read_ie(unsigned a return iss::Ok; } -template iss::status riscv_hart_msu_vp::write_ie(unsigned addr, reg_t val) { +template iss::status riscv_hart_msu_vp::write_ie(unsigned addr, reg_t val) { auto req_priv_lvl = (addr >> 8) & 0x3; auto mask = get_irq_mask(req_priv_lvl); this->csr[mie] = (this->csr[mie] & ~mask) | (val & mask); @@ -508,7 +452,7 @@ template iss::status riscv_hart_msu_vp::write_ie(unsigned return iss::Ok; } -template iss::status riscv_hart_msu_vp::read_ip(unsigned addr, reg_t& val) { +template iss::status riscv_hart_msu_vp::read_ip(unsigned addr, reg_t& val) { val = this->csr[mip]; if(addr < mip) val &= this->csr[mideleg]; @@ -517,111 +461,12 @@ template iss::status riscv_hart_msu_vp::read_ip(unsigned a return iss::Ok; } -template iss::status riscv_hart_msu_vp::read_satp(unsigned addr, reg_t& val) { - reg_t tvm = state.mstatus.TVM; - if(this->reg.PRIV == PRIV_S & tvm != 0) { - this->reg.trap_state = (1 << 31) | (2 << 16); - this->fault_data = this->reg.PC; - return iss::Err; - } - val = satp; - return iss::Ok; -} - -template iss::status riscv_hart_msu_vp::write_satp(unsigned addr, reg_t val) { - reg_t tvm = state.mstatus.TVM; - if(this->reg.PRIV == PRIV_S & tvm != 0) { - this->reg.trap_state = (1 << 31) | (2 << 16); - this->fault_data = this->reg.PC; - return iss::Err; - } - satp = val; - update_vm_info(); - return iss::Ok; -} -template iss::status riscv_hart_msu_vp::read_mem(phys_addr_t paddr, unsigned length, uint8_t* const data) { - switch(paddr.val) { - default: { - for(auto offs = 0U; offs < length; ++offs) { - *(data + offs) = mem[(paddr.val + offs) % mem.size()]; - } - } - } - return iss::Ok; -} - -template iss::status riscv_hart_msu_vp::write_mem(phys_addr_t paddr, unsigned length, const uint8_t* const data) { - mem_type::page_type& p = mem(paddr.val / mem.page_size); - std::copy(data, data + length, p.data() + (paddr.val & mem.page_addr_mask)); - // tohost handling in case of riscv-test - // according to https://github.com/riscv-software-src/riscv-isa-sim/issues/364#issuecomment-607657754: - if(paddr.access && iss::access_type::FUNC) { - if(paddr.val == this->tohost) { - reg_t cur_data = *reinterpret_cast(data); - // Extract Device (bits 63:56) - uint8_t device = traits::XLEN == 32 - ? *reinterpret_cast(p.data() + ((this->tohost + 4) & mem.page_addr_mask)) >> 24 - : (cur_data >> 56) & 0xFF; - // Extract Command (bits 55:48) - uint8_t command = traits::XLEN == 32 - ? *reinterpret_cast(p.data() + ((this->tohost + 4) & mem.page_addr_mask)) >> 16 - : (cur_data >> 48) & 0xFF; - // Extract payload (bits 47:0) - uint64_t payload_addr = cur_data & 0xFFFFFFFFFFFFULL; - if(payload_addr & 1) { - CPPLOG(FATAL) << "tohost value is 0x" << std::hex << payload_addr << std::dec << " (" << payload_addr - << "), stopping simulation"; - this->reg.trap_state = std::numeric_limits::max(); - this->interrupt_sim = payload_addr; - return iss::Ok; - } else if(device == 0 && command == 0) { - std::array loaded_payload; - if(read(address_type::PHYSICAL, access_type::DEBUG_READ, traits::MEM, payload_addr, 8 * sizeof(uint64_t), - reinterpret_cast(loaded_payload.data())) == iss::Err) - CPPLOG(ERR) << "Syscall read went wrong"; - uint64_t syscall_num = loaded_payload.at(0); - if(syscall_num == 64) { // SYS_WRITE - return this->execute_sys_write(this, loaded_payload, traits::MEM); - } else { - CPPLOG(ERR) << "tohost syscall with number 0x" << std::hex << syscall_num << std::dec << " (" << syscall_num - << ") not implemented"; - this->reg.trap_state = std::numeric_limits::max(); - this->interrupt_sim = payload_addr; - return iss::Ok; - } - } else { - CPPLOG(ERR) << "tohost functionality not implemented for device " << device << " and command " << command; - this->reg.trap_state = std::numeric_limits::max(); - this->interrupt_sim = payload_addr; - return iss::Ok; - } - } - if((traits::XLEN == 32 && paddr.val == this->fromhost + 4) || (traits::XLEN == 64 && paddr.val == this->fromhost)) { - uint64_t fhostvar = *reinterpret_cast(p.data() + (this->fromhost & mem.page_addr_mask)); - *reinterpret_cast(p.data() + (this->tohost & mem.page_addr_mask)) = fhostvar; - } - } - return iss::Ok; -} - -template inline void riscv_hart_msu_vp::reset(uint64_t address) { +template inline void riscv_hart_msu_vp::reset(uint64_t address) { BASE::reset(address); state.mstatus = hart_state::mstatus_reset_val; - update_vm_info(); } -template inline void riscv_hart_msu_vp::update_vm_info() { - this->vm[1] = decode_vm_info(this->reg.PRIV, satp); - BASE::addr_mode[3] = BASE::addr_mode[2] = this->vm[1].is_active() ? iss::address_type::VIRTUAL : iss::address_type::PHYSICAL; - if(state.mstatus.MPRV) - this->vm[0] = decode_vm_info(state.mstatus.MPP, satp); - else - this->vm[0] = this->vm[1]; - BASE::addr_mode[1] = BASE::addr_mode[0] = this->vm[0].is_active() ? iss::address_type::VIRTUAL : iss::address_type::PHYSICAL; - ptw.clear(); -} - -template void riscv_hart_msu_vp::check_interrupt() { +template void riscv_hart_msu_vp::check_interrupt() { auto status = state.mstatus; auto ip = this->csr[mip]; auto ie = this->csr[mie]; @@ -649,101 +494,8 @@ template void riscv_hart_msu_vp::check_interrupt() { } } -template typename riscv_hart_msu_vp::phys_addr_t riscv_hart_msu_vp::virt2phys(const iss::addr_t& addr) { - const auto type = addr.access & iss::access_type::FUNC; - auto it = ptw.find(addr.val >> PGSHIFT); - if(it != ptw.end()) { - const reg_t pte = it->second; - const reg_t ad = PTE_A | (type == iss::access_type::WRITE) * PTE_D; -#ifdef RISCV_ENABLE_DIRTY - // set accessed and possibly dirty bits. - *(uint32_t*)ppte |= ad; - return {addr.getAccessType(), addr.space, (pte & (~PGMASK)) | (addr.val & PGMASK)}; -#else - // take exception if access or possibly dirty bit is not set. - if((pte & ad) == ad) - return {addr.access, addr.space, (pte & (~PGMASK)) | (addr.val & PGMASK)}; - else - ptw.erase(it); // throw an exception -#endif - } else { - uint32_t mode = type != iss::access_type::FETCH && state.mstatus.MPRV ? // MPRV - state.mstatus.MPP - : this->reg.PRIV; - const vm_info& vm = this->vm[static_cast(type) / 2]; - - const bool s_mode = mode == PRIV_S; - const bool sum = state.mstatus.SUM; - const bool mxr = state.mstatus.MXR; - - // verify bits xlen-1:va_bits-1 are all equal - const int va_bits = PGSHIFT + vm.levels * vm.idxbits; - const reg_t mask = (reg_t(1) << (traits::XLEN > -(va_bits - 1))) - 1; - const reg_t masked_msbs = (addr.val >> (va_bits - 1)) & mask; - const int levels = (masked_msbs != 0 && masked_msbs != mask) ? 0 : vm.levels; - - reg_t base = vm.ptbase; - for(int i = levels - 1; i >= 0; i--) { - const int ptshift = i * vm.idxbits; - const reg_t idx = (addr.val >> (PGSHIFT + ptshift)) & ((1 << vm.idxbits) - 1); - - // check that physical address of PTE is legal - reg_t pte = 0; - const uint8_t res = this->read(iss::address_type::PHYSICAL, addr.access, traits::MEM, base + idx * vm.ptesize, vm.ptesize, - (uint8_t*)&pte); - if(res != 0) - throw trap_load_access_fault(addr.val); - const reg_t ppn = pte >> PTE_PPN_SHIFT; - - if(PTE_TABLE(pte)) { // next level of page table - base = ppn << PGSHIFT; - } else if((pte & PTE_U) ? s_mode && (type == iss::access_type::FETCH || !sum) : !s_mode) { - break; - } else if(!(pte & PTE_V) || (!(pte & PTE_R) && (pte & PTE_W))) { - break; - } else if(type == (type == iss::access_type::FETCH ? !(pte & PTE_X) - : type == iss::access_type::READ ? !(pte & PTE_R) && !(mxr && (pte & PTE_X)) - : !((pte & PTE_R) && (pte & PTE_W)))) { - break; - } else if((ppn & ((reg_t(1) << ptshift) - 1)) != 0) { - break; - } else { - const reg_t ad = PTE_A | ((type == iss::access_type::WRITE) * PTE_D); -#ifdef RISCV_ENABLE_DIRTY - // set accessed and possibly dirty bits. - *(uint32_t*)ppte |= ad; -#else - // take exception if access or possibly dirty bit is not set. - if((pte & ad) != ad) - break; -#endif - // for superpage mappings, make a fake leaf PTE for the TLB's benefit. - const reg_t vpn = addr.val >> PGSHIFT; - const reg_t value = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT; - const reg_t offset = addr.val & PGMASK; - ptw[vpn] = value | (pte & 0xff); - return {addr.access, addr.space, value | offset}; - } - } - } - switch(type) { - case access_type::FETCH: - this->fault_data = addr.val; - throw trap_instruction_page_fault(addr.val); - case access_type::READ: - this->fault_data = addr.val; - throw trap_load_page_fault(addr.val); - case access_type::WRITE: - this->fault_data = addr.val; - throw trap_store_page_fault(addr.val); - default: - abort(); - } -} - -template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t flags, uint64_t addr, uint64_t instr) { - auto cur_priv = this->reg.PRIV; +template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t flags, uint64_t addr, uint64_t instr) { // flags are ACTIVE[31:31], CAUSE[30:16], TRAPID[15:0] // calculate and write mcause val if(flags == std::numeric_limits::max()) @@ -751,11 +503,11 @@ template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t f auto trap_id = bit_sub<0, 16>(flags); auto cause = bit_sub<16, 15>(flags); if(trap_id == 0 && cause == 11) - cause = 0x8 + cur_priv; // adjust environment call cause + cause = 0x8 + this->reg.PRIV; // adjust environment call cause // calculate effective privilege level - auto new_priv = PRIV_M; + unsigned new_priv = PRIV_M; if(trap_id == 0) { // exception - if(cur_priv != PRIV_M && ((this->csr[medeleg] >> cause) & 0x1) != 0) + if(this->reg.PRIV != PRIV_M && ((this->csr[medeleg] >> cause) & 0x1) != 0) new_priv = (this->csr[sedeleg] >> cause) & 0x1 ? PRIV_U : PRIV_S; // store ret addr in xepc register this->csr[uepc | (new_priv << 8)] = static_cast(addr); // store actual address instruction of exception @@ -771,21 +523,23 @@ template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t f this->csr[utval | (new_priv << 8)] = static_cast(addr); break; case 2: - this->csr[utval | (new_priv << 8)] = (instr & 0x3) == 3 ? instr : instr & 0xffff; + this->csr[utval | (new_priv << 8)] = (!this->has_compressed() || (instr & 0x3) == 3) ? instr : instr & 0xffff; break; case 3: - // TODO: implement debug mode behavior - // csr[dpc] = addr; - // csr[dcsr] = (csr[dcsr] & ~0x1c3) | (1<<6) | PRIV_M; //FIXME: cause should not be 4 (stepi) - this->csr[utval | (new_priv << 8)] = addr; + if((FEAT & FEAT_DEBUG) && (this->csr[dcsr] & 0x8000)) { + this->reg.DPC = addr; + this->csr[dcsr] = (this->csr[dcsr] & ~0x1c3) | (1 << 6) | PRIV_M; // FIXME: cause should not be 4 (stepi) + new_priv = this->reg.PRIV | PRIV_D; + } else { + this->csr[utval | (new_priv << 8)] = addr; + } if(this->semihosting_cb) { // Check for semihosting call - phys_addr_t p_addr(access_type::DEBUG_READ, traits::MEM, addr - 4); std::array data; // check for SLLI_X0_X0_0X1F and SRAI_X0_X0_0X07 - this->read_mem(p_addr, 4, data.data()); - p_addr.val += 8; - this->read_mem(p_addr, 4, data.data() + 4); + this->memory.rd_mem(iss::access_type::DEBUG_READ, addr - 4, 4, data.data()); + addr += 8; + this->memory.rd_mem(iss::access_type::DEBUG_READ, addr - 4, 4, data.data() + 4); const std::array ref_data = {0x13, 0x10, 0xf0, 0x01, 0x13, 0x50, 0x70, 0x40}; if(data == ref_data) { @@ -797,7 +551,7 @@ template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t f #else sprintf(buffer.data(), "0x%016lx", addr); #endif - CLOG(INFO, disass) << "Semihosting call at address " << buffer.data() << " occurred "; + NSCLOG(INFO, LOGCAT) << "Semihosting call at address " << buffer.data() << " occurred "; this->semihosting_cb(this, &(this->reg.X10) /*a0*/, &(this->reg.X11) /*a1*/); return this->reg.NEXT_PC; @@ -814,7 +568,7 @@ template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t f } this->fault_data = 0; } else { - if(cur_priv != PRIV_M && ((this->csr[mideleg] >> cause) & 0x1) != 0) + if(this->reg.PRIV != PRIV_M && ((this->csr[mideleg] >> cause) & 0x1) != 0) new_priv = (this->csr[sideleg] >> cause) & 0x1 ? PRIV_U : PRIV_S; this->csr[uepc | (new_priv << 8)] = this->reg.NEXT_PC; // store next address if interrupt this->reg.pending_trap = 0; @@ -830,12 +584,12 @@ template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t f // store the actual privilege level in yPP and store interrupt enable flags switch(new_priv) { case PRIV_M: - state.mstatus.MPP = cur_priv; + state.mstatus.MPP = this->reg.PRIV; state.mstatus.MPIE = state.mstatus.MIE; state.mstatus.MIE = false; break; case PRIV_S: - state.mstatus.SPP = cur_priv; + state.mstatus.SPP = this->reg.PRIV; state.mstatus.SPIE = state.mstatus.SIE; state.mstatus.SIE = false; break; @@ -848,71 +602,82 @@ template uint64_t riscv_hart_msu_vp::enter_trap(uint64_t f } // get trap vector - auto ivec = this->csr[utvec | (new_priv << 8)]; + auto xtvec = this->csr[utvec | (new_priv << 8)]; // calculate addr// set NEXT_PC to trap addressess to jump to based on MODE // bits in mtvec - this->reg.NEXT_PC = ivec & ~0x3UL; - if((ivec & 0x1) == 1 && trap_id != 0) - this->reg.NEXT_PC += 4 * cause; + this->reg.NEXT_PC = xtvec & ~0x3UL; + if(trap_id != 0) { + if((xtvec & 0x3UL) == 3UL) { + reg_t data; + auto ret = read(address_type::LOGICAL, access_type::READ, 0, this->csr[mtvt], sizeof(reg_t), reinterpret_cast(&data)); + if(ret == iss::Err) + return this->reg.PC; + this->reg.NEXT_PC = data; + } else if((xtvec & 0x3UL) == 1UL) + this->reg.NEXT_PC += 4 * cause; + } std::array buffer; +#if defined(_MSC_VER) + sprintf(buffer.data(), "0x%016llx", addr); +#else sprintf(buffer.data(), "0x%016lx", addr); +#endif if((flags & 0xffffffff) != 0xffffffff) - CLOG(INFO, disass) << (trap_id ? "Interrupt" : "Trap") << " with cause '" + NSCLOG(INFO, LOGCAT) << (trap_id ? "Interrupt" : "Trap") << " with cause '" << (trap_id ? this->irq_str[cause] : this->trap_str[cause]) << "' (" << cause << ")" - << " at address " << buffer.data() << " occurred, changing privilege level from " << this->lvl[cur_priv] + << " at address " << buffer.data() << " occurred, changing privilege level from " << this->lvl[this->reg.PRIV] << " to " << this->lvl[new_priv]; // reset trap state this->reg.PRIV = new_priv; this->reg.trap_state = 0; - update_vm_info(); return this->reg.NEXT_PC; } -template uint64_t riscv_hart_msu_vp::leave_trap(uint64_t flags) { +template uint64_t riscv_hart_msu_vp::leave_trap(uint64_t flags) { auto cur_priv = this->reg.PRIV; auto inst_priv = flags & 0x3; - auto status = state.mstatus; auto tsr = state.mstatus.TSR; if(cur_priv == PRIV_S && inst_priv == PRIV_S && tsr != 0) { - this->reg.trap_state = (1 << 31) | (2 << 16); - this->fault_data = this->reg.PC; - return this->reg.PC; + this->reg.trap_state = 0x80ULL << 24 | (2 << 16); // illegal instruction + this->reg.NEXT_PC = std::numeric_limits::max(); + } else { + auto status = state.mstatus; + // pop the relevant lower-privilege interrupt enable and privilege mode stack + // clear respective yIE + switch(inst_priv) { + case PRIV_M: + this->reg.PRIV = state.mstatus.MPP; + state.mstatus.MPP = 0; // clear mpp to U mode + state.mstatus.MIE = state.mstatus.MPIE; + state.mstatus.MPIE = 1; + break; + case PRIV_S: + this->reg.PRIV = state.mstatus.SPP; + state.mstatus.SPP = 0; // clear spp to U mode + state.mstatus.SIE = state.mstatus.SPIE; + state.mstatus.SPIE = 1; + break; + case PRIV_U: + this->reg.PRIV = 0; + state.mstatus.UIE = state.mstatus.UPIE; + state.mstatus.UPIE = 1; + break; + } + // sets the pc to the value stored in the x epc register. + this->reg.NEXT_PC = this->csr[uepc | inst_priv << 8]; + NSCLOG(INFO, LOGCAT) << "Executing xRET , changing privilege level from " << this->lvl[cur_priv] << " to " + << this->lvl[this->reg.PRIV]; + check_interrupt(); } - - // pop the relevant lower-privilege interrupt enable and privilege mode stack - // clear respective yIE - switch(inst_priv) { - case PRIV_M: - this->reg.PRIV = state.mstatus.MPP; - state.mstatus.MPP = 0; // clear mpp to U mode - state.mstatus.MIE = state.mstatus.MPIE; - state.mstatus.MPIE = 1; - break; - case PRIV_S: - this->reg.PRIV = state.mstatus.SPP; - state.mstatus.SPP = 0; // clear spp to U mode - state.mstatus.SIE = state.mstatus.SPIE; - state.mstatus.SPIE = 1; - break; - case PRIV_U: - this->reg.PRIV = 0; - state.mstatus.UIE = state.mstatus.UPIE; - state.mstatus.UPIE = 1; - break; - } - // sets the pc to the value stored in the x epc register. - this->reg.NEXT_PC = this->csr[uepc | inst_priv << 8]; - CLOG(INFO, disass) << "Executing xRET , changing privilege level from " << this->lvl[cur_priv] << " to " << this->lvl[this->reg.PRIV]; - update_vm_info(); - check_interrupt(); + this->reg.trap_state = this->reg.pending_trap; return this->reg.NEXT_PC; } -template void riscv_hart_msu_vp::wait_until(uint64_t flags) { +template void riscv_hart_msu_vp::wait_until(uint64_t flags) { auto status = state.mstatus; auto tw = status.TW; - if(this->reg.PRIV == PRIV_S && tw != 0) { + if(this->reg.PRIV < PRIV_M && tw != 0) { this->reg.trap_state = (1 << 31) | (2 << 16); this->fault_data = this->reg.PC; } diff --git a/src/iss/arch/riscv_hart_mu_p.h b/src/iss/arch/riscv_hart_mu_p.h index f5e8527..6ab545c 100644 --- a/src/iss/arch/riscv_hart_mu_p.h +++ b/src/iss/arch/riscv_hart_mu_p.h @@ -41,14 +41,14 @@ #include "util/logging.h" #include #include +#include #include +#include #ifndef FMT_HEADER_ONLY #define FMT_HEADER_ONLY #endif -#include -#include #include -#include +#include "../mem/memory_with_htif.h" #include namespace iss { @@ -71,7 +71,7 @@ public: #else switch(priv_lvl) { case PRIV_U: - return 0x00000011UL; // 0b1...0 0001 0001 + return FEAT & features_e::FEAT_EXT_N? 0x00000011UL : 0UL; // 0b1...0 0001 0001 default: // +-SD // | +-TSR @@ -88,7 +88,7 @@ public: // | |||||| | | | || +-UPIE // | ||||||/|/|/| || |+-MIE // | ||||||/|/|/| || || +-UIE - return 0b00000000000000000001100010011001; + return 0b10000000001000000001100010011001; } #endif } else if(sizeof(reg_t) == 8) { @@ -97,24 +97,23 @@ public: #else switch(priv_lvl) { case PRIV_U: - return 0x00000011UL; // 0b1...0 0001 0001 + return FEAT & features_e::FEAT_EXT_N? 0x8000000000000011ULL : 0ULL; // 0b1...0 0001 0001 default: - // +-SD - // | +-TSR - // | |+-TW - // | ||+-TVM - // | |||+-MXR - // | ||||+-SUM - // | |||||+-MPRV - // | |||||| +-XS - // | |||||| | +-FS - // | |||||| | | +-MPP - // | |||||| | | | +-SPP - // | |||||| | | | |+-MPIE - // | |||||| | | | || +-UPIE - // | ||||||/|/|/| || |+-MIE - // | ||||||/|/|/| || || +-UIE - return 0b00000000000000000001100010011001; + // +-TSR + // |+-TW + // ||+-TVM + // |||+-MXR + // ||||+-SUM + // |||||+-MPRV + // |||||| +-XS + // |||||| | +-FS + // |||||| | | +-MPP + // |||||| | | | +-SPP + // |||||| | | | |+-MPIE + // |||||| | | | || +-UPIE + // ||||||/|/|/| || |+-MIE + // ||||||/|/|/| || || +-UIE + return 0b00000000001000000001100010011001 | 0x8000000000000000ULL; } #endif } else @@ -137,7 +136,7 @@ public: return m[mode]; } - riscv_hart_mu_p(feature_config cfg = feature_config{}); + riscv_hart_mu_p(); virtual ~riscv_hart_mu_p() = default; @@ -151,6 +150,7 @@ public: uint64_t enter_trap(uint64_t flags) override { return riscv_hart_mu_p::enter_trap(flags, this->fault_data, this->fault_data); } uint64_t enter_trap(uint64_t flags, uint64_t addr, uint64_t instr) override; uint64_t leave_trap(uint64_t flags) override; + void wait_until(uint64_t flags) override; void set_csr(unsigned addr, reg_t val) { this->csr[addr & this->csr.page_addr_mask] = val; } @@ -160,7 +160,6 @@ protected: hart_state state; - std::unordered_map ptw; std::unordered_map atomic_reservation; iss::status read_status(unsigned addr, reg_t& val); @@ -180,14 +179,12 @@ protected: iss::status write_edeleg(unsigned addr, reg_t val); void check_interrupt(); - feature_config cfg; - mmio::memory_with_htif default_mem; + mem::memory_with_htif default_mem; }; template -riscv_hart_mu_p::riscv_hart_mu_p(feature_config cfg) +riscv_hart_mu_p::riscv_hart_mu_p() : state() -, cfg(cfg) , default_mem(base::get_priv_if()) { const std::array rwaddrs{{ mepc, @@ -248,13 +245,13 @@ riscv_hart_mu_p::riscv_hart_mu_p(feature_config cfg) } this->rd_func = util::delegate::from(this); this->wr_func = util::delegate::from(this); - this->memories.prepend(*this); + this->memories.root(*this); this->memories.append(default_mem); } template iss::status riscv_hart_mu_p::read(const address_type type, const access_type access, const uint32_t space, - const uint64_t addr, const unsigned length, uint8_t* const data) { + const uint64_t addr,const unsigned length, uint8_t* const data) { #ifndef NDEBUG if(access && iss::access_type::DEBUG) { CPPLOG(TRACEALL) << "debug read of " << length << " bytes @addr 0x" << std::hex << addr; @@ -384,16 +381,8 @@ iss::status riscv_hart_mu_p::write(const address_type type, return iss::Err; return this->write_csr(addr, *reinterpret_cast(data)); } break; - case traits::FENCE: { - switch(addr) { - case 2: - case 3: { - ptw.clear(); - auto tvm = state.mstatus.TVM; - return iss::Ok; - } - } - } break; + case traits::FENCE: + break; case traits::RES: { atomic_reservation[addr] = data[0]; } break; @@ -505,12 +494,12 @@ template void riscv_hart_mu_p< } template -uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_t addr, uint64_t instr) { +uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_t addr, uint64_t tval) { // flags are ACTIVE[31:31], CAUSE[30:16], TRAPID[15:0] // calculate and write mcause val if(flags == std::numeric_limits::max()) flags = this->reg.trap_state; - auto trap_id = bit_sub<0, 16>(flags); + auto const trap_id = bit_sub<0, 16>(flags); auto cause = bit_sub<16, 15>(flags); if(trap_id == 0 && cause == 11) cause = 0x8 + this->reg.PRIV; // adjust environment call cause @@ -533,7 +522,7 @@ uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_ this->csr[utval | (new_priv << 8)] = static_cast(addr); break; case 2: - this->csr[utval | (new_priv << 8)] = (!this->has_compressed() || (instr & 0x3) == 3) ? instr : instr & 0xffff; + this->csr[utval | (new_priv << 8)] = (!this->has_compressed() || (tval & 0x3) == 3) ? tval : tval & 0xffff; break; case 3: if((FEAT & FEAT_DEBUG) && (this->csr[dcsr] & 0x8000)) { @@ -545,7 +534,6 @@ uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_ } if(this->semihosting_cb) { // Check for semihosting call - phys_addr_t p_addr(access_type::DEBUG_READ, traits::MEM, addr - 4); std::array data; // check for SLLI_X0_X0_0X1F and SRAI_X0_X0_0X07 this->memory.rd_mem(iss::access_type::DEBUG_READ, addr - 4, 4, data.data()); @@ -562,7 +550,7 @@ uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_ #else sprintf(buffer.data(), "0x%016lx", addr); #endif - CLOG(INFO, disass) << "Semihosting call at address " << buffer.data() << " occurred "; + NSCLOG(INFO, LOGCAT) << "Semihosting call at address " << buffer.data() << " occurred "; this->semihosting_cb(this, &(this->reg.X10) /*a0*/, &(this->reg.X11) /*a1*/); return this->reg.NEXT_PC; @@ -611,15 +599,15 @@ uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_ auto xtvec = this->csr[utvec | (new_priv << 8)]; // calculate addr// set NEXT_PC to trap addressess to jump to based on MODE // bits in mtvec - if(trap_id != 0 && (xtvec & 0x3UL) == 3UL) { - reg_t data; - auto ret = read(address_type::LOGICAL, access_type::READ, 0, this->csr[mtvt], sizeof(reg_t), reinterpret_cast(&data)); - if(ret == iss::Err) - return this->reg.PC; - this->reg.NEXT_PC = data; - } else { - this->reg.NEXT_PC = xtvec & ~0x3UL; - if((xtvec & 0x1) == 1 && trap_id != 0) + this->reg.NEXT_PC = xtvec & ~0x3UL; + if(trap_id != 0) { + if((xtvec & 0x3UL) == 3UL) { + reg_t data; + auto ret = read(address_type::LOGICAL, access_type::READ, 0, this->csr[mtvt], sizeof(reg_t), reinterpret_cast(&data)); + if(ret == iss::Err) + return this->reg.PC; + this->reg.NEXT_PC = data; + } else if((xtvec & 0x3UL) == 1UL) this->reg.NEXT_PC += 4 * cause; } std::array buffer; @@ -629,7 +617,7 @@ uint64_t riscv_hart_mu_p::enter_trap(uint64_t flags, uint64_ sprintf(buffer.data(), "0x%016lx", addr); #endif if((flags & 0xffffffff) != 0xffffffff) - CLOG(INFO, disass) << (trap_id ? "Interrupt" : "Trap") << " with cause '" + NSCLOG(INFO, LOGCAT) << (trap_id ? "Interrupt" : "Trap") << " with cause '" << (trap_id ? this->irq_str[cause] : this->trap_str[cause]) << "' (" << cause << ")" << " at address " << buffer.data() << " occurred, changing privilege level from " << this->lvl[this->reg.PRIV] << " to " << this->lvl[new_priv]; @@ -643,8 +631,7 @@ template uint64_t riscv_hart_m auto cur_priv = this->reg.PRIV; auto inst_priv = (flags & 0x3) ? 3 : 0; if(inst_priv > cur_priv) { - auto trap_val = 0x80ULL << 24 | (2 << 16); // illegal instruction - this->reg.trap_state = trap_val; + this->reg.trap_state = 0x80ULL << 24 | (2 << 16); // illegal instruction this->reg.NEXT_PC = std::numeric_limits::max(); } else { auto status = state.mstatus; @@ -665,13 +652,22 @@ template uint64_t riscv_hart_m } // sets the pc to the value stored in the x epc register. this->reg.NEXT_PC = this->csr[uepc | inst_priv << 8]; - CLOG(INFO, disass) << "Executing xRET , changing privilege level from " << this->lvl[cur_priv] << " to " + NSCLOG(INFO, LOGCAT) << "Executing xRET , changing privilege level from " << this->lvl[cur_priv] << " to " << this->lvl[this->reg.PRIV]; check_interrupt(); } + this->reg.trap_state = this->reg.pending_trap; return this->reg.NEXT_PC; } +template void riscv_hart_mu_p::wait_until(uint64_t flags) { + auto status = state.mstatus; + auto tw = status.TW; + if(this->reg.PRIV == PRIV_S && tw != 0) { + this->reg.trap_state = (1 << 31) | (2 << 16); + this->fault_data = this->reg.PC; + } +} } // namespace arch } // namespace iss diff --git a/src/iss/arch/tgc5c.h b/src/iss/arch/tgc5c.h index 23021ac..bca4293 100644 --- a/src/iss/arch/tgc5c.h +++ b/src/iss/arch/tgc5c.h @@ -244,7 +244,6 @@ struct tgc5c: public arch_if { uint32_t last_branch = 0; } reg; #pragma pack(pop) - std::array addr_mode; uint64_t interrupt_sim=0; diff --git a/src/iss/arch/tgc_mapper.h b/src/iss/arch/tgc_mapper.h index 02ebacc..3c8dd36 100644 --- a/src/iss/arch/tgc_mapper.h +++ b/src/iss/arch/tgc_mapper.h @@ -23,35 +23,30 @@ using tgc5c_xrb_nn_plat_type = iss::arch::hwl -using tgc5d_plat_type = iss::arch::riscv_hart_mu_p; +using tgc5d_plat_type = iss::arch::riscv_hart_mu_p; #endif #ifdef CORE_TGC5D_XRB_MAC #include "riscv_hart_mu_p.h" #include using tgc5d_xrb_mac_plat_type = - iss::arch::riscv_hart_mu_p; + iss::arch::riscv_hart_mu_p; #endif #ifdef CORE_TGC5D_XRB_NN #include "hwl.h" #include "riscv_hart_mu_p.h" #include using tgc5d_xrb_nn_plat_type = - iss::arch::hwl>; + iss::arch::hwl>; #endif #ifdef CORE_TGC5E #include "riscv_hart_mu_p.h" #include -using tgc5e_plat_type = iss::arch::riscv_hart_mu_p; +using tgc5e_plat_type = iss::arch::riscv_hart_mu_p; #endif #ifdef CORE_TGC5X #include "riscv_hart_mu_p.h" #include -using tgc5x_plat_type = iss::arch::riscv_hart_mu_p; +using tgc5x_plat_type = iss::arch::riscv_hart_mu_p; #endif #endif diff --git a/src/iss/mmio/clic.h b/src/iss/mem/clic.h similarity index 96% rename from src/iss/mmio/clic.h rename to src/iss/mem/clic.h index 758baa4..7074645 100644 --- a/src/iss/mmio/clic.h +++ b/src/iss/mem/clic.h @@ -1,11 +1,11 @@ #include "iss/arch/riscv_hart_common.h" #include "iss/vm_types.h" -#include "memory_if.h" #include +#include "../mem/memory_if.h" namespace iss { -namespace mmio { +namespace mem { struct clic_config { uint64_t clic_base{0xc0000000}; unsigned clic_int_ctl_bits{4}; @@ -199,12 +199,12 @@ template iss::status clic::read_cause(unsigned a switch(mode) { case 0: val |= clic_uprev_lvl << 16; - val |= hart_if.mstatus.UPIE << 27; + val |= hart_if.state.mstatus.UPIE << 27; break; default: val |= clic_mprev_lvl << 16; - val |= hart_if.mstatus.MPIE << 27; - val |= hart_if.mstatus.MPP << 28; + val |= hart_if.state.mstatus.MPIE << 27; + val |= hart_if.state.mstatus.MPP << 28; break; } } else @@ -220,12 +220,12 @@ template iss::status clic::write_cause(unsigned switch(mode) { case 0: clic_uprev_lvl = ((val >> 16) & 0xff) | (1 << (8 - cfg.clic_int_ctl_bits)) - 1; - hart_if.mstatus.UPIE = (val >> 27) & 0x1; + hart_if.state.mstatus.UPIE = (val >> 27) & 0x1; break; default: clic_mprev_lvl = ((val >> 16) & 0xff) | (1 << (8 - cfg.clic_int_ctl_bits)) - 1; - hart_if.mstatus.MPIE = (val >> 27) & 0x1; - hart_if.mstatus.MPP = (val >> 28) & 0x3; + hart_if.state.mstatus.MPIE = (val >> 27) & 0x1; + hart_if.state.mstatus.MPP = (val >> 28) & 0x3; break; } } else { @@ -248,5 +248,5 @@ template iss::status clic::write_intthresh(unsig return iss::Ok; } -} // namespace mmio +} // namespace mem } // namespace iss diff --git a/src/iss/mem/memory_if.cpp b/src/iss/mem/memory_if.cpp new file mode 100644 index 0000000..2beedf8 --- /dev/null +++ b/src/iss/mem/memory_if.cpp @@ -0,0 +1,67 @@ +#include "memory_if.h" +#include + +namespace iss { +namespace mem { +void memory_hierarchy::root(memory_elem& e) { + hierarchy.push_front(&e); + root_set=true; + update_chain(); +} +void memory_hierarchy::prepend(memory_elem& e) { + if(root_set) + hierarchy.insert(hierarchy.begin()+1, &e); + else + hierarchy.push_front(&e); + update_chain(); +} +void memory_hierarchy::append(memory_elem& e) { + hierarchy.push_back(&e); + update_chain(); +} +void memory_hierarchy::insert_before(memory_elem&) {} +void memory_hierarchy::insert_after(memory_elem&) {} +void memory_hierarchy::replace_last(memory_elem& e) { + hierarchy.pop_back(); + hierarchy.push_back(&e); +} +void memory_hierarchy::update_chain() { + bool tail = false; + for(size_t i = 0; i < hierarchy.size(); ++i) { + hierarchy[i]->register_csrs(); + if(i) + hierarchy[i - 1]->set_next(hierarchy[i]->get_mem_if()); + } +} + +void memory_hierarchy::prepend(std::unique_ptr&& p) { + prepend(*p); + owned_elems.push_back(std::move(p)); +} + +void memory_hierarchy::append(std::unique_ptr&& p) { + append(*p); + owned_elems.push_back(std::move(p)); +} + +void memory_hierarchy::insert_before(std::unique_ptr&& p) { + insert_before(*p); + owned_elems.push_back(std::move(p)); +} + +void memory_hierarchy::insert_after(std::unique_ptr&& p) { + insert_after(*p); + owned_elems.push_back(std::move(p)); +} + +void memory_hierarchy::replace_last(std::unique_ptr&& p) { + auto e = hierarchy.back(); + replace_last(*p); + auto it = std::find_if(std::begin(owned_elems), std::end(owned_elems), [e](std::unique_ptr const& p) {return p.get()==e;}); + if(it!=std::end(owned_elems)) + owned_elems.erase(it); + owned_elems.push_back(std::move(p)); +} + +} // namespace mem +} // namespace iss diff --git a/src/iss/mmio/memory_if.h b/src/iss/mem/memory_if.h similarity index 84% rename from src/iss/mmio/memory_if.h rename to src/iss/mem/memory_if.h index 70adac2..14e042c 100644 --- a/src/iss/mmio/memory_if.h +++ b/src/iss/mem/memory_if.h @@ -37,12 +37,14 @@ #include "iss/vm_types.h" #include +#include #include #include #include +#include namespace iss { -namespace mmio { +namespace mem { using rd_mem_func_sig = iss::status(iss::access_type, uint64_t, unsigned, uint8_t*); using wr_mem_func_sig = iss::status(iss::access_type, uint64_t, unsigned, uint8_t const*); @@ -53,6 +55,7 @@ struct memory_if { }; struct memory_elem { + virtual ~memory_elem() = default; virtual memory_if get_mem_if() = 0; virtual void set_next(memory_if) = 0; virtual void register_csrs() {} @@ -60,17 +63,25 @@ struct memory_elem { }; struct memory_hierarchy { + void root(memory_elem&); void prepend(memory_elem&); void append(memory_elem&); void insert_before(memory_elem&); void insert_after(memory_elem&); void replace_last(memory_elem&); + void prepend(std::unique_ptr&&); + void append(std::unique_ptr&&); + void insert_before(std::unique_ptr&&); + void insert_after(std::unique_ptr&&); + void replace_last(std::unique_ptr&&); protected: void update_chain(); - std::deque> hierarchy; + std::deque hierarchy; + std::vector> owned_elems; + bool root_set{false}; }; -} // namespace mmio +} // namespace mem } // namespace iss -#endif \ No newline at end of file +#endif diff --git a/src/iss/mmio/memory_with_htif.h b/src/iss/mem/memory_with_htif.h similarity index 96% rename from src/iss/mmio/memory_with_htif.h rename to src/iss/mem/memory_with_htif.h index 2a4d6e1..110d6e1 100644 --- a/src/iss/mmio/memory_with_htif.h +++ b/src/iss/mem/memory_with_htif.h @@ -3,12 +3,12 @@ #include "iss/arch/riscv_hart_common.h" #include "iss/vm_types.h" -#include "memory_if.h" #include #include +#include "../mem/memory_if.h" namespace iss { -namespace mmio { +namespace mem { template struct memory_with_htif : public memory_elem { using this_class = memory_with_htif; constexpr static unsigned WORD_LEN = sizeof(WORD_TYPE) * 8; @@ -51,6 +51,6 @@ protected: mem_type mem; arch::priv_if hart_if; }; -} // namespace mmio +} // namespace mem } // namespace iss #endif // _MEMORY_WITH_HTIF_ diff --git a/src/iss/mem/mmu.h b/src/iss/mem/mmu.h new file mode 100644 index 0000000..82f0a65 --- /dev/null +++ b/src/iss/mem/mmu.h @@ -0,0 +1,325 @@ + +#include "iss/arch/riscv_hart_common.h" +#include "iss/vm_types.h" +#include +#include "../mem/memory_if.h" + +namespace iss { +namespace mem { +enum { + PGSHIFT = 12, + PTE_PPN_SHIFT = 10, + // page table entry (PTE) fields + PTE_V = 0x001, // Valid + PTE_R = 0x002, // Read + PTE_W = 0x004, // Write + PTE_X = 0x008, // Execute + PTE_U = 0x010, // User + PTE_G = 0x020, // Global + PTE_A = 0x040, // Accessed + PTE_D = 0x080, // Dirty + PTE_SOFT = 0x300 // Reserved for Software +}; + + +template inline bool PTE_TABLE(T PTE) { return (((PTE) & (PTE_V | PTE_R | PTE_W | PTE_X)) == PTE_V); } + +struct vm_info { + int levels; + int idxbits; + int ptesize; + uint64_t ptbase; + bool is_active() { return levels; } +}; + +inline void read_reg_with_offset(uint32_t reg, uint8_t offs, uint8_t* const data, unsigned length) { + auto reg_ptr = reinterpret_cast(®); + switch(offs) { + default: + for(auto i = 0U; i < length; ++i) + *(data + i) = *(reg_ptr + i); + break; + case 1: + for(auto i = 0U; i < length; ++i) + *(data + i) = *(reg_ptr + 1 + i); + break; + case 2: + for(auto i = 0U; i < length; ++i) + *(data + i) = *(reg_ptr + 2 + i); + break; + case 3: + *data = *(reg_ptr + 3); + break; + } +} + +inline void write_reg_with_offset(uint32_t& reg, uint8_t offs, const uint8_t* const data, unsigned length) { + auto reg_ptr = reinterpret_cast(®); + switch(offs) { + default: + for(auto i = 0U; i < length; ++i) + *(reg_ptr + i) = *(data + i); + break; + case 1: + for(auto i = 0U; i < length; ++i) + *(reg_ptr + 1 + i) = *(data + i); + break; + case 2: + for(auto i = 0U; i < length; ++i) + *(reg_ptr + 2 + i) = *(data + i); + break; + case 3: + *(reg_ptr + 3) = *data; + break; + } +} +//TODO: update vminfo on trap enter and leave as well as mstatus write, reset +template struct mmu : public memory_elem { + using this_class = mmu; + using reg_t = WORD_TYPE; + constexpr static unsigned WORD_LEN = sizeof(WORD_TYPE) * 8; + + constexpr static reg_t PGSIZE = 1 << PGSHIFT; + constexpr static reg_t PGMASK = PGSIZE - 1; + + + + mmu(arch::priv_if hart_if) + : hart_if(hart_if) { + hart_if.csr_rd_cb[satp] = MK_CSR_RD_CB(read_satp); + hart_if.csr_wr_cb[satp] = MK_CSR_WR_CB(write_satp); + } + + virtual ~mmu() = default; + + memory_if get_mem_if() override { + return memory_if{.rd_mem{util::delegate::from(this)}, + .wr_mem{util::delegate::from(this)}}; + } + + void set_next(memory_if mem) override { down_stream_mem = mem; } + +private: + iss::status read_mem(iss::access_type access, uint64_t addr, unsigned length, uint8_t* data) { + if(unlikely((addr & ~PGMASK) != ((addr + length - 1) & ~PGMASK))) { // we may cross a page boundary + vm_info vm = decode_vm_info(hart_if.PRIV, satp); + if(vm.levels != 0) { // VM is active + auto split_addr = (addr + length) & ~PGMASK; + auto len1 = split_addr - addr; + auto res = down_stream_mem.rd_mem(access, addr, len1, data); + if(res == iss::Ok) + res = down_stream_mem.rd_mem(access, split_addr, length - len1, data + len1); + return res; + } + } + return down_stream_mem.rd_mem(access, addr, length, data); + } + + iss::status write_mem(iss::access_type access, uint64_t addr, unsigned length, uint8_t const* data) { + if(unlikely((addr & ~PGMASK) != ((addr + length - 1) & ~PGMASK))) { // we may cross a page boundary + vm_info vm = decode_vm_info(hart_if.PRIV, satp); + if(vm.levels != 0) { // VM is active + auto split_addr = (addr + length) & ~PGMASK; + auto len1 = split_addr - addr; + auto res = down_stream_mem.wr_mem(access, addr, len1, data); + if(res == iss::Ok) + res = down_stream_mem.wr_mem(access, split_addr, length - len1, data + len1); + return res; + } + } + return down_stream_mem.wr_mem(access, virt2phys(access, addr), length, data); + } + void update_vm_info(); + + iss::status read_plain(unsigned addr, reg_t& val) { + val = hart_if.csr[addr]; + return iss::Ok; + } + + iss::status write_plain(unsigned addr, reg_t const& val) { + hart_if.csr[addr] = val; + return iss::Ok; + } + + iss::status read_satp(unsigned addr, reg_t& val) { + auto tvm = bit_sub<20, 1>(hart_if.state.mstatus()); + if(hart_if.PRIV == arch::PRIV_S & tvm != 0) { + hart_if.raise_trap(2, 0, hart_if.PC); +// hart_if.reg.trap_state = (1 << 31) | (2 << 16); +// hart_if.fault_data = hart_if.reg.PC; + return iss::Err; + } + val = satp; + return iss::Ok; + } + + iss::status write_satp(unsigned addr, reg_t val) { + reg_t tvm = hart_if.state.mstatus.TVM; + if(hart_if.PRIV == arch::PRIV_S & tvm != 0) { + hart_if.raise_trap(2, 0, hart_if.PC); +// hart_if.reg.trap_state = (1 << 31) | (2 << 16); +// hart_if.fault_data = hart_if.reg.PC; + return iss::Err; + } + satp = val; + update_vm_info(); + return iss::Ok; + } + + uint64_t virt2phys(iss::access_type access, uint64_t addr); + + static inline vm_info decode_vm_info(uint32_t state, uint32_t sptbr) { + if(state == arch::PRIV_M) + return {0, 0, 0, 0}; + if(state <= arch::PRIV_S) + switch(bit_sub<31, 1>(sptbr)) { + case 0: + return {0, 0, 0, 0}; // off + case 1: + return {2, 10, 4, bit_sub<0, 22>(sptbr) << PGSHIFT}; // SV32 + default: + abort(); + } + abort(); + return {0, 0, 0, 0}; // dummy + } + + static inline vm_info decode_vm_info(uint32_t state, uint64_t sptbr) { + if(state == arch::PRIV_M) + return {0, 0, 0, 0}; + if(state <= arch::PRIV_S) + switch(bit_sub<60, 4>(sptbr)) { + case 0: + return {0, 0, 0, 0}; // off + case 8: + return {3, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV39 + case 9: + return {4, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV48 + case 10: + return {5, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV57 + case 11: + return {6, 9, 8, bit_sub<0, 44>(sptbr) << PGSHIFT}; // SV64 + default: + abort(); + } + abort(); + return {0, 0, 0, 0}; // dummy + } + +protected: + reg_t satp; + std::unordered_map ptw; + std::array vmt; + std::array addr_mode; + + arch::priv_if hart_if; + memory_if down_stream_mem; +}; + +template uint64_t mmu::virt2phys(iss::access_type access, uint64_t addr) { + const auto type = access & iss::access_type::FUNC; + auto it = ptw.find(addr >> PGSHIFT); + if(it != ptw.end()) { + const reg_t pte = it->second; + const reg_t ad = PTE_A | (type == iss::access_type::WRITE) * PTE_D; +#ifdef RISCV_ENABLE_DIRTY + // set accessed and possibly dirty bits. + *(uint32_t*)ppte |= ad; + return {addr.getAccessType(), addr.space, (pte & (~PGMASK)) | (addr.val & PGMASK)}; +#else + // take exception if access or possibly dirty bit is not set. + if((pte & ad) == ad) + return {(pte & (~PGMASK)) | (addr & PGMASK)}; + else + ptw.erase(it); // throw an exception +#endif + } else { + uint32_t mode = type != iss::access_type::FETCH && hart_if.state.mstatus.MPRV ? // MPRV + hart_if.state.mstatus.MPP + : hart_if.PRIV; + + const vm_info& vm = vmt[static_cast(type) / 2]; + + const bool s_mode = mode == arch::PRIV_S; + const bool sum = hart_if.state.mstatus.SUM; + const bool mxr = hart_if.state.mstatus.MXR; + + // verify bits xlen-1:va_bits-1 are all equal + const int va_bits = PGSHIFT + vm.levels * vm.idxbits; + const reg_t mask = (reg_t(1) << (sizeof(reg_t)*8 - (va_bits - 1))) - 1; + const reg_t masked_msbs = (addr >> (va_bits - 1)) & mask; + const int levels = (masked_msbs != 0 && masked_msbs != mask) ? 0 : vm.levels; + + reg_t base = vm.ptbase; + for(int i = levels - 1; i >= 0; i--) { + const int ptshift = i * vm.idxbits; + const reg_t idx = (addr >> (PGSHIFT + ptshift)) & ((1 << vm.idxbits) - 1); + + // check that physical address of PTE is legal + reg_t pte = 0; + const uint8_t res = down_stream_mem.rd_mem(iss::access_type::READ, base + idx * vm.ptesize, vm.ptesize, + (uint8_t*)&pte); + if(res != 0) + throw arch::trap_load_access_fault(addr); + const reg_t ppn = pte >> PTE_PPN_SHIFT; + + if(PTE_TABLE(pte)) { // next level of page table + base = ppn << PGSHIFT; + } else if((pte & PTE_U) ? s_mode && (type == iss::access_type::FETCH || !sum) : !s_mode) { + break; + } else if(!(pte & PTE_V) || (!(pte & PTE_R) && (pte & PTE_W))) { + break; + } else if(type == (type == iss::access_type::FETCH ? !(pte & PTE_X) + : type == iss::access_type::READ ? !(pte & PTE_R) && !(mxr && (pte & PTE_X)) + : !((pte & PTE_R) && (pte & PTE_W)))) { + break; + } else if((ppn & ((reg_t(1) << ptshift) - 1)) != 0) { + break; + } else { + const reg_t ad = PTE_A | ((type == iss::access_type::WRITE) * PTE_D); +#ifdef RISCV_ENABLE_DIRTY + // set accessed and possibly dirty bits. + *(uint32_t*)ppte |= ad; +#else + // take exception if access or possibly dirty bit is not set. + if((pte & ad) != ad) + break; +#endif + // for superpage mappings, make a fake leaf PTE for the TLB's benefit. + const reg_t vpn = addr >> PGSHIFT; + const reg_t value = (ppn | (vpn & ((reg_t(1) << ptshift) - 1))) << PGSHIFT; + const reg_t offset = addr & PGMASK; + ptw[vpn] = value | (pte & 0xff); + return value | offset; + } + } + } + switch(type) { + case access_type::FETCH: + hart_if.raise_trap(12, 0, addr); + throw arch::trap_instruction_page_fault(addr); + case access_type::READ: + hart_if.raise_trap(13, 0, addr); + throw arch::trap_load_page_fault(addr); + case access_type::WRITE: + hart_if.raise_trap(15, 0, addr); + throw arch::trap_store_page_fault(addr); + default: + abort(); + } +} + +template inline void mmu::update_vm_info() { + vmt[1] = decode_vm_info(hart_if.PRIV, satp); + addr_mode[3] = addr_mode[2] = vmt[1].is_active() ? iss::address_type::VIRTUAL : iss::address_type::PHYSICAL; + if(hart_if.state.mstatus.MPRV) + vmt[0] = decode_vm_info(hart_if.state.mstatus.MPP, satp); + else + vmt[0] = vmt[1]; + addr_mode[1] = addr_mode[0] = vmt[0].is_active() ? iss::address_type::VIRTUAL : iss::address_type::PHYSICAL; + ptw.clear(); +} + + +} // namespace mem +} // namespace iss diff --git a/src/iss/mmio/pmp.h b/src/iss/mem/pmp.h similarity index 97% rename from src/iss/mmio/pmp.h rename to src/iss/mem/pmp.h index 848b7ea..332cdf7 100644 --- a/src/iss/mmio/pmp.h +++ b/src/iss/mem/pmp.h @@ -1,11 +1,11 @@ #include "iss/arch/riscv_hart_common.h" #include "iss/vm_types.h" -#include "memory_if.h" #include +#include "../mem/memory_if.h" namespace iss { -namespace mmio { +namespace mem { struct clic_config { uint64_t clic_base{0xc0000000}; unsigned clic_int_ctl_bits{4}; @@ -61,9 +61,8 @@ template struct pmp : public memory_elem { using reg_t = WORD_TYPE; constexpr static unsigned WORD_LEN = sizeof(WORD_TYPE) * 8; - pmp(arch::priv_if hart_if, clic_config cfg) - : hart_if(hart_if) - , cfg(cfg) { + pmp(arch::priv_if hart_if) + : hart_if(hart_if) { for(size_t i = arch::pmpaddr0; i <= arch::pmpaddr15; ++i) { hart_if.csr_rd_cb[i] = MK_CSR_RD_CB(read_plain); hart_if.csr_wr_cb[i] = MK_CSR_WR_CB(write_plain); @@ -74,7 +73,7 @@ template struct pmp : public memory_elem { } } - ~pmp() = default; + virtual ~pmp() = default; memory_if get_mem_if() override { return memory_if{.rd_mem{util::delegate::from(this)}, @@ -208,5 +207,5 @@ template bool pmp::pmp_check(const access_type t return !any_active || hart_if.reg.PRIV == arch::PRIV_M; } -} // namespace mmio +} // namespace mem } // namespace iss diff --git a/src/iss/mmio/memory_if.cpp b/src/iss/mmio/memory_if.cpp deleted file mode 100644 index 03c0b70..0000000 --- a/src/iss/mmio/memory_if.cpp +++ /dev/null @@ -1,26 +0,0 @@ -#include "memory_if.h" - -namespace iss { -namespace mmio { -void memory_hierarchy::prepend(memory_elem& e) { - hierarchy.push_front(e); - update_chain(); -} -void memory_hierarchy::append(memory_elem& e) { - hierarchy.push_back(e); - update_chain(); -} -void memory_hierarchy::insert_before(memory_elem&) {} -void memory_hierarchy::insert_after(memory_elem&) {} -void memory_hierarchy::replace_last(memory_elem&) {} -void memory_hierarchy::update_chain() { - bool tail = false; - for(size_t i = 0; i < hierarchy.size(); ++i) { - hierarchy[i].get().register_csrs(); - if(i) - hierarchy[i - 1].get().set_next(hierarchy[i].get().get_mem_if()); - } -} - -} // namespace mmio -} // namespace iss \ No newline at end of file diff --git a/src/sysc/sc_core_adapter.h b/src/sysc/sc_core_adapter.h index df40c5a..d654800 100644 --- a/src/sysc/sc_core_adapter.h +++ b/src/sysc/sc_core_adapter.h @@ -146,10 +146,10 @@ public: void wait_until(uint64_t flags) override { SCCDEBUG(owner->hier_name()) << "Sleeping until interrupt"; + PLAT::wait_until(flags); while(this->reg.pending_trap == 0 && (this->csr[iss::arch::mip] & this->csr[iss::arch::mie]) == 0) { sc_core::wait(wfi_evt); } - PLAT::wait_until(flags); } void local_irq(short id, bool value) override { diff --git a/src/vm/asmjit/vm_tgc5c.cpp b/src/vm/asmjit/vm_tgc5c.cpp index 7505267..5c1cb72 100644 --- a/src/vm/asmjit/vm_tgc5c.cpp +++ b/src/vm/asmjit/vm_tgc5c.cpp @@ -4785,8 +4785,6 @@ continuation_e vm_impl::gen_single_inst_behavior(virt_addr_t &pc, jit_hold code_word_t instr = 0; phys_addr_t paddr(pc); auto *const data = (uint8_t *)&instr; - if(this->core.has_mmu()) - paddr = this->core.virt2phys(pc); auto res = this->core.read(paddr, 4, data); if (res != iss::Ok) return ILLEGAL_FETCH; @@ -4926,4 +4924,4 @@ volatile std::array dummy = { }; } } -// clang-format on \ No newline at end of file +// clang-format on diff --git a/src/vm/interp/vm_tgc5c.cpp b/src/vm/interp/vm_tgc5c.cpp index 8f2b515..57993c5 100644 --- a/src/vm/interp/vm_tgc5c.cpp +++ b/src/vm/interp/vm_tgc5c.cpp @@ -251,22 +251,8 @@ private: decoder instr_decoder; iss::status fetch_ins(virt_addr_t pc, uint8_t * data){ - if(this->core.has_mmu()) { - auto phys_pc = this->core.virt2phys(pc); -// if ((pc.val & upper_bits) != ((pc.val + 2) & upper_bits)) { // we may cross a page boundary -// if (this->core.read(phys_pc, 2, data) != iss::Ok) return iss::Err; -// if ((data[0] & 0x3) == 0x3) // this is a 32bit instruction -// if (this->core.read(this->core.v2p(pc + 2), 2, data + 2) != iss::Ok) -// return iss::Err; -// } else { - if (this->core.read(phys_pc, 4, data) != iss::Ok) - return iss::Err; -// } - } else { - if (this->core.read(iss::address_type::PHYSICAL, pc.access, pc.space, pc.val, 4, data) != iss::Ok) - return iss::Err; - - } + if (this->core.read(iss::address_type::PHYSICAL, pc.access, pc.space, pc.val, 4, data) != iss::Ok) + return iss::Err; return iss::Ok; } }; diff --git a/src/vm/llvm/vm_tgc5c.cpp b/src/vm/llvm/vm_tgc5c.cpp index 527a63a..ef76949 100644 --- a/src/vm/llvm/vm_tgc5c.cpp +++ b/src/vm/llvm/vm_tgc5c.cpp @@ -4944,8 +4944,6 @@ vm_impl::gen_single_inst_behavior(virt_addr_t &pc, BasicBlock *this_block) // const typename traits::addr_t upper_bits = ~traits::PGMASK; phys_addr_t paddr(pc); auto *const data = (uint8_t *)&instr; - if(this->core.has_mmu()) - paddr = this->core.virt2phys(pc); auto res = this->core.read(paddr, 4, data); if (res != iss::Ok) return std::make_tuple(ILLEGAL_FETCH, nullptr); diff --git a/src/vm/tcc/vm_tgc5c.cpp b/src/vm/tcc/vm_tgc5c.cpp index 85723e6..45669de 100644 --- a/src/vm/tcc/vm_tgc5c.cpp +++ b/src/vm/tcc/vm_tgc5c.cpp @@ -3743,8 +3743,6 @@ vm_impl::gen_single_inst_behavior(virt_addr_t &pc, tu_builder& tu) { enum {TRAP_ID=1<<16}; code_word_t instr = 0; phys_addr_t paddr(pc); - if(this->core.has_mmu()) - paddr = this->core.virt2phys(pc); auto res = this->core.read(paddr, 4, reinterpret_cast(&instr)); if (res != iss::Ok) return ILLEGAL_FETCH; @@ -3833,4 +3831,4 @@ volatile std::array dummy = { }; } } -// clang-format on \ No newline at end of file +// clang-format on