diff --git a/Bender.yml b/Bender.yml index 139ca8c321..9bea4ef86a 100644 --- a/Bender.yml +++ b/Bender.yml @@ -25,9 +25,10 @@ sources: - core/include/cv64a6_imafdcv_sv39_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv39/tlb.sv - - core/mmu_sv39/mmu.sv - - core/mmu_sv39/ptw.sv + - core/cva6_mmu/cva6_tlb.sv + - core/cva6_mmu/cva6_shared_tlb.sv + - core/cva6_mmu/cva6_mmu.sv + - core/cva6_mmu/cva6_ptw.sv - corev_apu/tb/common/mock_uart.sv - target: cv64a6_imafdc_sv39 @@ -35,9 +36,10 @@ sources: - core/include/cv64a6_imafdc_sv39_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv39/tlb.sv - - core/mmu_sv39/mmu.sv - - core/mmu_sv39/ptw.sv + - core/cva6_mmu/cva6_tlb.sv + - core/cva6_mmu/cva6_shared_tlb.sv + - core/cva6_mmu/cva6_mmu.sv + - core/cva6_mmu/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv64a6_imafdc_sv39_wb @@ -45,9 +47,10 @@ sources: - core/include/cv64a6_imafdc_sv39_wb_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv39/tlb.sv - - core/mmu_sv39/mmu.sv - - core/mmu_sv39/ptw.sv + - core/cva6_mmu/cva6_tlb.sv + - core/cva6_mmu/cva6_shared_tlb.sv + - core/cva6_mmu/cva6_mmu.sv + - core/cva6_mmu/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imac_sv0 @@ -55,9 +58,10 @@ sources: - core/include/cv32a6_imac_sv0_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv32/cva6_tlb_sv32.sv - - core/mmu_sv32/cva6_mmu_sv32.sv - - core/mmu_sv32/cva6_ptw_sv32.sv + - core/cva6_mmu/cva6_tlb.sv + - core/cva6_mmu/cva6_shared_tlb.sv + - core/cva6_mmu/cva6_mmu.sv + - core/cva6_mmu/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imac_sv32 @@ -65,9 +69,10 @@ sources: - core/include/cv32a6_imac_sv32_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv32/cva6_tlb_sv32.sv - - core/mmu_sv32/cva6_mmu_sv32.sv - - core/mmu_sv32/cva6_ptw_sv32.sv + - core/cva6_mmu/cva6_tlb.sv + - core/cva6_mmu/cva6_shared_tlb.sv + - core/cva6_mmu/cva6_mmu.sv + - core/cva6_mmu/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv - target: cv32a6_imafc_sv32 @@ -75,9 +80,10 @@ sources: - core/include/cv32a6_imafc_sv32_config_pkg.sv - core/include/riscv_pkg.sv - core/include/ariane_pkg.sv - - core/mmu_sv32/cva6_tlb_sv32.sv - - core/mmu_sv32/cva6_mmu_sv32.sv - - core/mmu_sv32/cva6_ptw_sv32.sv + - core/cva6_mmu/cva6_tlb.sv + - core/cva6_mmu/cva6_shared_tlb.sv + - core/cva6_mmu/cva6_mmu.sv + - core/cva6_mmu/cva6_ptw.sv - core/cva6_accel_first_pass_decoder_stub.sv # included via target core/include/${TARGET_CFG}_config_pkg.sv diff --git a/Flist.ariane b/Flist.ariane index a1f4976382..ae1d1d06ce 100644 --- a/Flist.ariane +++ b/Flist.ariane @@ -82,19 +82,20 @@ core/issue_stage.sv core/load_unit.sv core/load_store_unit.sv core/lsu_bypass.sv -core/mmu_sv39/mmu.sv +core/cva6_mmu/cva6_mmu.sv core/mult.sv core/multiplier.sv core/serdiv.sv core/perf_counters.sv -core/mmu_sv39/ptw.sv +core/cva6_mmu/cva6_ptw.sv core/ariane_regfile_ff.sv core/re_name.sv core/scoreboard.sv core/store_buffer.sv core/amo_buffer.sv core/store_unit.sv -core/mmu_sv39/tlb.sv +core/cva6_mmu/cva6_tlb.sv +core/cva6_mmu/cva6_shared_tlb.sv core/commit_stage.sv core/cache_subsystem/wt_dcache_ctrl.sv core/cache_subsystem/wt_dcache_mem.sv diff --git a/ariane.core b/ariane.core index 501f296c14..a1d4b27e93 100644 --- a/ariane.core +++ b/ariane.core @@ -33,20 +33,18 @@ filesets: - src/lsu_arbiter.sv - src/lsu.sv - src/miss_handler.sv - - src/mmu_sv39/mmu.sv - - src/mmu_sv32/cva6_mmu_sv32.sv + - src/cva6_mmu/cva6_mmu.sv - src/mult.sv - src/nbdcache.sv - src/pcgen_stage.sv - src/perf_counters.sv - - src/mmu_sv39/ptw.sv - - src/mmu_sv32/cva6_ptw_sv32.sv + - src/cva6_mmu/cva6_ptw.sv - src/regfile_ff.sv - src/scoreboard.sv - src/store_buffer.sv - src/store_unit.sv - - src/mmu_sv39/tlb.sv - - src/mmu_sv32/cva6_tlb_sv32.sv + - src/cva6_mmu/cva6_tlb.sv + - src/cva6_mmu/cva6_shared_tlb.sv file_type : systemVerilogSource depend : - pulp-platform.org::axi_mem_if diff --git a/core/Flist.cva6 b/core/Flist.cva6 index daf4b28a67..7bd972bf7d 100644 --- a/core/Flist.cva6 +++ b/core/Flist.cva6 @@ -182,15 +182,10 @@ ${CVA6_REPO_DIR}/common/local/util/tc_sram_wrapper.sv ${CVA6_REPO_DIR}/vendor/pulp-platform/tech_cells_generic/src/rtl/tc_sram.sv ${CVA6_REPO_DIR}/common/local/util/sram.sv -// MMU Sv39 -${CVA6_REPO_DIR}/core/mmu_sv39/mmu.sv -${CVA6_REPO_DIR}/core/mmu_sv39/ptw.sv -${CVA6_REPO_DIR}/core/mmu_sv39/tlb.sv - -// MMU Sv32 -${CVA6_REPO_DIR}/core/mmu_sv32/cva6_mmu_sv32.sv -${CVA6_REPO_DIR}/core/mmu_sv32/cva6_ptw_sv32.sv -${CVA6_REPO_DIR}/core/mmu_sv32/cva6_tlb_sv32.sv -${CVA6_REPO_DIR}/core/mmu_sv32/cva6_shared_tlb_sv32.sv +// MMU +${CVA6_REPO_DIR}/core/cva6_mmu/cva6_mmu.sv +${CVA6_REPO_DIR}/core/cva6_mmu/cva6_ptw.sv +${CVA6_REPO_DIR}/core/cva6_mmu/cva6_tlb.sv +${CVA6_REPO_DIR}/core/cva6_mmu/cva6_shared_tlb.sv // end of manifest diff --git a/core/cva6_mmu/cva6_mmu.sv b/core/cva6_mmu/cva6_mmu.sv new file mode 100644 index 0000000000..1936652f10 --- /dev/null +++ b/core/cva6_mmu/cva6_mmu.sv @@ -0,0 +1,884 @@ +// Copyright (c) 2018 ETH Zurich and University of Bologna. +// Copyright (c) 2021 Thales. +// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. +// Copyright (c) 2024 PlanV Technology +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez, PlanV Technology +// Date: 26/02/2024 +// +// Description: Memory Management Unit for CVA6, contains TLB and +// address translation unit. SV32 SV39 and SV39x4 as defined in RISC-V +// privilege specification 1.11-WIP. +// This module is an merge of the MMU Sv39 developed +// by Florian Zaruba, the MMU Sv32 developed by Sebastien Jacq and the MMU Sv39x4 developed by Bruno Sá. + + +module cva6_mmu + import ariane_pkg::*; +#( + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + // parameter ariane_pkg::ariane_cfg_t ArianeCfg = ariane_pkg::ArianeDefaultConfig, //This is the required config param in the hypervisor version for now + parameter int unsigned INSTR_TLB_ENTRIES = 4, + parameter int unsigned DATA_TLB_ENTRIES = 4, + parameter logic HYP_EXT = 0, + parameter int ASID_WIDTH [HYP_EXT:0], + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 + +) ( + input logic clk_i, + input logic rst_ni, + input logic flush_i, + input logic [HYP_EXT*2:0] enable_translation_i, //[v_i,enable_g_translation,enable_translation] + input logic [HYP_EXT*2:0] en_ld_st_translation_i, // enable virtual memory translation for ld/st + // IF interface + input icache_arsp_t icache_areq_i, + output icache_areq_t icache_areq_o, + // input icache_areq_o_t icache_areq_i, // this is the data type in the hypervisor version for now + // output icache_areq_i_t icache_areq_o, + + // LSU interface + // this is a more minimalistic interface because the actual addressing logic is handled + // in the LSU as we distinguish load and stores, what we do here is simple address translation + input exception_t misaligned_ex_i, + input logic lsu_req_i, // request address translation + input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in + input riscv::xlen_t lsu_tinst_i, // transformed instruction in + input logic lsu_is_store_i, // the translation is requested by a store + output logic csr_hs_ld_st_inst_o, // hyp load store instruction + // if we need to walk the page table we can't grant in the same cycle + // Cycle 0 + output logic lsu_dtlb_hit_o, // sent in same cycle as the request if translation hits in DTLB + output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) + // Cycle 1 + output logic lsu_valid_o, // translation is valid + output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address + output exception_t lsu_exception_o, // address translation threw an exception + // General control signals + input riscv::priv_lvl_t priv_lvl_i, + input riscv::priv_lvl_t ld_st_priv_lvl_i, + input logic [HYP_EXT:0] sum_i, + input logic [HYP_EXT:0] mxr_i, + input logic hlvx_inst_i, + input logic hs_ld_st_inst_i, + // input logic flag_mprv_i, + input logic [riscv::PPNW-1:0] satp_ppn_i[HYP_EXT*2:0], //[hgatp,vsatp,satp] + + input logic [ASID_WIDTH[0]-1:0] asid_i [HYP_EXT*2:0], //[vmid,vs_asid,asid] + input logic [ASID_WIDTH[0]-1:0] asid_to_be_flushed_i [ HYP_EXT:0], + input logic [ riscv::VLEN-1:0] vaddr_to_be_flushed_i[ HYP_EXT:0], + + input logic [HYP_EXT*2:0] flush_tlb_i, + + // Performance counters + output logic itlb_miss_o, + output logic dtlb_miss_o, + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + // PMP + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i +); + logic [ASID_WIDTH[0]-1:0] dtlb_mmu_asid_i[HYP_EXT:0]; + logic [ASID_WIDTH[0]-1:0] itlb_mmu_asid_i[HYP_EXT:0]; + + genvar b; + generate + for (b = 0; b < HYP_EXT + 1; b++) begin : gen_tlbs_asid + assign dtlb_mmu_asid_i[b] = b==0 ? + ((en_ld_st_translation_i[2*HYP_EXT] || flush_tlb_i[HYP_EXT]) ? asid_i[HYP_EXT] : asid_i[0]): + asid_i[HYP_EXT*2]; + assign itlb_mmu_asid_i[b] = b==0 ? + (enable_translation_i[2*HYP_EXT] ? asid_i[HYP_EXT] : asid_i[0]): + asid_i[HYP_EXT*2]; + end + endgenerate + + // memory management, pte for cva6 + localparam type pte_cva6_t = struct packed { + logic [riscv::PPNW-1:0] ppn; // PPN length for + logic [1:0] rsw; + logic d; + logic a; + logic g; + logic u; + logic x; + logic w; + logic r; + logic v; + }; + + localparam type tlb_update_cva6_t = struct packed { + logic valid; + logic [PT_LEVELS-2:0][HYP_EXT:0] is_page; + logic [VPN_LEN-1:0] vpn; + logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] asid; + logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled + pte_cva6_t [HYP_EXT:0] content; + }; + + logic [HYP_EXT:0] iaccess_err; // insufficient privilege to access this instruction page + logic [HYP_EXT:0] daccess_err; // insufficient privilege to access this data page + logic ptw_active; // PTW is currently walking a page table + logic walking_instr; // PTW is walking because of an ITLB miss + logic [HYP_EXT*2:0] ptw_error; // PTW threw an exception + logic ptw_access_exception; // PTW threw an access exception (PMPs) + logic [HYP_EXT:0][riscv::PLEN-1:0] ptw_bad_paddr; // PTW guest page fault bad guest physical addr + + logic [riscv::VLEN-1:0] update_vaddr, shared_tlb_vaddr; + + tlb_update_cva6_t update_itlb, update_dtlb, update_shared_tlb; + + logic itlb_lu_access; + pte_cva6_t [ HYP_EXT:0] itlb_content; + logic [ PT_LEVELS-2:0] itlb_is_page; + logic itlb_lu_hit; + logic [ riscv::GPLEN-1:0] itlb_gpaddr; + logic [ASID_WIDTH[0]-1:0] itlb_lu_asid; + + logic dtlb_lu_access; + pte_cva6_t [ HYP_EXT:0] dtlb_content; + logic [ PT_LEVELS-2:0] dtlb_is_page; + logic [ASID_WIDTH[0]-1:0] dtlb_lu_asid; + logic dtlb_lu_hit; + logic [ riscv::GPLEN-1:0] dtlb_gpaddr; + + logic shared_tlb_access; + logic shared_tlb_hit, itlb_req; + + // Assignments + + assign itlb_lu_access = icache_areq_i.fetch_req; + assign dtlb_lu_access = lsu_req_i; + + + cva6_tlb #( + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t), + .TLB_ENTRIES (INSTR_TLB_ENTRIES), + .HYP_EXT (HYP_EXT), + .ASID_WIDTH (ASID_WIDTH), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS) + ) i_itlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i (flush_tlb_i), + .v_st_enbl_i (enable_translation_i), + .update_i (update_itlb), + .lu_access_i (itlb_lu_access), + .lu_asid_i (itlb_mmu_asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (icache_areq_i.fetch_vaddr), + .lu_content_o (itlb_content), + .lu_gpaddr_o (itlb_gpaddr), + .lu_is_page_o (itlb_is_page), + .lu_hit_o (itlb_lu_hit) + ); + + cva6_tlb #( + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t), + .TLB_ENTRIES (DATA_TLB_ENTRIES), + .HYP_EXT (HYP_EXT), + .ASID_WIDTH (ASID_WIDTH), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS) + ) i_dtlb ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i (flush_tlb_i), + .v_st_enbl_i (en_ld_st_translation_i), + .update_i (update_dtlb), + .lu_access_i (dtlb_lu_access), + .lu_asid_i (dtlb_mmu_asid_i), + .asid_to_be_flushed_i (asid_to_be_flushed_i), + .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), + .lu_vaddr_i (lsu_vaddr_i), + .lu_content_o (dtlb_content), + .lu_gpaddr_o (dtlb_gpaddr), + .lu_is_page_o (dtlb_is_page), + .lu_hit_o (dtlb_lu_hit) + ); + + + cva6_shared_tlb #( + .SHARED_TLB_DEPTH (64), + .SHARED_TLB_WAYS (2), + .HYP_EXT (HYP_EXT), + .ASID_WIDTH (ASID_WIDTH), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS), + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t) + ) i_shared_tlb ( + .clk_i(clk_i), + .rst_ni(rst_ni), + .flush_i(flush_tlb_i), + .v_st_enbl_i({enable_translation_i, en_ld_st_translation_i}), + + .dtlb_asid_i (dtlb_mmu_asid_i), + .itlb_asid_i (itlb_mmu_asid_i), + // from TLBs + // did we miss? + .itlb_access_i(itlb_lu_access), + .itlb_hit_i (itlb_lu_hit), + .itlb_vaddr_i (icache_areq_i.fetch_vaddr), + + .dtlb_access_i(dtlb_lu_access), + .dtlb_hit_i (dtlb_lu_hit), + .dtlb_vaddr_i (lsu_vaddr_i), + + // to TLBs, update logic + .itlb_update_o(update_itlb), + .dtlb_update_o(update_dtlb), + + // Performance counters + .itlb_miss_o(itlb_miss_o), + .dtlb_miss_o(dtlb_miss_o), + + .shared_tlb_access_o(shared_tlb_access), + .shared_tlb_hit_o (shared_tlb_hit), + .shared_tlb_vaddr_o (shared_tlb_vaddr), + + .itlb_req_o (itlb_req), + // to update shared tlb + .shared_tlb_update_i(update_shared_tlb) + ); + + cva6_ptw #( + .CVA6Cfg (CVA6Cfg), + // .ArianeCfg ( ArianeCfg ), // this is the configuration needed in the hypervisor extension for now + .pte_cva6_t (pte_cva6_t), + .tlb_update_cva6_t(tlb_update_cva6_t), + .HYP_EXT (HYP_EXT), + .ASID_WIDTH (ASID_WIDTH), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS) + ) i_ptw ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i(flush_i), + + .ptw_active_o (ptw_active), + .walking_instr_o (walking_instr), + .ptw_error_o (ptw_error), + .ptw_access_exception_o(ptw_access_exception), + + .enable_translation_i (enable_translation_i), + .en_ld_st_translation_i(en_ld_st_translation_i), + + .lsu_is_store_i(lsu_is_store_i), + // PTW memory interface + .req_port_i (req_port_i), + .req_port_o (req_port_o), + + .asid_i(asid_i), + + .update_vaddr_o(update_vaddr), + + // to Shared TLB, update logic + .shared_tlb_update_o(update_shared_tlb), + + + // from shared TLB + // did we miss? + .shared_tlb_access_i(shared_tlb_access), + .shared_tlb_hit_i (shared_tlb_hit), + .shared_tlb_vaddr_i (shared_tlb_vaddr), + + .itlb_req_i(itlb_req), + + .hlvx_inst_i(hlvx_inst_i), + // from CSR file + .satp_ppn_i (satp_ppn_i), + .mxr_i (mxr_i), + + // Performance counters + .shared_tlb_miss_o(), //open for now + + // PMP + .pmpcfg_i (pmpcfg_i), + .pmpaddr_i (pmpaddr_i), + .bad_paddr_o(ptw_bad_paddr) + + ); + + //----------------------- + // Instruction Interface + //----------------------- + logic match_any_execute_region; + logic pmp_instr_allow; + localparam int PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; + + // The instruction interface is a simple request response interface + always_comb begin : instr_interface + // MMU disabled: just pass through + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + icache_areq_o.fetch_paddr = icache_areq_i.fetch_vaddr[((riscv::PLEN > riscv::VLEN) ? riscv::VLEN -1: riscv::PLEN -1 ):0]; + // two potential exception sources: + // 1. HPTW threw an exception -> signal with a page fault exception + // 2. We got an access error because of insufficient permissions -> throw an access exception + icache_areq_o.fetch_exception = '0; + // Check whether we are allowed to access this memory region from a fetch perspective + iaccess_err[0] = icache_areq_i.fetch_req && (enable_translation_i[0] || HYP_EXT == 0) && // + (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content[0].u) // + || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content[0].u)); + + if (HYP_EXT == 1) + iaccess_err[HYP_EXT] = icache_areq_i.fetch_req && enable_translation_i[HYP_EXT] && !itlb_content[HYP_EXT].u; + // MMU enabled: address from TLB, request delayed until hit. Error when TLB + // hit and no access right or TLB hit and translated address not valid (e.g. + // AXI decode error), or when PTW performs walk due to ITLB miss and raises + // an error. + if ((|enable_translation_i[HYP_EXT:0])) begin + // we work with SV39 or SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal + if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) + if (HYP_EXT == 1) + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, + {riscv::GPLEN{1'b0}}, + {riscv::XLEN{1'b0}}, + enable_translation_i[HYP_EXT*2], + 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, 1'b1 + }; + + icache_areq_o.fetch_valid = 1'b0; + + icache_areq_o.fetch_paddr = { + (enable_translation_i[HYP_EXT] && HYP_EXT == 1)? itlb_content[HYP_EXT].ppn : itlb_content[0].ppn, + icache_areq_i.fetch_vaddr[11:0] + }; + + if (itlb_is_page[0]) begin + + icache_areq_o.fetch_paddr[PPNWMin:12] = icache_areq_i.fetch_vaddr[PPNWMin:12]; + + end else if (PT_LEVELS == 3 && itlb_is_page[PT_LEVELS-2]) begin + + icache_areq_o.fetch_paddr[PPNWMin-(VPN_LEN/PT_LEVELS):12] = icache_areq_i.fetch_vaddr[PPNWMin-(VPN_LEN/PT_LEVELS):12]; + + end + // ---------// + // ITLB Hit + // --------// + // if we hit the ITLB output the request signal immediately + if (itlb_lu_hit) begin + icache_areq_o.fetch_valid = icache_areq_i.fetch_req; + if (HYP_EXT == 1 && iaccess_err[HYP_EXT]) + icache_areq_o.fetch_exception = { + riscv::INSTR_GUEST_PAGE_FAULT, + {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, + itlb_gpaddr[riscv::GPLEN-1:0], + {riscv::XLEN{1'b0}}, + enable_translation_i[HYP_EXT*2], + 1'b1 + }; + // we got an access error + else if (iaccess_err[0]) + // throw a page fault + if (HYP_EXT == 1) + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, + {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, + {riscv::GPLEN{1'b0}}, + {riscv::XLEN{1'b0}}, + enable_translation_i[HYP_EXT*2], + 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr}, + 1'b1 + }; + else if (!pmp_instr_allow) + if (HYP_EXT == 1) + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {riscv::XLEN'(icache_areq_i.fetch_vaddr)}, + {riscv::GPLEN{1'b0}}, + {riscv::XLEN{1'b0}}, + enable_translation_i[HYP_EXT*2], + 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, riscv::XLEN'(icache_areq_i.fetch_vaddr), 1'b1 + }; + end else if (ptw_active && walking_instr) begin + // ---------// + // ITLB Miss + // ---------// + // watch out for exceptions happening during walking the page table + icache_areq_o.fetch_valid = ptw_error[0] | ptw_access_exception; + if (ptw_error[0]) + if (HYP_EXT == 1 && ptw_error[HYP_EXT]) + icache_areq_o.fetch_exception = { + riscv::INSTR_GUEST_PAGE_FAULT, + {riscv::XLEN'(update_vaddr)}, + ptw_bad_paddr[HYP_EXT][riscv::GPLEN-1:0], + (ptw_error[HYP_EXT*2] ? (riscv::IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {riscv::XLEN{1'b0}}), + enable_translation_i[2*HYP_EXT], + 1'b1 + }; + else if (HYP_EXT == 1) + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, + {riscv::XLEN'(update_vaddr)}, + {riscv::GPLEN{1'b0}}, + {riscv::XLEN{1'b0}}, + enable_translation_i[2*HYP_EXT], + 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_PAGE_FAULT, {riscv::XLEN'(update_vaddr)}, 1'b1 + }; + else if (HYP_EXT == 1) + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {riscv::XLEN'(update_vaddr)}, + {riscv::GPLEN{1'b0}}, + {riscv::XLEN{1'b0}}, + enable_translation_i[2*HYP_EXT], + 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + ptw_bad_paddr[0][riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + + // if it didn't match any execute region throw an `Instruction Access Fault` + // or: if we are not translating, check PMPs immediately on the paddr + if ((!match_any_execute_region && (!ptw_error[0]|| HYP_EXT==0) ) || (!(|enable_translation_i[HYP_EXT:0]) && !pmp_instr_allow)) + if (HYP_EXT == 1) + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + {riscv::XLEN'(icache_areq_o.fetch_paddr)}, + {riscv::GPLEN{1'b0}}, + {riscv::XLEN{1'b0}}, + enable_translation_i[2*HYP_EXT], + 1'b1 + }; + else + icache_areq_o.fetch_exception = { + riscv::INSTR_ACCESS_FAULT, + riscv::VLEN'(icache_areq_o.fetch_paddr[riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0]), + 1'b1 + }; + end + + // check for execute flag on memory + assign match_any_execute_region = config_pkg::is_inside_execute_regions( + CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} + ); + // assign match_any_execute_region = ariane_pkg::is_inside_execute_regions(ArianeCfg, {{64-riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr}); // this is the package used in the hypervisor extension for now + + // Instruction fetch + pmp #( + .CVA6Cfg (CVA6Cfg), //comment for hypervisor extension + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + // .NR_ENTRIES ( ArianeCfg.NrPMPEntries ) // configuration used in hypervisor extension + ) i_pmp_if ( + .addr_i (icache_areq_o.fetch_paddr), + .priv_lvl_i, + // we will always execute on the instruction fetch port + .access_type_i(riscv::ACCESS_EXEC), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_instr_allow) + ); + + //----------------------- + // Data Interface + //----------------------- + logic [HYP_EXT:0][riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; + logic [riscv::XLEN-1:0] lsu_tinst_n, lsu_tinst_q; + logic hs_ld_st_inst_n, hs_ld_st_inst_q; + pte_cva6_t [HYP_EXT:0] dtlb_pte_n, dtlb_pte_q; + exception_t misaligned_ex_n, misaligned_ex_q; + logic lsu_req_n, lsu_req_q; + logic lsu_is_store_n, lsu_is_store_q; + logic dtlb_hit_n, dtlb_hit_q; + logic [PT_LEVELS-2:0] dtlb_is_page_n, dtlb_is_page_q; + + // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) + assign lsu_dtlb_hit_o = (en_ld_st_translation_i[HYP_EXT:0]) ? dtlb_lu_hit : 1'b1; + + // Wires to PMP checks + riscv::pmp_access_t pmp_access_type; + logic pmp_data_allow; + + + // The data interface is simpler and only consists of a request/response interface + always_comb begin : data_interface + // save request and DTLB response + lsu_vaddr_n[0] = lsu_vaddr_i; + lsu_req_n = lsu_req_i; + misaligned_ex_n = misaligned_ex_i; + dtlb_pte_n = dtlb_content; + dtlb_hit_n = dtlb_lu_hit; + lsu_is_store_n = lsu_is_store_i; + dtlb_is_page_n = dtlb_is_page; + + lsu_valid_o = lsu_req_q; + lsu_exception_o = misaligned_ex_q; + pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; + + // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions + misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; + + // Check if the User flag is set, then we may only access it in supervisor mode + // if SUM is enabled + daccess_err[0] = (en_ld_st_translation_i[0] || HYP_EXT==0)&& + ((ld_st_priv_lvl_i == riscv::PRIV_LVL_S && (en_ld_st_translation_i[HYP_EXT*2] ? !sum_i[HYP_EXT] : !sum_i[0] ) && dtlb_pte_q[0].u) || // SUM is not set and we are trying to access a user page in supervisor mode + (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q[0].u)); + + if (HYP_EXT == 1) begin + lsu_tinst_n = lsu_tinst_i; + hs_ld_st_inst_n = hs_ld_st_inst_i; + lsu_vaddr_n[HYP_EXT] = dtlb_gpaddr; + csr_hs_ld_st_inst_o = hs_ld_st_inst_i || hs_ld_st_inst_q; + daccess_err[HYP_EXT] = en_ld_st_translation_i[HYP_EXT] && !dtlb_pte_q[1].u; + end + + lsu_paddr_o = (riscv::PLEN)'(lsu_vaddr_q[0]); + lsu_dtlb_ppn_o = (riscv::PPNW)'(lsu_vaddr_n[0][((riscv::PLEN > riscv::VLEN) ? riscv::VLEN -1: riscv::PLEN -1 ):12]); + + // translation is enabled and no misaligned exception occurred + if ((|en_ld_st_translation_i[HYP_EXT:0]) && !misaligned_ex_q.valid) begin + lsu_valid_o = 1'b0; + + lsu_dtlb_ppn_o = (en_ld_st_translation_i[HYP_EXT] && HYP_EXT == 1)? dtlb_content[HYP_EXT].ppn :dtlb_content[0].ppn; + lsu_paddr_o = { + (en_ld_st_translation_i[HYP_EXT] && HYP_EXT == 1)? dtlb_pte_q[HYP_EXT].ppn : dtlb_pte_q[0].ppn, + lsu_vaddr_q[0][11:0] + }; + // Mega page + if (dtlb_is_page_q[0]) begin + + lsu_dtlb_ppn_o[PPNWMin:12] = lsu_vaddr_n[0][PPNWMin:12]; + lsu_paddr_o[PPNWMin:12] = lsu_vaddr_q[0][PPNWMin:12]; + + end else if (PT_LEVELS == 3 && dtlb_is_page_q[PT_LEVELS-2]) begin + + lsu_paddr_o[PPNWMin-(VPN_LEN/PT_LEVELS):12] = lsu_vaddr_q[0][PPNWMin-(VPN_LEN/PT_LEVELS):12]; + lsu_dtlb_ppn_o[PPNWMin-(VPN_LEN/PT_LEVELS):12] = lsu_vaddr_n[0][PPNWMin-(VPN_LEN/PT_LEVELS):12]; + + end + + // --------- + // DTLB Hit + // -------- + if (dtlb_hit_q && lsu_req_q) begin + lsu_valid_o = 1'b1; + // exception priority: + // PAGE_FAULTS have higher priority than ACCESS_FAULTS + // virtual memory based exceptions are PAGE_FAULTS + // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) + + // this is a store + if (lsu_is_store_q) begin + // check if the page is write-able and we are not violating privileges + // also check if the dirty flag is set + if(HYP_EXT==1 && en_ld_st_translation_i[HYP_EXT] && (!dtlb_pte_q[HYP_EXT].w || daccess_err[HYP_EXT] || !dtlb_pte_q[HYP_EXT].d)) begin + lsu_exception_o = { + riscv::STORE_GUEST_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + lsu_vaddr_q[1][riscv::GPLEN-1:0], + {riscv::XLEN{1'b0}}, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else if ((en_ld_st_translation_i[0] || HYP_EXT==0) && (!dtlb_pte_q[0].w || daccess_err[0] || !dtlb_pte_q[0].d)) begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + 1'b1 + }; + end + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + {riscv::XLEN'(lsu_paddr_o)}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + riscv::XLEN'(lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0]), + 1'b1 + }; + end + end + + // this is a load + end else begin + if (HYP_EXT == 1 && daccess_err[HYP_EXT]) begin + lsu_exception_o = { + riscv::LOAD_GUEST_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + lsu_vaddr_q[1][riscv::GPLEN-1:0], + {riscv::XLEN{1'b0}}, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + // check for sufficient access privileges - throw a page fault if necessary + end else if (daccess_err[0]) begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + 1'b1 + }; + end + // Check if any PMPs are violated + end else if (!pmp_data_allow) begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, lsu_vaddr_q[0]}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + end + end else + + // --------- + // DTLB Miss + // --------- + // watch out for exceptions + if (ptw_active && !walking_instr) begin + // page table walker threw an exception + if (ptw_error[0]) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + if (lsu_is_store_q) begin + if (HYP_EXT == 1 && ptw_error[HYP_EXT]) begin + lsu_exception_o = { + riscv::STORE_GUEST_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + ptw_bad_paddr[HYP_EXT][riscv::GPLEN-1:0], + (ptw_error[HYP_EXT*2] ? (riscv::IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {riscv::XLEN{1'b0}}), + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::STORE_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + 1'b1 + }; + end + end + end else begin + if (HYP_EXT == 1 && ptw_error[HYP_EXT]) begin + lsu_exception_o = { + riscv::LOAD_GUEST_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + ptw_bad_paddr[HYP_EXT][riscv::GPLEN-1:0], + (ptw_error[HYP_EXT*2] ? (riscv::IS_XLEN64 ? riscv::READ_64_PSEUDOINSTRUCTION : riscv::READ_32_PSEUDOINSTRUCTION) : {riscv::XLEN{1'b0}}), + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LOAD_PAGE_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + 1'b1 + }; + end + end + end + end + + if (ptw_access_exception) begin + // an error makes the translation valid + lsu_valid_o = 1'b1; + // the page table walker can only throw page faults + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + ptw_bad_paddr[0][riscv::PLEN-1:(riscv::PLEN > riscv::VLEN) ? (riscv::PLEN - riscv::VLEN) : 0], + 1'b1 + }; + end + end + end + end // If translation is not enabled, check the paddr immediately against PMPs +else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin + if (lsu_is_store_q) begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else + lsu_exception_o = { + riscv::ST_ACCESS_FAULT, + lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end else begin + if (HYP_EXT == 1) begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + {{riscv::XLEN - riscv::VLEN{lsu_vaddr_q[0][riscv::VLEN-1]}}, update_vaddr}, + {riscv::GPLEN{1'b0}}, + lsu_tinst_q, + en_ld_st_translation_i[HYP_EXT*2], + 1'b1 + }; + end else begin + lsu_exception_o = { + riscv::LD_ACCESS_FAULT, + lsu_paddr_o[riscv::PLEN-1:(riscv::PLEN>riscv::VLEN)?(riscv::PLEN-riscv::VLEN) : 0], + 1'b1 + }; + end + end + end + end + + // Load/store PMP check + pmp #( + .CVA6Cfg (CVA6Cfg), // COMMENT IN HYPERVISOR EXTENSION + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + // .NR_ENTRIES ( ArianeCfg.NrPMPEntries ) // CONFIGURATION USED IN HYPERVISOR EXTENSION + ) i_pmp_data ( + .addr_i (lsu_paddr_o), + .priv_lvl_i (ld_st_priv_lvl_i), + .access_type_i(pmp_access_type), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (pmp_data_allow) + ); + + // ---------- + // Registers + // ---------- + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + lsu_vaddr_q <= '0; + lsu_req_q <= '0; + misaligned_ex_q <= '0; + dtlb_pte_q <= '0; + dtlb_hit_q <= '0; + lsu_is_store_q <= '0; + dtlb_is_page_q <= '0; + + if (HYP_EXT == 1) begin + lsu_tinst_q <= '0; + hs_ld_st_inst_q <= '0; + end + end else begin + lsu_vaddr_q <= lsu_vaddr_n; + lsu_req_q <= lsu_req_n; + misaligned_ex_q <= misaligned_ex_n; + dtlb_pte_q <= dtlb_pte_n; + dtlb_hit_q <= dtlb_hit_n; + lsu_is_store_q <= lsu_is_store_n; + dtlb_is_page_q <= dtlb_is_page_n; + + if (HYP_EXT == 1) begin + lsu_tinst_q <= lsu_tinst_n; + hs_ld_st_inst_q <= hs_ld_st_inst_n; + end + end + end +endmodule diff --git a/core/cva6_mmu/cva6_ptw.sv b/core/cva6_mmu/cva6_ptw.sv new file mode 100644 index 0000000000..5923c36232 --- /dev/null +++ b/core/cva6_mmu/cva6_ptw.sv @@ -0,0 +1,626 @@ +// Copyright (c) 2018 ETH Zurich and University of Bologna. +// Copyright (c) 2021 Thales. +// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. +// Copyright (c) 2024 PlanV Technology +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez, PlanV Technology +// Date: 26/02/2024 +// Description: Hardware-PTW (Page-Table-Walker) for CVA6 supporting sv32, sv39 and sv39x4. +// This module is an merge of the PTW Sv39 developed by Florian Zaruba, +// the PTW Sv32 developed by Sebastien Jacq and the PTW Sv39x4 by Bruno Sá. + +/* verilator lint_off WIDTH */ + +module cva6_ptw + import ariane_pkg::*; +#( + parameter type pte_cva6_t = logic, + parameter type tlb_update_cva6_t = logic, + parameter int unsigned HYP_EXT = 0, + parameter int ASID_WIDTH[HYP_EXT:0], + parameter int unsigned VPN_LEN = 1, + parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, + parameter int unsigned PT_LEVELS = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic flush_i, // flush everything, we need to do this because + // actually everything we do is speculative at this stage + // e.g.: there could be a CSR instruction that changes everything + output logic ptw_active_o, + output logic walking_instr_o, // set when walking for TLB + output logic [HYP_EXT*2:0] ptw_error_o, // set when an error occurred + output logic ptw_access_exception_o, // set when an PMP access exception occured + input logic [HYP_EXT*2:0] enable_translation_i, //[v_i,enable_g_translation,enable_translation] + input logic [HYP_EXT*2:0] en_ld_st_translation_i, // enable virtual memory translation for load/stores + input logic hlvx_inst_i, // is a HLVX load/store instruction + + input logic lsu_is_store_i, // this translation was triggered by a store + // PTW memory interface + input dcache_req_o_t req_port_i, + output dcache_req_i_t req_port_o, + + + // to TLBs, update logic + output tlb_update_cva6_t shared_tlb_update_o, + + output logic [riscv::VLEN-1:0] update_vaddr_o, + + input logic [ASID_WIDTH[0]-1:0] asid_i[HYP_EXT*2:0], //[vmid,vs_asid,asid] + + // from TLBs + // did we miss? + input logic shared_tlb_access_i, + input logic shared_tlb_hit_i, + input logic [riscv::VLEN-1:0] shared_tlb_vaddr_i, + + input logic itlb_req_i, + + // from CSR file + input logic [riscv::PPNW-1:0] satp_ppn_i[HYP_EXT*2:0], //[hgatp,vsatp,satp] + input logic [ HYP_EXT:0] mxr_i, + + // Performance counters + output logic shared_tlb_miss_o, + + // PMP + + input riscv::pmpcfg_t [15:0] pmpcfg_i, + input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, + output logic [HYP_EXT:0][riscv::PLEN-1:0] bad_paddr_o + +); + + // input registers + logic data_rvalid_q; + riscv::xlen_t data_rdata_q; + + pte_cva6_t [HYP_EXT*2:0] pte; //[gpte_d,gpte_q,pte] + // register to perform context switch between stages + // pte_cva6_t gpte_q, gpte_d; + assign pte[0] = pte_cva6_t'(data_rdata_q[riscv::PPNW+9:0]); + + enum logic [2:0] { + IDLE, + WAIT_GRANT, + PTE_LOOKUP, + WAIT_RVALID, + PROPAGATE_ERROR, + PROPAGATE_ACCESS_ERROR, + LATENCY + } + state_q, state_d; + + logic [PT_LEVELS-2:0] misaligned_page; + logic [HYP_EXT:0][PT_LEVELS-2:0] ptw_lvl_n, ptw_lvl_q; + + // define 3 PTW stages to be used in sv39x4. sv32 and sv39 are always in S_STAGE + // S_STAGE -> S/VS-stage normal translation controlled by the satp/vsatp CSRs + // G_INTERMED_STAGE -> Converts the S/VS-stage non-leaf GPA pointers to HPA (controlled by hgatp) + // G_FINAL_STAGE -> Converts the S/VS-stage final GPA to HPA (controlled by hgatp) + enum logic [1:0] { + S_STAGE, + G_INTERMED_STAGE, + G_FINAL_STAGE + } + ptw_stage_q, ptw_stage_d; + + // is this an instruction page table walk? + logic is_instr_ptw_q, is_instr_ptw_n; + logic global_mapping_q, global_mapping_n; + // latched tag signal + logic tag_valid_n, tag_valid_q; + // register the ASIDs + logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] tlb_update_asid_q, tlb_update_asid_n; + // register the VPN we need to walk, SV39 defines a 39 bit virtual address + logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; + logic [HYP_EXT*2:0][PT_LEVELS-2:0][(VPN_LEN/PT_LEVELS)-1:0] vaddr_lvl; + // register the VPN we need to walk, SV39x4 defines a 41 bit virtual address for the G-Stage + logic [riscv::GPLEN-1:0] gpaddr_q, gpaddr_n, gpaddr_base; + logic [PT_LEVELS-2:0][riscv::GPLEN-1:0] gpaddr; + // 4 byte aligned physical pointer + logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; + logic [riscv::PLEN-1:0] gptw_pptr_q, gptw_pptr_n; + + // Assignments + assign update_vaddr_o = vaddr_q; + + assign ptw_active_o = (state_q != IDLE); + assign walking_instr_o = is_instr_ptw_q; + // directly output the correct physical address + assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; + assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; + // we are never going to kill this request + assign req_port_o.kill_req = '0; + // we are never going to write with the HPTW + assign req_port_o.data_wdata = '0; + // we only issue one single request at a time + assign req_port_o.data_id = '0; + + // ----------- + // TLB Update + // ----------- + + assign gpaddr_base = {pte[0].ppn[riscv::GPPNW-1:0], vaddr_q[11:0]}; + + genvar z, w; + generate + for (z = 0; z < PT_LEVELS - 1; z++) begin + + // check if the ppn is correctly aligned: + // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault + // exception. + assign misaligned_page[z] = (ptw_lvl_q[0] == (z)) && (pte[0].ppn[(VPN_LEN/PT_LEVELS)*(PT_LEVELS-1-z)-1:0] != '0); + + //record the vaddr corresponding to each level + for (w = 0; w < HYP_EXT * 2 + 1; w++) begin + assign vaddr_lvl[w][z] = w==0 ? vaddr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-2))] : + w==1 ? gptw_pptr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-2))]: + gpaddr_q[12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-1))-1:12+((VPN_LEN/PT_LEVELS)*(PT_LEVELS-z-2))]; + end + + if (HYP_EXT == 1) begin + assign gpaddr[z][VPN_LEN-(VPN_LEN/PT_LEVELS):0]= (ptw_lvl_q[0] == z) ? vaddr_q[VPN_LEN-(VPN_LEN/PT_LEVELS):0] : gpaddr_base[VPN_LEN-(VPN_LEN/PT_LEVELS):0]; + assign gpaddr[z][VPN_LEN:VPN_LEN-(VPN_LEN/PT_LEVELS)+1]= (ptw_lvl_q[0] == 0) ? vaddr_q[VPN_LEN:VPN_LEN-(VPN_LEN/PT_LEVELS)+1] : gpaddr_base[VPN_LEN:VPN_LEN-(VPN_LEN/PT_LEVELS)+1]; + assign gpaddr[z][riscv::GPLEN-1:VPN_LEN+1] = gpaddr_base[riscv::GPLEN-1:VPN_LEN+1]; + end + + + end + endgenerate + + always_comb begin : tlb_update + // update the correct page table level + for (int unsigned y = 0; y < HYP_EXT + 1; y++) begin + for (int unsigned x = 0; x < PT_LEVELS - 1; x++) begin + if((&enable_translation_i[HYP_EXT:0] || &en_ld_st_translation_i[HYP_EXT:0])&& HYP_EXT==1) begin + shared_tlb_update_o.is_page[x][y] = (ptw_lvl_q[y==HYP_EXT?0 : 1] == x); + end else if (enable_translation_i[0] || en_ld_st_translation_i[0] || HYP_EXT == 0) begin + shared_tlb_update_o.is_page[x][y] = y == 0 ? (ptw_lvl_q[0] == x) : 1'b0; + end else begin + shared_tlb_update_o.is_page[x][y] = y != 0 ? (ptw_lvl_q[0] == x) : 1'b0; + end + end + + // set the global mapping bit + if ((enable_translation_i[HYP_EXT] || en_ld_st_translation_i[HYP_EXT]) && HYP_EXT == 1) begin + shared_tlb_update_o.content[y] = y == 0 ? pte[HYP_EXT] | (global_mapping_q << 5) : pte[0]; + end else begin + shared_tlb_update_o.content[y] = y == 0 ? (pte[0] | (global_mapping_q << 5)) : '0; + end + end + // output the correct ASIDs + shared_tlb_update_o.asid = tlb_update_asid_q; + + bad_paddr_o[0] = ptw_access_exception_o ? ptw_pptr_q : 'b0; + if (HYP_EXT == 1) + bad_paddr_o[HYP_EXT][riscv::GPLEN:0] = ptw_error_o[HYP_EXT] ? ((ptw_stage_q == G_INTERMED_STAGE) ? gptw_pptr_q[riscv::GPLEN:0] : gpaddr_q) : 'b0; + end + + assign req_port_o.tag_valid = tag_valid_q; + + logic allow_access; + + + + pmp #( + .PLEN (riscv::PLEN), + .PMP_LEN (riscv::PLEN - 2), + .NR_ENTRIES(CVA6Cfg.NrPMPEntries) + ) i_pmp_ptw ( + .addr_i (ptw_pptr_q), + // PTW access are always checked as if in S-Mode... + .priv_lvl_i (riscv::PRIV_LVL_S), + // ...and they are always loads + .access_type_i(riscv::ACCESS_READ), + // Configuration + .conf_addr_i (pmpaddr_i), + .conf_i (pmpcfg_i), + .allow_o (allow_access) + ); + + + assign req_port_o.data_be = riscv::XLEN == 32 ? be_gen_32( + req_port_o.address_index[1:0], req_port_o.data_size + ) : be_gen( + req_port_o.address_index[2:0], req_port_o.data_size + ); + + assign shared_tlb_update_o.vpn = VPN_LEN'(vaddr_q[riscv::SV+HYP_EXT*2-1:12]); + + //------------------- + // Page table walker + //------------------- + // A virtual address va is translated into a physical address pa as follows: + // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, + // PAGESIZE=2^12 and LEVELS=3.) + // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For + // Sv32, PTESIZE=4.) + // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, or if any bits or encodings + // that are reserved for future standard use are set within pte, stop and raise + // a page-fault exception corresponding to the original access type. + // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. + // Otherwise, this PTE is a pointer to the next level of the page table. + // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let + // a = pte.ppn × PAGESIZE and go to step 2. + // 5. A leaf PTE has been found. Determine if the requested memory access + // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and + // raise an access exception. Otherwise, the translation is successful. + // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. + // The translated physical address is given as follows: + // - pa.pgoff = va.pgoff. + // - If i > 0, then this is a superpage translation and + // pa.ppn[i-1:0] = va.vpn[i-1:0]. + // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. + always_comb begin : ptw + automatic logic [riscv::PLEN-1:0] pptr; + // automatic logic [riscv::GPLEN-1:0] gpaddr; + // default assignments + // PTW memory interface + tag_valid_n = 1'b0; + req_port_o.data_req = 1'b0; + req_port_o.data_size = 2'(PT_LEVELS); + req_port_o.data_we = 1'b0; + ptw_error_o = '0; + ptw_access_exception_o = 1'b0; + shared_tlb_update_o.valid = 1'b0; + is_instr_ptw_n = is_instr_ptw_q; + ptw_lvl_n = ptw_lvl_q; + ptw_pptr_n = ptw_pptr_q; + state_d = state_q; + ptw_stage_d = ptw_stage_q; + global_mapping_n = global_mapping_q; + // input registers + tlb_update_asid_n = tlb_update_asid_q; + vaddr_n = vaddr_q; + pptr = ptw_pptr_q; + + if (HYP_EXT == 1) begin + gpaddr_n = gpaddr_q; + gptw_pptr_n = gptw_pptr_q; + end + + shared_tlb_miss_o = 1'b0; + + if (HYP_EXT == 1) pte[HYP_EXT*2] = pte[HYP_EXT]; + + case (state_q) + + IDLE: begin + // by default we start with the top-most page table + ptw_lvl_n = '0; + global_mapping_n = 1'b0; + is_instr_ptw_n = 1'b0; + + + if (HYP_EXT == 1) begin + pte[HYP_EXT*2] = '0; + gpaddr_n = '0; + end + + + // if we got an ITLB miss + if ((|enable_translation_i[HYP_EXT:0] || |en_ld_st_translation_i[HYP_EXT:0] || HYP_EXT == 0) && shared_tlb_access_i && ~shared_tlb_hit_i) begin + if ((&enable_translation_i[HYP_EXT:0] || &en_ld_st_translation_i[HYP_EXT:0]) && HYP_EXT==1) begin + ptw_stage_d = G_INTERMED_STAGE; + pptr = { + satp_ppn_i[HYP_EXT], + shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + gptw_pptr_n = pptr; + ptw_pptr_n = { + satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], + pptr[riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + end else if (((|enable_translation_i[HYP_EXT:0] && !enable_translation_i[0]) || (|en_ld_st_translation_i[HYP_EXT:0] && !en_ld_st_translation_i[0])) && HYP_EXT==1) begin + ptw_stage_d = G_FINAL_STAGE; + gpaddr_n = shared_tlb_vaddr_i[riscv::SV+HYP_EXT*2-1:0]; + ptw_pptr_n = { + satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], + shared_tlb_vaddr_i[riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + end else begin + ptw_stage_d = S_STAGE; + if((enable_translation_i[HYP_EXT*2] || en_ld_st_translation_i[HYP_EXT*2]) && HYP_EXT==1) + ptw_pptr_n = { + satp_ppn_i[HYP_EXT], + shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + else + ptw_pptr_n = { + satp_ppn_i[0], + shared_tlb_vaddr_i[riscv::SV-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + end + + is_instr_ptw_n = itlb_req_i; + vaddr_n = shared_tlb_vaddr_i; + state_d = WAIT_GRANT; + shared_tlb_miss_o = 1'b1; + + for (int unsigned b = 0; b < HYP_EXT + 1; b++) begin + tlb_update_asid_n[b] = b==0 ? ((enable_translation_i[2*HYP_EXT] || en_ld_st_translation_i[2*HYP_EXT]) ? asid_i[HYP_EXT] : asid_i[0]) : asid_i[HYP_EXT*2]; + end + end + end + + WAIT_GRANT: begin + // send a request out + req_port_o.data_req = 1'b1; + // wait for the WAIT_GRANT + if (req_port_i.data_gnt) begin + // send the tag valid signal one cycle later + tag_valid_n = 1'b1; + state_d = PTE_LOOKUP; + end + end + + PTE_LOOKUP: begin + // we wait for the valid signal + if (data_rvalid_q) begin + + // check if the global mapping bit is set + if (pte[0].g && (ptw_stage_q == S_STAGE || HYP_EXT == 0)) global_mapping_n = 1'b1; + + // ------------- + // Invalid PTE + // ------------- + // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. + if (!pte[0].v || (!pte[0].r && pte[0].w)) // || (|pte.reserved)) + state_d = PROPAGATE_ERROR; + // ----------- + // Valid PTE + // ----------- + else begin + state_d = LATENCY; + // it is a valid PTE + // if pte.r = 1 or pte.x = 1 it is a valid PTE + if (pte[0].r || pte[0].x) begin + if (HYP_EXT == 1) begin + case (ptw_stage_q) + S_STAGE: begin + if ((is_instr_ptw_q && enable_translation_i[HYP_EXT]) || (!is_instr_ptw_q && en_ld_st_translation_i[HYP_EXT])) begin + state_d = WAIT_GRANT; + ptw_stage_d = G_FINAL_STAGE; + if (HYP_EXT == 1) pte[HYP_EXT*2] = pte[0]; + ptw_lvl_n[HYP_EXT] = ptw_lvl_q[0]; + gpaddr_n = gpaddr[ptw_lvl_q[0]]; + ptw_pptr_n = { + satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], + gpaddr[ptw_lvl_q[0]][riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + ptw_lvl_n[0] = '0; + end + end + G_INTERMED_STAGE: begin + state_d = WAIT_GRANT; + ptw_stage_d = S_STAGE; + ptw_lvl_n[0] = ptw_lvl_q[HYP_EXT]; + pptr = {pte[0].ppn[riscv::GPPNW-1:0], gptw_pptr_q[11:0]}; + if (ptw_lvl_q[0] == 1) pptr[20:0] = gptw_pptr_q[20:0]; + if (ptw_lvl_q[0] == 0) pptr[29:0] = gptw_pptr_q[29:0]; + ptw_pptr_n = pptr; + end + default: ; + endcase + end + // Valid translation found (either 1G, 2M or 4K entry) + if (is_instr_ptw_q) begin + // ------------ + // Update ITLB + // ------------ + // If page is not executable, we can directly raise an error. This + // doesn't put a useless entry into the TLB. The same idea applies + // to the access flag since we let the access flag be managed by SW. + if (!pte[0].x || !pte[0].a) begin + state_d = PROPAGATE_ERROR; + if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; + end else if((ptw_stage_q == G_FINAL_STAGE) || !enable_translation_i[HYP_EXT] || HYP_EXT==0) + shared_tlb_update_o.valid = 1'b1; + + end else begin + // ------------ + // Update DTLB + // ------------ + // Check if the access flag has been set, otherwise throw a page-fault + // and let the software handle those bits. + // If page is not readable (there are no write-only pages) + // we can directly raise an error. This doesn't put a useless + // entry into the TLB. + if (pte[0].a && ((pte[0].r && !hlvx_inst_i) || (pte[0].x && (mxr_i[0] || hlvx_inst_i || (ptw_stage_q == S_STAGE && mxr_i[HYP_EXT] && en_ld_st_translation_i[HYP_EXT*2] && HYP_EXT==1))))) begin + if((ptw_stage_q == G_FINAL_STAGE) || !en_ld_st_translation_i[HYP_EXT] || HYP_EXT==0) + shared_tlb_update_o.valid = 1'b1; + end else begin + state_d = PROPAGATE_ERROR; + if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; + end + // Request is a store: perform some additional checks + // If the request was a store and the page is not write-able, raise an error + // the same applies if the dirty flag is not set + if (lsu_is_store_i && (!pte[0].w || !pte[0].d)) begin + shared_tlb_update_o.valid = 1'b0; + state_d = PROPAGATE_ERROR; + if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; + end + end + + //if there is a misaligned page, propagate error + if (|misaligned_page) begin + state_d = PROPAGATE_ERROR; + if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; + shared_tlb_update_o.valid = 1'b0; + end + + // check if 63:41 are all zeros + if (HYP_EXT==1 && ((enable_translation_i[HYP_EXT*2] && is_instr_ptw_q) || (en_ld_st_translation_i[HYP_EXT*2] && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte[0].ppn[riscv::PPNW-HYP_EXT:riscv::GPPNW]) == 1'b0)) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = G_FINAL_STAGE; + end + // this is a pointer to the next TLB level + end else begin + // pointer to next level of page table + + if (ptw_lvl_q[0] == PT_LEVELS - 1) begin + // Should already be the last level page table => Error + ptw_lvl_n[0] = ptw_lvl_q[0]; + state_d = PROPAGATE_ERROR; + if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; + + + end else begin + ptw_lvl_n[0] = ptw_lvl_q[0] + 1'b1; + state_d = WAIT_GRANT; + + if (HYP_EXT == 1) begin + case (ptw_stage_q) + S_STAGE: begin + if (HYP_EXT==1 && ((is_instr_ptw_q && enable_translation_i[HYP_EXT]) || (!is_instr_ptw_q && en_ld_st_translation_i[HYP_EXT]))) begin + ptw_stage_d = G_INTERMED_STAGE; + if (HYP_EXT == 1) pte[HYP_EXT*2] = pte[0]; + ptw_lvl_n[HYP_EXT] = ptw_lvl_q[0] + 1; + pptr = {pte[0].ppn, vaddr_lvl[0][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; + gptw_pptr_n = pptr; + ptw_pptr_n = { + satp_ppn_i[HYP_EXT*2][riscv::PPNW-1:2], + pptr[riscv::SV+HYP_EXT*2-1:riscv::SV-(VPN_LEN/PT_LEVELS)], + (PT_LEVELS)'(0) + }; + ptw_lvl_n[0] = '0; + end else begin + ptw_pptr_n = {pte[0].ppn, vaddr_lvl[0][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; + end + end + G_INTERMED_STAGE: begin + ptw_pptr_n = {pte[0].ppn, vaddr_lvl[HYP_EXT][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; + end + G_FINAL_STAGE: begin + ptw_pptr_n = { + pte[0].ppn, vaddr_lvl[HYP_EXT*2][ptw_lvl_q[0]], (PT_LEVELS)'(0) + }; + end + endcase + end else ptw_pptr_n = {pte[0].ppn, vaddr_lvl[0][ptw_lvl_q[0]], (PT_LEVELS)'(0)}; + + if (HYP_EXT == 1 && (pte[0].a || pte[0].d || pte[0].u)) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + + end + + // check if 63:41 are all zeros + if (HYP_EXT==1 && (((enable_translation_i[HYP_EXT*2] && is_instr_ptw_q) || (en_ld_st_translation_i[HYP_EXT*2] && !is_instr_ptw_q)) && ptw_stage_q == S_STAGE && !((|pte[0].ppn[riscv::PPNW-1:riscv::GPPNW]) == 1'b0))) begin + state_d = PROPAGATE_ERROR; + ptw_stage_d = ptw_stage_q; + end + end + end + + // Check if this access was actually allowed from a PMP perspective + if (!allow_access) begin + shared_tlb_update_o.valid = 1'b0; + // we have to return the failed address in bad_addr + ptw_pptr_n = ptw_pptr_q; + if (HYP_EXT == 1) ptw_stage_d = ptw_stage_q; + state_d = PROPAGATE_ACCESS_ERROR; + end + end + // we've got a data WAIT_GRANT so tell the cache that the tag is valid + end + // Propagate error to MMU/LSU + PROPAGATE_ERROR: begin + state_d = LATENCY; + ptw_error_o[0] = 1'b1; + if (HYP_EXT == 1) begin + ptw_error_o[HYP_EXT] = (ptw_stage_q != S_STAGE) ? 1'b1 : 1'b0; + ptw_error_o[HYP_EXT*2] = (ptw_stage_q == G_INTERMED_STAGE) ? 1'b1 : 1'b0; + end + end + PROPAGATE_ACCESS_ERROR: begin + state_d = LATENCY; + ptw_access_exception_o = 1'b1; + end + // wait for the rvalid before going back to IDLE + WAIT_RVALID: begin + if (data_rvalid_q) state_d = IDLE; + end + LATENCY: begin + state_d = IDLE; + end + default: begin + state_d = IDLE; + end + endcase + + // ------- + // Flush + // ------- + // should we have flushed before we got an rvalid, wait for it until going back to IDLE + if (flush_i) begin + // on a flush check whether we are + // 1. in the PTE Lookup check whether we still need to wait for an rvalid + // 2. waiting for a grant, if so: wait for it + // if not, go back to idle + if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) + state_d = WAIT_RVALID; + else state_d = LATENCY; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + state_q <= IDLE; + is_instr_ptw_q <= 1'b0; + ptw_lvl_q <= '0; + tag_valid_q <= 1'b0; + tlb_update_asid_q <= '0; + vaddr_q <= '0; + ptw_pptr_q <= '0; + global_mapping_q <= 1'b0; + data_rdata_q <= '0; + data_rvalid_q <= 1'b0; + if (HYP_EXT == 1) begin + gpaddr_q <= '0; + gptw_pptr_q <= '0; + ptw_stage_q <= S_STAGE; + pte[HYP_EXT] <= '0; + end + end else begin + state_q <= state_d; + ptw_pptr_q <= ptw_pptr_n; + is_instr_ptw_q <= is_instr_ptw_n; + ptw_lvl_q <= ptw_lvl_n; + tag_valid_q <= tag_valid_n; + tlb_update_asid_q <= tlb_update_asid_n; + vaddr_q <= vaddr_n; + global_mapping_q <= global_mapping_n; + data_rdata_q <= req_port_i.data_rdata; + data_rvalid_q <= req_port_i.data_rvalid; + + if (HYP_EXT == 1) begin + gpaddr_q <= gpaddr_n; + gptw_pptr_q <= gptw_pptr_n; + ptw_stage_q <= ptw_stage_d; + pte[HYP_EXT] <= pte[HYP_EXT*2]; + end + end + end + +endmodule +/* verilator lint_on WIDTH */ diff --git a/core/mmu_sv32/cva6_shared_tlb_sv32.sv b/core/cva6_mmu/cva6_shared_tlb.sv similarity index 52% rename from core/mmu_sv32/cva6_shared_tlb_sv32.sv rename to core/cva6_mmu/cva6_shared_tlb.sv index 98e2a044a9..43157de299 100644 --- a/core/mmu_sv32/cva6_shared_tlb_sv32.sv +++ b/core/cva6_mmu/cva6_shared_tlb.sv @@ -1,4 +1,6 @@ // Copyright (c) 2023 Thales. +// Copyright (c) 2024, PlanV Technology +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 // Copyright and related rights are licensed under the Solderpad Hardware // License, Version 0.51 (the "License"); you may not use this file except in // compliance with the License. You may obtain a copy of the License at @@ -8,31 +10,33 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. // -// Author: Sebastien Jacq - Thales Research & Technology -// Date: 08/03/2023 +// Author: Angela Gonzalez PlanV Technology +// Date: 26/02/2024 // // Description: N-way associative shared TLB, it allows to reduce the number -// of ITLB and DTLB entries. -// +// of ITLB and DTLB entries. This module is an update of the +// shared TLB sv32 developed by Sebastien Jacq (Thales Research & Technology) +// to be used with sv32, sv39 and sv39x4. /* verilator lint_off WIDTH */ -module cva6_shared_tlb_sv32 - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, +module cva6_shared_tlb #( + parameter type pte_cva6_t = logic, + parameter type tlb_update_cva6_t = logic, parameter int SHARED_TLB_DEPTH = 64, parameter int SHARED_TLB_WAYS = 2, - parameter int ASID_WIDTH = 1 + parameter int unsigned HYP_EXT = 0, + parameter int ASID_WIDTH[HYP_EXT:0], //[vmid_width,asid_width] + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 ) ( - input logic clk_i, // Clock + input logic clk_i, // Clock input logic rst_ni, // Asynchronous reset active low - input logic flush_i, - - input logic enable_translation_i, // CSRs indicate to enable SV32 - input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores + input logic [HYP_EXT*2:0] flush_i, // Flush signal [g_stage,vs stage, normal translation signal] + input logic [1:0][HYP_EXT*2:0] v_st_enbl_i, // v_i,g-stage enabled, s-stage enabled - input logic [ASID_WIDTH-1:0] asid_i, + input logic [ASID_WIDTH[0]-1:0] dtlb_asid_i[HYP_EXT:0], //[vmid,vs_asid,asid] + input logic [ASID_WIDTH[0]-1:0] itlb_asid_i[HYP_EXT:0], //[vmid,vs_asid,asid] // from TLBs // did we miss? @@ -45,8 +49,8 @@ module cva6_shared_tlb_sv32 input logic [riscv::VLEN-1:0] dtlb_vaddr_i, // to TLBs, update logic - output tlb_update_sv32_t itlb_update_o, - output tlb_update_sv32_t dtlb_update_o, + output tlb_update_cva6_t itlb_update_o, + output tlb_update_cva6_t dtlb_update_o, // Performance counters output logic itlb_miss_o, @@ -59,7 +63,7 @@ module cva6_shared_tlb_sv32 output logic itlb_req_o, // Update shared TLB in case of miss - input tlb_update_sv32_t shared_tlb_update_i + input tlb_update_cva6_t shared_tlb_update_i ); @@ -72,10 +76,10 @@ module cva6_shared_tlb_sv32 endfunction typedef struct packed { - logic [8:0] asid; //9 bits wide - logic [9:0] vpn1; //10 bits wide - logic [9:0] vpn0; //10 bits wide - logic is_4M; + logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] asid; + logic [PT_LEVELS+HYP_EXT-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn; + logic [PT_LEVELS-2:0][HYP_EXT:0] is_page; + logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled } shared_tag_t; shared_tag_t shared_tag_wr; @@ -99,24 +103,30 @@ module cva6_shared_tlb_sv32 logic [ SHARED_TLB_WAYS-1:0] pte_wr_en; logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_wr_addr; - logic [$bits(riscv::pte_sv32_t)-1:0] pte_wr_data; + logic [ $bits(pte_cva6_t)-1:0] pte_wr_data [ HYP_EXT:0]; logic [ SHARED_TLB_WAYS-1:0] pte_rd_en; logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_rd_addr; - logic [$bits(riscv::pte_sv32_t)-1:0] pte_rd_data [SHARED_TLB_WAYS-1:0]; + logic [ $bits(pte_cva6_t)-1:0] pte_rd_data [SHARED_TLB_WAYS-1:0] [HYP_EXT:0]; logic [ SHARED_TLB_WAYS-1:0] pte_req; logic [ SHARED_TLB_WAYS-1:0] pte_we; logic [$clog2(SHARED_TLB_DEPTH)-1:0] pte_addr; - logic [9:0] vpn0_d, vpn1_d, vpn0_q, vpn1_q; + logic [PT_LEVELS+HYP_EXT-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn_d, vpn_q; + logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] vpn_match; + logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] page_match; + logic [SHARED_TLB_WAYS-1:0][PT_LEVELS-1:0] level_match; + + logic [SHARED_TLB_WAYS-1:0][HYP_EXT:0] match_asid; + logic [SHARED_TLB_WAYS-1:0] match_stage; - riscv::pte_sv32_t [SHARED_TLB_WAYS-1:0] pte; + pte_cva6_t [SHARED_TLB_WAYS-1:0][HYP_EXT:0] pte; logic [riscv::VLEN-1-12:0] itlb_vpn_q; logic [riscv::VLEN-1-12:0] dtlb_vpn_q; - logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_d; + logic [ASID_WIDTH[0]-1:0] tlb_update_asid_q[HYP_EXT:0], tlb_update_asid_d[HYP_EXT:0]; logic shared_tlb_access_q, shared_tlb_access_d; logic shared_tlb_hit_d; @@ -125,6 +135,8 @@ module cva6_shared_tlb_sv32 logic itlb_req_d, itlb_req_q; logic dtlb_req_d, dtlb_req_q; + int i_req_d, i_req_q; + // replacement strategy logic [SHARED_TLB_WAYS-1:0] way_valid; logic update_lfsr; // shift the LFSR @@ -140,14 +152,53 @@ module cva6_shared_tlb_sv32 assign itlb_req_o = itlb_req_q; + genvar i, x; + generate + for (i = 0; i < SHARED_TLB_WAYS; i++) begin : gen_match_tlb_ways + //identify page_match for all TLB Entries + + for (x = 0; x < PT_LEVELS; x++) begin : gen_match + assign page_match[i][x] = x==0 ? 1 :((HYP_EXT==0 || x==(PT_LEVELS-1)) ? // PAGE_MATCH CONTAINS THE MATCH INFORMATION FOR EACH TAG OF is_1G and is_2M in sv39x4. HIGHER LEVEL (Giga page), THEN THERE IS THE Mega page AND AT THE LOWER LEVEL IS ALWAYS 1 + &(shared_tag_rd[i].is_page[PT_LEVELS-1-x] | (~v_st_enbl_i[i_req_q][HYP_EXT:0])): + ((&v_st_enbl_i[i_req_q][HYP_EXT:0]) ? + ((shared_tag_rd[i].is_page[PT_LEVELS-1-x][0] && (shared_tag_rd[i].is_page[PT_LEVELS-2-x][HYP_EXT] || shared_tag_rd[i].is_page[PT_LEVELS-1-x][HYP_EXT])) + || (shared_tag_rd[i].is_page[PT_LEVELS-1-x][HYP_EXT] && (shared_tag_rd[i].is_page[PT_LEVELS-2-x][0] || shared_tag_rd[i].is_page[PT_LEVELS-1-x][0]))): + shared_tag_rd[i].is_page[PT_LEVELS-1-x][0] && v_st_enbl_i[i_req_q][0] || shared_tag_rd[i].is_page[PT_LEVELS-1-x][HYP_EXT] && v_st_enbl_i[i_req_q][HYP_EXT])); + + //identify if vpn matches at all PT levels for all TLB entries + assign vpn_match[i][x] = (HYP_EXT==1 && x==(PT_LEVELS-1) && ~v_st_enbl_i[i_req_q][0]) ? // + vpn_q[x] == shared_tag_rd[i].vpn[x] && vpn_q[x+1][(VPN_LEN%PT_LEVELS)-1:0] == shared_tag_rd[i].vpn[x+1][(VPN_LEN%PT_LEVELS)-1:0]: // + vpn_q[x] == shared_tag_rd[i].vpn[x]; + + //identify if there is a hit at each PT level for all TLB entries + assign level_match[i][x] = &vpn_match[i][PT_LEVELS-1:x] && page_match[i][x]; + + end + end + endgenerate + + genvar w; + generate + for (w = 0; w < PT_LEVELS; w++) begin + assign vpn_d[w] = ((|v_st_enbl_i[1][HYP_EXT:0]) && itlb_access_i && ~itlb_hit_i && ~dtlb_access_i) ? // + itlb_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(w+1))-1:12+((VPN_LEN/PT_LEVELS)*w)] : // + (((|v_st_enbl_i[0][HYP_EXT:0]) && dtlb_access_i && ~dtlb_hit_i) ? // + dtlb_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(w+1))-1:12+((VPN_LEN/PT_LEVELS)*w)] : vpn_q[w]); + end + endgenerate + + if (HYP_EXT == 1) //THIS UPDATES THE EXTRA BITS OF VPN IN SV39x4 + assign vpn_d[PT_LEVELS][(VPN_LEN%PT_LEVELS)-1:0] = ((|v_st_enbl_i[1][HYP_EXT:0]) && itlb_access_i && ~itlb_hit_i && ~dtlb_access_i) ? // + itlb_vaddr_i[VPN_LEN-1:VPN_LEN-(VPN_LEN%PT_LEVELS)] : // + (((|v_st_enbl_i[0][HYP_EXT:0]) && dtlb_access_i && ~dtlb_hit_i) ? // + dtlb_vaddr_i[VPN_LEN-1: VPN_LEN-(VPN_LEN%PT_LEVELS)] : vpn_q[PT_LEVELS][(VPN_LEN%PT_LEVELS)-1:0]); + /////////////////////////////////////////////////////// // tag comparison, hit generation /////////////////////////////////////////////////////// always_comb begin : itlb_dtlb_miss itlb_miss_o = 1'b0; dtlb_miss_o = 1'b0; - vpn0_d = vpn0_q; - vpn1_d = vpn1_q; tag_rd_en = '0; pte_rd_en = '0; @@ -162,42 +213,37 @@ module cva6_shared_tlb_sv32 tag_rd_addr = '0; pte_rd_addr = '0; + i_req_d = i_req_q; // if we got an ITLB miss - if (enable_translation_i & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin + if ((|v_st_enbl_i[1][HYP_EXT:0]) & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin tag_rd_en = '1; tag_rd_addr = itlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; pte_rd_en = '1; pte_rd_addr = itlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; - vpn0_d = itlb_vaddr_i[21:12]; - vpn1_d = itlb_vaddr_i[31:22]; - itlb_miss_o = 1'b1; itlb_req_d = 1'b1; + tlb_update_asid_d = itlb_asid_i; - tlb_update_asid_d = asid_i; - - shared_tlb_access_d = 1'b1; + shared_tlb_access_d = '1; shared_tlb_vaddr_d = itlb_vaddr_i; + i_req_d = 1; // we got an DTLB miss - end else if (en_ld_st_translation_i & dtlb_access_i & ~dtlb_hit_i) begin + end else if ((|v_st_enbl_i[0][HYP_EXT:0]) & dtlb_access_i & ~dtlb_hit_i) begin tag_rd_en = '1; tag_rd_addr = dtlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; pte_rd_en = '1; pte_rd_addr = dtlb_vaddr_i[12+:$clog2(SHARED_TLB_DEPTH)]; - vpn0_d = dtlb_vaddr_i[21:12]; - vpn1_d = dtlb_vaddr_i[31:22]; - dtlb_miss_o = 1'b1; dtlb_req_d = 1'b1; + tlb_update_asid_d = dtlb_asid_i; - tlb_update_asid_d = asid_i; - - shared_tlb_access_d = 1'b1; + shared_tlb_access_d = '1; shared_tlb_vaddr_d = dtlb_vaddr_i; + i_req_d = 0; end end //itlb_dtlb_miss @@ -205,23 +251,41 @@ module cva6_shared_tlb_sv32 shared_tlb_hit_d = 1'b0; dtlb_update_o = '0; itlb_update_o = '0; + //number of ways for (int unsigned i = 0; i < SHARED_TLB_WAYS; i++) begin - if (shared_tag_valid[i] && ((tlb_update_asid_q == shared_tag_rd[i].asid) || pte[i].g) && vpn1_q == shared_tag_rd[i].vpn1) begin - if (shared_tag_rd[i].is_4M || vpn0_q == shared_tag_rd[i].vpn0) begin + // first level match, this may be a giga page, check the ASID flags as well + // if the entry is associated to a global address, don't match the ASID (ASID is don't care) + match_asid[i][0] = (((tlb_update_asid_q[0][ASID_WIDTH[0]-1:0] == shared_tag_rd[i].asid[0][ASID_WIDTH[0]-1:0]) || pte[i][0].g) && v_st_enbl_i[i_req_q][0]) || !v_st_enbl_i[i_req_q][0]; + + if (HYP_EXT == 1) begin + match_asid[i][HYP_EXT] = (tlb_update_asid_q[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == shared_tag_rd[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] && v_st_enbl_i[i_req_q][HYP_EXT]) || !v_st_enbl_i[i_req_q][HYP_EXT]; + end + + // check if translation is a: S-Stage and G-Stage, S-Stage only or G-Stage only translation and virtualization mode is on/off + match_stage[i] = shared_tag_rd[i].v_st_enbl == v_st_enbl_i[i_req_q]; + + if (shared_tag_valid[i] && &match_asid[i] && match_stage[i]) begin + if (|level_match[i]) begin shared_tlb_hit_d = 1'b1; if (itlb_req_q) begin itlb_update_o.valid = 1'b1; itlb_update_o.vpn = itlb_vpn_q; - itlb_update_o.is_4M = shared_tag_rd[i].is_4M; - itlb_update_o.asid = tlb_update_asid_q; + itlb_update_o.is_page = shared_tag_rd[i].is_page; itlb_update_o.content = pte[i]; + itlb_update_o.v_st_enbl = shared_tag_rd[i].v_st_enbl; + for (int unsigned a = 0; a < HYP_EXT + 1; a++) begin + itlb_update_o.asid[a] = tlb_update_asid_q[a]; + end end else if (dtlb_req_q) begin dtlb_update_o.valid = 1'b1; dtlb_update_o.vpn = dtlb_vpn_q; - dtlb_update_o.is_4M = shared_tag_rd[i].is_4M; - dtlb_update_o.asid = tlb_update_asid_q; + dtlb_update_o.is_page = shared_tag_rd[i].is_page; dtlb_update_o.content = pte[i]; + dtlb_update_o.v_st_enbl = shared_tag_rd[i].v_st_enbl; + for (int unsigned a = 0; a < HYP_EXT + 1; a++) begin + dtlb_update_o.asid[a] = tlb_update_asid_q[a]; + end end end end @@ -233,14 +297,14 @@ module cva6_shared_tlb_sv32 if (~rst_ni) begin itlb_vpn_q <= '0; dtlb_vpn_q <= '0; - tlb_update_asid_q <= '0; + tlb_update_asid_q <= '{default: 0}; shared_tlb_access_q <= '0; shared_tlb_vaddr_q <= '0; shared_tag_valid_q <= '0; - vpn0_q <= '0; - vpn1_q <= '0; + vpn_q <= 0; itlb_req_q <= '0; dtlb_req_q <= '0; + i_req_q <= 0; shared_tag_valid <= '0; end else begin itlb_vpn_q <= itlb_vaddr_i[riscv::SV-1:12]; @@ -249,10 +313,10 @@ module cva6_shared_tlb_sv32 shared_tlb_access_q <= shared_tlb_access_d; shared_tlb_vaddr_q <= shared_tlb_vaddr_d; shared_tag_valid_q <= shared_tag_valid_d; - vpn0_q <= vpn0_d; - vpn1_q <= vpn1_d; + vpn_q <= vpn_d; itlb_req_q <= itlb_req_d; dtlb_req_q <= dtlb_req_d; + i_req_q <= i_req_d; shared_tag_valid <= shared_tag_valid_q[tag_rd_addr]; end end @@ -265,7 +329,7 @@ module cva6_shared_tlb_sv32 tag_wr_en = '0; pte_wr_en = '0; - if (flush_i) begin + if (|flush_i) begin shared_tag_valid_d = '0; end else if (shared_tlb_update_i.valid) begin for (int unsigned i = 0; i < SHARED_TLB_WAYS; i++) begin @@ -279,15 +343,32 @@ module cva6_shared_tlb_sv32 end //update_flush assign shared_tag_wr.asid = shared_tlb_update_i.asid; - assign shared_tag_wr.vpn1 = shared_tlb_update_i.vpn[19:10]; - assign shared_tag_wr.vpn0 = shared_tlb_update_i.vpn[9:0]; - assign shared_tag_wr.is_4M = shared_tlb_update_i.is_4M; + assign shared_tag_wr.is_page = shared_tlb_update_i.is_page; + assign shared_tag_wr.v_st_enbl = v_st_enbl_i[i_req_q]; + + genvar z; + generate + for (z = 0; z < PT_LEVELS; z++) begin : gen_shared_tag + assign shared_tag_wr.vpn[z] = shared_tlb_update_i.vpn[((VPN_LEN/PT_LEVELS)*(z+1))-1:((VPN_LEN/PT_LEVELS)*z)]; + end + if (HYP_EXT == 1) begin : gen_shared_tag_hyp + //THIS UPDATES THE EXTRA BITS OF VPN IN SV39x4 + assign shared_tag_wr.vpn[PT_LEVELS][(VPN_LEN%PT_LEVELS)-1:0] = shared_tlb_update_i.vpn[VPN_LEN-1: VPN_LEN-(VPN_LEN%PT_LEVELS)]; + end + endgenerate + assign tag_wr_addr = shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]; assign tag_wr_data = shared_tag_wr; assign pte_wr_addr = shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]; - assign pte_wr_data = shared_tlb_update_i.content; + + genvar h; + generate + for (h = 0; h < HYP_EXT + 1; h++) begin : gen_pte_wr_data + assign pte_wr_data[h] = shared_tlb_update_i.content[h]; + end + endgenerate assign way_valid = shared_tag_valid_q[shared_tlb_update_i.vpn[$clog2(SHARED_TLB_DEPTH)-1:0]]; assign repl_way = (all_ways_valid) ? rnd_way : inv_way; @@ -344,23 +425,25 @@ module cva6_shared_tlb_sv32 assign shared_tag_rd[i] = shared_tag_t'(tag_rd_data[i]); - // PTE RAM - sram #( - .DATA_WIDTH($bits(riscv::pte_sv32_t)), - .NUM_WORDS (SHARED_TLB_DEPTH) - ) pte_sram ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .req_i (pte_req[i]), - .we_i (pte_we[i]), - .addr_i (pte_addr), - .wuser_i('0), - .wdata_i(pte_wr_data), - .be_i ('1), - .ruser_o(), - .rdata_o(pte_rd_data[i]) - ); - assign pte[i] = riscv::pte_sv32_t'(pte_rd_data[i]); + for (genvar a = 0; a < HYP_EXT + 1; a++) begin : g_content_sram + // PTE RAM + sram #( + .DATA_WIDTH($bits(pte_cva6_t)), + .NUM_WORDS (SHARED_TLB_DEPTH) + ) pte_sram ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .req_i (pte_req[i]), + .we_i (pte_we[i]), + .addr_i (pte_addr), + .wuser_i('0), + .wdata_i(pte_wr_data[a]), + .be_i ('1), + .ruser_o(), + .rdata_o(pte_rd_data[i][a]) + ); + assign pte[i][a] = pte_cva6_t'(pte_rd_data[i][a]); + end end endmodule diff --git a/core/cva6_mmu/cva6_tlb.sv b/core/cva6_mmu/cva6_tlb.sv new file mode 100644 index 0000000000..a64c0bcc4e --- /dev/null +++ b/core/cva6_mmu/cva6_tlb.sv @@ -0,0 +1,459 @@ +// Copyright (c) 2018 ETH Zurich and University of Bologna. +// Copyright (c) 2021 Thales. +// Copyright (c) 2022 Bruno Sá and Zero-Day Labs. +// Copyright (c) 2024 PlanV Technology +// SPDX-License-Identifier: Apache-2.0 WITH SHL-2.1 +// Copyright and related rights are licensed under the Solderpad Hardware +// License, Version 0.51 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law +// or agreed to in writing, software, hardware and materials distributed under +// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +// +// Author: Angela Gonzalez PlanV Technology +// Date: 26/02/2024 +// +// Description: Translation Lookaside Buffer, parameterizable to Sv32 or Sv39 , +// or sv39x4 fully set-associative +// This module is an merge of the Sv32 TLB developed by Sebastien +// Jacq (Thales Research & Technology), the Sv39 TLB developed +// by Florian Zaruba and David Schaffenrath and the Sv39x4 by Bruno Sá. + +module cva6_tlb + import ariane_pkg::*; +#( + parameter type pte_cva6_t = logic, + parameter type tlb_update_cva6_t = logic, + parameter int unsigned TLB_ENTRIES = 4, + parameter int unsigned HYP_EXT = 0, + parameter int ASID_WIDTH[HYP_EXT:0], //[vmid_width,asid_width] + parameter int unsigned VPN_LEN = 1, + parameter int unsigned PT_LEVELS = 1 +) ( + input logic clk_i, // Clock + input logic rst_ni, // Asynchronous reset active low + input logic [HYP_EXT*2:0] flush_i, // Flush signal [g_stage,vs stage, normal translation signal] + input logic [HYP_EXT*2:0] v_st_enbl_i, // v_i,g-stage enabled, s-stage enabled + // Update TLB + input tlb_update_cva6_t update_i, + // Lookup signals + input logic lu_access_i, + input logic [ASID_WIDTH[0]-1:0] lu_asid_i[HYP_EXT:0], //[lu_vmid,lu_asid] + input logic [riscv::VLEN-1:0] lu_vaddr_i, + output logic [riscv::GPLEN-1:0] lu_gpaddr_o, + output pte_cva6_t [HYP_EXT:0] lu_content_o, + input logic [ASID_WIDTH[0]-1:0] asid_to_be_flushed_i[HYP_EXT:0], //[vmid,asid] + input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i[HYP_EXT:0], // [gpaddr,vaddr] + output logic [PT_LEVELS-2:0] lu_is_page_o, + output logic lu_hit_o +); + + // computes the paddr based on the page size, ppn and offset + function automatic logic [(riscv::GPLEN-1):0] make_gpaddr( + input logic s_st_enbl, input logic is_1G, input logic is_2M, + input logic [(riscv::VLEN-1):0] vaddr, input riscv::pte_t pte); + logic [(riscv::GPLEN-1):0] gpaddr; + if (s_st_enbl) begin + gpaddr = {pte.ppn[(riscv::GPPNW-1):0], vaddr[11:0]}; + // Giga page + if (is_1G) gpaddr[29:12] = vaddr[29:12]; + // Mega page + if (is_2M) gpaddr[20:12] = vaddr[20:12]; + end else begin + gpaddr = vaddr[(riscv::GPLEN-1):0]; + end + return gpaddr; + endfunction : make_gpaddr + + // computes the final gppn based on the guest physical address + function automatic logic [(riscv::GPPNW-1):0] make_gppn(input logic s_st_enbl, input logic is_1G, + input logic is_2M, input logic [28:0] vpn, + input riscv::pte_t pte); + logic [(riscv::GPPNW-1):0] gppn; + if (s_st_enbl) begin + gppn = pte.ppn[(riscv::GPPNW-1):0]; + if (is_2M) gppn[8:0] = vpn[8:0]; + if (is_1G) gppn[17:0] = vpn[17:0]; + end else begin + gppn = vpn; + end + return gppn; + endfunction : make_gppn + + // SV39 defines three levels of page tables + struct packed { + logic [HYP_EXT:0][ASID_WIDTH[0]-1:0] asid; + logic [PT_LEVELS+HYP_EXT-1:0][(VPN_LEN/PT_LEVELS)-1:0] vpn; + logic [PT_LEVELS-2:0][HYP_EXT:0] is_page; + logic [HYP_EXT*2:0] v_st_enbl; // v_i,g-stage enabled, s-stage enabled + logic valid; + } [TLB_ENTRIES-1:0] + tags_q, tags_n; + + pte_cva6_t [TLB_ENTRIES-1:0][HYP_EXT:0] content_q, content_n; + + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] vpn_match; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] level_match; + logic [TLB_ENTRIES-1:0][HYP_EXT:0][PT_LEVELS-1:0] vaddr_vpn_match; + logic [TLB_ENTRIES-1:0][HYP_EXT:0][PT_LEVELS-1:0] vaddr_level_match; + logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic + logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy + logic [TLB_ENTRIES-1:0][HYP_EXT:0] match_asid; + logic [TLB_ENTRIES-1:0][PT_LEVELS-1:0] page_match; + logic [TLB_ENTRIES-1:0][HYP_EXT:0][PT_LEVELS-1:0] vpage_match; + logic [TLB_ENTRIES-1:0][PT_LEVELS-2:0] is_page_o; + logic [TLB_ENTRIES-1:0] match_stage; + pte_cva6_t g_content; + logic [TLB_ENTRIES-1:0][(riscv::GPPNW-1):0] gppn; + logic [HYP_EXT*2:0] v_st_enbl; + + assign v_st_enbl = (HYP_EXT == 1) ? v_st_enbl_i : '1; + //------------- + // Translation + //------------- + + genvar i, x, z, w; + generate + for (i = 0; i < TLB_ENTRIES; i++) begin + for (x = 0; x < PT_LEVELS; x++) begin + //identify page_match for all TLB Entries + assign page_match[i][x] = x==0 ? 1 :((HYP_EXT==0 || x==(PT_LEVELS-1)) ? // PAGE_MATCH CONTAINS THE MATCH INFORMATION FOR EACH TAG OF is_1G and is_2M in sv39x4. HIGHER LEVEL (Giga page), THEN THERE IS THE Mega page AND AT THE LOWER LEVEL IS ALWAYS 1 + &(tags_q[i].is_page[PT_LEVELS-1-x] | (~v_st_enbl[HYP_EXT:0])): + ((&v_st_enbl[HYP_EXT:0]) ? + ((tags_q[i].is_page[PT_LEVELS-1-x][0] && (tags_q[i].is_page[PT_LEVELS-2-x][HYP_EXT] || tags_q[i].is_page[PT_LEVELS-1-x][HYP_EXT])) + || (tags_q[i].is_page[PT_LEVELS-1-x][HYP_EXT] && (tags_q[i].is_page[PT_LEVELS-2-x][0] || tags_q[i].is_page[PT_LEVELS-1-x][0]))): + tags_q[i].is_page[PT_LEVELS-1-x][0] && v_st_enbl[0] || tags_q[i].is_page[PT_LEVELS-1-x][HYP_EXT] && v_st_enbl[HYP_EXT])); + + //identify if vpn matches at all PT levels for all TLB entries + assign vpn_match[i][x] = (HYP_EXT == 1 && x == (PT_LEVELS - 1) && ~v_st_enbl[0]) ? // + lu_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x] && lu_vaddr_i[12+VPN_LEN-1: 12+VPN_LEN-(VPN_LEN%PT_LEVELS)] == tags_q[i].vpn[x+1][(VPN_LEN%PT_LEVELS)-1:0]: // + lu_vaddr_i[12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x]; + + //identify if there is a hit at each PT level for all TLB entries + assign level_match[i][x] = &vpn_match[i][PT_LEVELS-1:x] && page_match[i][x]; + + //identify vpage_match for all TLB Entries and vaddr_level match (if there is a hit at each PT level for all TLB entries on the vaddr) + for (z = 0; z < HYP_EXT + 1; z++) begin + assign vpage_match[i][z][x] = x == 0 ? 1 : tags_q[i].is_page[PT_LEVELS-1-x][z]; + assign vaddr_level_match[i][z][x]= &vaddr_vpn_match[i][z][PT_LEVELS-1:x] && vpage_match[i][z][x]; + + end + //identify if virtual address vpn matches at all PT levels for all TLB entries + assign vaddr_vpn_match[i][0][x] = vaddr_to_be_flushed_i[0][12+((VPN_LEN/PT_LEVELS)*(x+1))-1:12+((VPN_LEN/PT_LEVELS)*x)] == tags_q[i].vpn[x]; + + end + + + + if (HYP_EXT == 1) begin + //identify if GPADDR matches the GPPN + assign vaddr_vpn_match[i][HYP_EXT][0] = (vaddr_to_be_flushed_i[HYP_EXT][20:12] == gppn[i][8:0]); + assign vaddr_vpn_match[i][HYP_EXT][HYP_EXT] = (vaddr_to_be_flushed_i[HYP_EXT][29:21] == gppn[i][17:9]); + assign vaddr_vpn_match[i][HYP_EXT][HYP_EXT*2] = (vaddr_to_be_flushed_i[HYP_EXT][30+riscv::GPPN2:30] == gppn[i][18+riscv::GPPN2:18]); + + end + + for (w = 0; w < PT_LEVELS - 1; w++) begin + assign is_page_o[i][w] = page_match[i][PT_LEVELS - 1 - w]; //THIS REORGANIZES THE PAGES TO MATCH THE OUTPUT STRUCTURE (2M,1G) + end + end + endgenerate + + always_comb begin : translation + + // default assignment + lu_hit = '{default: 0}; + lu_hit_o = 1'b0; + lu_content_o = '{default: 0}; + lu_is_page_o = '{default: 0}; + match_asid = '{default: 0}; + match_stage = '{default: 0}; + g_content = '{default: 0}; + lu_gpaddr_o = '{default: 0}; + + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + // first level match, this may be a giga page, check the ASID flags as well + // if the entry is associated to a global address, don't match the ASID (ASID is don't care) + match_asid[i][0] = (((lu_asid_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0]) || content_q[i][0].g) && v_st_enbl[0]) || !v_st_enbl[0]; + + if (HYP_EXT == 1) begin + match_asid[i][HYP_EXT] = (lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] && v_st_enbl[HYP_EXT]) || !v_st_enbl[HYP_EXT]; + end + + // check if translation is a: S-Stage and G-Stage, S-Stage only or G-Stage only translation and virtualization mode is on/off + match_stage[i] = tags_q[i].v_st_enbl == v_st_enbl; + + if (tags_q[i].valid && &match_asid[i] && match_stage[i]) begin + + if (HYP_EXT == 1 && vpn_match[i][HYP_EXT*2]) + lu_gpaddr_o = make_gpaddr( + v_st_enbl[0], + tags_q[i].is_page[0][0], + tags_q[i].is_page[1][0], + lu_vaddr_i, + content_q[i][0] + ); + + if (|level_match[i]) begin + lu_is_page_o = is_page_o[i]; + lu_content_o = content_q[i]; + lu_hit_o = 1'b1; + lu_hit[i] = 1'b1; + + if (HYP_EXT == 1) begin + // Compute G-Stage PPN based on the gpaddr + g_content = content_q[i][HYP_EXT]; + if (tags_q[i].is_page[HYP_EXT][HYP_EXT]) g_content.ppn[8:0] = lu_gpaddr_o[20:12]; + if (tags_q[i].is_page[0][HYP_EXT]) g_content.ppn[17:0] = lu_gpaddr_o[29:12]; + // Output G-stage and S-stage content + lu_content_o[HYP_EXT] = level_match[i][PT_LEVELS-1] ? content_q[i][HYP_EXT] : g_content; + end + end + end + end + end + + + + logic [HYP_EXT:0]asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high + logic [HYP_EXT:0] vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high + + localparam int VADDR_WIDTH[1:0] = {riscv::GPLEN, riscv::VLEN}; + + genvar a; + generate + for (a = 0; a < HYP_EXT + 1; a++) begin + assign asid_to_be_flushed_is0[a] = ~(|asid_to_be_flushed_i[a][ASID_WIDTH[a]-1:0]); + assign vaddr_to_be_flushed_is0[a] = ~(|vaddr_to_be_flushed_i[a][VADDR_WIDTH[a]-1:0]); + end + endgenerate + + // ------------------ + // Update and Flush + // ------------------ + always_comb begin : update_flush + tags_n = tags_q; + content_n = content_q; + + for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin + + + if (HYP_EXT == 1) begin + gppn[i] = make_gppn( + tags_q[i].v_st_enbl[0], + tags_q[i].is_page[0][0], + tags_q[i].is_page[1][0], + { + tags_q[i].vpn[3][(VPN_LEN%PT_LEVELS)-1:0], + tags_q[i].vpn[2], + tags_q[i].vpn[1], + tags_q[i].vpn[0] + }, + content_q[i][0] + ); + end + + + if (flush_i[0]) begin + if (!tags_q[i].v_st_enbl[HYP_EXT*2] || HYP_EXT == 0) begin + // invalidate logic + // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) + if (asid_to_be_flushed_is0[0] && vaddr_to_be_flushed_is0[0]) tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0[0] && (|vaddr_level_match[i][0] ) && (~vaddr_to_be_flushed_is0[0])) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) + else if ((!content_q[i][0].g) && (|vaddr_level_match[i][0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] ) && (!vaddr_to_be_flushed_is0[0]) && (!asid_to_be_flushed_is0[0])) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) + else if ((!content_q[i][0].g) && (vaddr_to_be_flushed_is0[0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] ) && (!asid_to_be_flushed_is0[0])) + tags_n[i].valid = 1'b0; + end + end else if (flush_i[HYP_EXT] && HYP_EXT == 1) begin + if (tags_q[i].v_st_enbl[HYP_EXT*2] && tags_q[i].v_st_enbl[0]) begin + // invalidate logic + // flush everything if current VMID matches and ASID is 0 and vaddr is 0 ("SFENCE.VMA/HFENCE.VVMA x0 x0" case) + if (asid_to_be_flushed_is0[0] && vaddr_to_be_flushed_is0[0] && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) + tags_n[i].valid = 1'b0; + // flush vaddr in all addressing space if current VMID matches ("SFENCE.VMA/HFENCE.VVMA vaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0[0] && (|vaddr_level_match[i][0]) && (~vaddr_to_be_flushed_is0[0]) && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[1][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global and asid and vaddr and current VMID matches with the entry to be flushed ("SFENCE.VMA/HFENCE.VVMA vaddr asid" case) + else if ((!content_q[i][0].g) && (|vaddr_level_match[i][0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) && (!vaddr_to_be_flushed_is0[0]) && (!asid_to_be_flushed_is0[0])) + tags_n[i].valid = 1'b0; + // the entry is flushed if it's not global, and the asid and the current VMID matches and vaddr is 0. ("SFENCE.VMA/HFENCE.VVMA 0 asid" case) + else if ((!content_q[i][0].g) && (vaddr_to_be_flushed_is0[0]) && (asid_to_be_flushed_i[0][ASID_WIDTH[0]-1:0] == tags_q[i].asid[0][ASID_WIDTH[0]-1:0] && ((tags_q[i].v_st_enbl[HYP_EXT] && lu_asid_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) || !tags_q[i].v_st_enbl[HYP_EXT])) && (!asid_to_be_flushed_is0[0])) + tags_n[i].valid = 1'b0; + end + end else if (flush_i[HYP_EXT*2] && HYP_EXT == 1) begin + if (tags_q[i].v_st_enbl[HYP_EXT]) begin + // invalidate logic + // flush everything if vmid is 0 and addr is 0 ("HFENCE.GVMA x0 x0" case) + if (asid_to_be_flushed_is0[HYP_EXT] && vaddr_to_be_flushed_is0[HYP_EXT]) + tags_n[i].valid = 1'b0; + // flush gpaddr in all addressing space ("HFENCE.GVMA gpaddr x0" case), it should happen only for leaf pages + else if (asid_to_be_flushed_is0[HYP_EXT] && (|vaddr_level_match[i][HYP_EXT] ) && (~vaddr_to_be_flushed_is0[HYP_EXT])) + tags_n[i].valid = 1'b0; + // the entry vmid and gpaddr both matches with the entry to be flushed ("HFENCE.GVMA gpaddr vmid" case) + else if ((|vaddr_level_match[i][HYP_EXT]) && (asid_to_be_flushed_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) && (~vaddr_to_be_flushed_is0[HYP_EXT]) && (~asid_to_be_flushed_is0[HYP_EXT])) + tags_n[i].valid = 1'b0; + // the entry is flushed if the vmid matches and gpaddr is 0. ("HFENCE.GVMA 0 vmid" case) + else if ((vaddr_to_be_flushed_is0[HYP_EXT]) && (asid_to_be_flushed_i[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0] == tags_q[i].asid[HYP_EXT][ASID_WIDTH[HYP_EXT]-1:0]) && (!asid_to_be_flushed_is0[HYP_EXT])) + tags_n[i].valid = 1'b0; + end + // normal replacement + end else if (update_i.valid & replace_en[i] && !lu_hit_o) begin + //update tag + tags_n[i] = { + update_i.asid, + ((PT_LEVELS + HYP_EXT) * (VPN_LEN / PT_LEVELS))'(update_i.vpn), + update_i.is_page, + update_i.v_st_enbl, + 1'b1 + }; + // update content as well + content_n[i] = update_i.content; + end + end + end + + // ----------------------------------------------- + // PLRU - Pseudo Least Recently Used Replacement + // ----------------------------------------------- + logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; + always_comb begin : plru_replacement + plru_tree_n = plru_tree_q; + // The PLRU-tree indexing: + // lvl0 0 + // / \ +// / \ +// lvl1 1 2 +// / \ / \ +// lvl2 3 4 5 6 +// / \ /\/\ /\ +// ... ... ... ... +// Just predefine which nodes will be set/cleared +// E.g. for a TLB with 8 entries, the for-loop is semantically +// equivalent to the following pseudo-code: +// unique case (1'b1) +// lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; +// lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; +// lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; +// lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; +// lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; +// lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; +// lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; +// lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; +// default: begin /* No hit */ end +// endcase +for ( + int unsigned i = 0; i < TLB_ENTRIES; i++ + ) begin + automatic int unsigned idx_base, shift, new_index; + // we got a hit so update the pointer as it was least recently used + if (lu_hit[i] & lu_access_i) begin + // Set the nodes to the values we would expect + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + // to circumvent the 32 bit integer arithmetic assignment + new_index = ~((i >> (shift - 1)) & 32'b1); + plru_tree_n[idx_base+(i>>shift)] = new_index[0]; + end + end + end + // Decode tree to write enable signals + // Next for-loop basically creates the following logic for e.g. an 8 entry + // TLB (note: pseudo-code obviously): + // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} + // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} + // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} + // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} + // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} + // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} + // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} + // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} + // For each entry traverse the tree. If every tree-node matches, + // the corresponding bit of the entry's index, this is + // the next entry to replace. + for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin + automatic logic en; + automatic int unsigned idx_base, shift, new_index; + en = 1'b1; + for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin + idx_base = $unsigned((2 ** lvl) - 1); + // lvl0 <=> MSB, lvl1 <=> MSB-1, ... + shift = $clog2(TLB_ENTRIES) - lvl; + + // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); + new_index = (i >> (shift - 1)) & 32'b1; + if (new_index[0]) begin + en &= plru_tree_q[idx_base+(i>>shift)]; + end else begin + en &= ~plru_tree_q[idx_base+(i>>shift)]; + end + end + replace_en[i] = en; + end + end + + // sequential process + always_ff @(posedge clk_i or negedge rst_ni) begin + if (~rst_ni) begin + tags_q <= '{default: 0}; + content_q <= '{default: 0}; + plru_tree_q <= '{default: 0}; + end else begin + tags_q <= tags_n; + content_q <= content_n; + plru_tree_q <= plru_tree_n; + end + end + //-------------- + // Sanity checks + //-------------- + + //pragma translate_off +`ifndef VERILATOR + + initial begin : p_assertions + assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) + else begin + $error("TLB size must be a multiple of 2 and greater than 1"); + $stop(); + end + assert (ASID_WIDTH[0] >= 1) + else begin + $error("ASID width must be at least 1"); + $stop(); + end + end + + // Just for checking + function int countSetBits(logic [TLB_ENTRIES-1:0] vector); + automatic int count = 0; + foreach (vector[idx]) begin + count += vector[idx]; + end + return count; + endfunction + + assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) + else begin + $error("More then one hit in TLB!"); + $stop(); + end + assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) + else begin + $error("More then one TLB entry selected for next replace!"); + $stop(); + end + +`endif + //pragma translate_on + +endmodule diff --git a/core/include/cv64a6_imafdc_sv39_config_pkg.sv b/core/include/cv64a6_imafdc_sv39_config_pkg.sv index 29c5499114..2f648619b0 100644 --- a/core/include/cv64a6_imafdc_sv39_config_pkg.sv +++ b/core/include/cv64a6_imafdc_sv39_config_pkg.sv @@ -55,8 +55,8 @@ package cva6_config_pkg; localparam CVA6ConfigNrStorePipeRegs = 0; localparam CVA6ConfigNrLoadBufEntries = 2; - localparam CVA6ConfigInstrTlbEntries = 16; - localparam CVA6ConfigDataTlbEntries = 16; + localparam CVA6ConfigInstrTlbEntries = 4; + localparam CVA6ConfigDataTlbEntries = 4; localparam CVA6ConfigRASDepth = 2; localparam CVA6ConfigBTBEntries = 32; diff --git a/core/include/riscv_pkg.sv b/core/include/riscv_pkg.sv index 18ae2cfc23..fb4fcf25b8 100644 --- a/core/include/riscv_pkg.sv +++ b/core/include/riscv_pkg.sv @@ -40,6 +40,10 @@ package riscv; localparam VLEN = (XLEN == 32) ? 32 : 64; // virtual address length localparam PLEN = (XLEN == 32) ? 34 : 56; // physical address length + localparam GPLEN = (XLEN == 32) ? 34 : 41; + localparam GPPNW = (XLEN == 32) ? 22 : 29; + localparam GPPN2 = (XLEN == 32) ? riscv::VLEN - 33 : 10; + localparam IS_XLEN32 = (XLEN == 32) ? 1'b1 : 1'b0; localparam IS_XLEN64 = (XLEN == 32) ? 1'b0 : 1'b1; localparam ModeW = (XLEN == 32) ? 1 : 4; @@ -334,10 +338,15 @@ package riscv; localparam logic [XLEN-1:0] ST_ACCESS_FAULT = 7; // Illegal access as governed by PMPs and PMAs localparam logic [XLEN-1:0] ENV_CALL_UMODE = 8; // environment call from user mode localparam logic [XLEN-1:0] ENV_CALL_SMODE = 9; // environment call from supervisor mode + localparam logic [XLEN-1:0] ENV_CALL_VSMODE = 10; // environment call from virtual supervisor mode localparam logic [XLEN-1:0] ENV_CALL_MMODE = 11; // environment call from machine mode localparam logic [XLEN-1:0] INSTR_PAGE_FAULT = 12; // Instruction page fault localparam logic [XLEN-1:0] LOAD_PAGE_FAULT = 13; // Load page fault localparam logic [XLEN-1:0] STORE_PAGE_FAULT = 15; // Store page fault + localparam logic [XLEN-1:0] INSTR_GUEST_PAGE_FAULT = 20; // Instruction guest-page fault + localparam logic [XLEN-1:0] LOAD_GUEST_PAGE_FAULT = 21; // Load guest-page fault + localparam logic [XLEN-1:0] VIRTUAL_INSTRUCTION = 22; // virtual instruction + localparam logic [XLEN-1:0] STORE_GUEST_PAGE_FAULT = 23; // Store guest-page fault localparam logic [XLEN-1:0] DEBUG_REQUEST = 24; // Debug request localparam int unsigned IRQ_S_SOFT = 1; @@ -361,6 +370,14 @@ package riscv; localparam logic [XLEN-1:0] S_EXT_INTERRUPT = (1 << (XLEN - 1)) | XLEN'(IRQ_S_EXT); localparam logic [XLEN-1:0] M_EXT_INTERRUPT = (1 << (XLEN - 1)) | XLEN'(IRQ_M_EXT); + // ---------------------- + // PseudoInstructions Codes + // ---------------------- + localparam logic [XLEN-1:0] READ_32_PSEUDOINSTRUCTION = 32'h00002000; + localparam logic [XLEN-1:0] WRITE_32_PSEUDOINSTRUCTION = 32'h00002020; + localparam logic [XLEN-1:0] READ_64_PSEUDOINSTRUCTION = 64'h00003000; + localparam logic [XLEN-1:0] WRITE_64_PSEUDOINSTRUCTION = 64'h00003020; + // ----- // CSRs // ----- diff --git a/core/load_store_unit.sv b/core/load_store_unit.sv index cb899c6c05..66a74c7437 100644 --- a/core/load_store_unit.sv +++ b/core/load_store_unit.sv @@ -175,63 +175,65 @@ module load_store_unit // ------------------- // MMU e.g.: TLBs/PTW // ------------------- - if (MMU_PRESENT && (riscv::XLEN == 64)) begin : gen_mmu_sv39 - mmu #( - .CVA6Cfg (CVA6Cfg), - .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), - .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) - ) i_cva6_mmu ( - // misaligned bypass - .misaligned_ex_i(misaligned_exception), - .lsu_is_store_i (st_translation_req), - .lsu_req_i (translation_req), - .lsu_vaddr_i (mmu_vaddr), - .lsu_valid_o (translation_valid), - .lsu_paddr_o (mmu_paddr), - .lsu_exception_o(mmu_exception), - .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request - .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request - // connecting PTW to D$ IF - .req_port_i (dcache_req_ports_i[0]), - .req_port_o (dcache_req_ports_o[0]), - // icache address translation requests - .icache_areq_i (icache_areq_i), - .asid_to_be_flushed_i, - .vaddr_to_be_flushed_i, - .icache_areq_o (icache_areq_o), - .pmpcfg_i, - .pmpaddr_i, - .* - ); - end else if (MMU_PRESENT && (riscv::XLEN == 32)) begin : gen_mmu_sv32 - cva6_mmu_sv32 #( + if (MMU_PRESENT) begin : gen_mmu + + localparam HYP_EXT = 0; //CVA6Cfg.CVA6ConfigHExtEn + localparam VPN_LEN = (riscv::XLEN == 64) ? (HYP_EXT ? 29 : 27) : 20; + localparam PT_LEVELS = (riscv::XLEN == 64) ? 3 : 2; + localparam int mmu_ASID_WIDTH[HYP_EXT:0] = {ASID_WIDTH}; + + + cva6_mmu #( .CVA6Cfg (CVA6Cfg), .INSTR_TLB_ENTRIES(ariane_pkg::INSTR_TLB_ENTRIES), .DATA_TLB_ENTRIES (ariane_pkg::DATA_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) + .HYP_EXT (HYP_EXT), + .ASID_WIDTH (mmu_ASID_WIDTH), + .VPN_LEN (VPN_LEN), + .PT_LEVELS (PT_LEVELS) ) i_cva6_mmu ( + .clk_i (clk_i), + .rst_ni (rst_ni), + .flush_i (flush_i), + .enable_translation_i ({enable_translation_i}), + .en_ld_st_translation_i({en_ld_st_translation_i}), + .icache_areq_i (icache_areq_i), + .icache_areq_o (icache_areq_o), // misaligned bypass - .misaligned_ex_i(misaligned_exception), - .lsu_is_store_i (st_translation_req), - .lsu_req_i (translation_req), - .lsu_vaddr_i (mmu_vaddr), + .misaligned_ex_i (misaligned_exception), + .lsu_req_i (translation_req), + .lsu_vaddr_i (mmu_vaddr), + .lsu_tinst_i (0), + .lsu_is_store_i (st_translation_req), + .csr_hs_ld_st_inst_o (), + .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request + .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request + .lsu_valid_o (translation_valid), .lsu_paddr_o (mmu_paddr), .lsu_exception_o(mmu_exception), - .lsu_dtlb_hit_o (dtlb_hit), // send in the same cycle as the request - .lsu_dtlb_ppn_o (dtlb_ppn), // send in the same cycle as the request - // connecting PTW to D$ IF - .req_port_i (dcache_req_ports_i[0]), - .req_port_o (dcache_req_ports_o[0]), - // icache address translation requests - .icache_areq_i (icache_areq_i), - .asid_to_be_flushed_i, - .vaddr_to_be_flushed_i, - .icache_areq_o (icache_areq_o), + + .priv_lvl_i (priv_lvl_i), + .ld_st_priv_lvl_i(ld_st_priv_lvl_i), + + .sum_i ({sum_i}), + .mxr_i ({mxr_i}), + .hlvx_inst_i (0), + .hs_ld_st_inst_i(0), + + .satp_ppn_i ({satp_ppn_i}), + .asid_i ({asid_i}), + .asid_to_be_flushed_i ({asid_to_be_flushed_i}), + .vaddr_to_be_flushed_i({vaddr_to_be_flushed_i}), + .flush_tlb_i ({flush_tlb_i}), + + .itlb_miss_o(itlb_miss_o), + .dtlb_miss_o(dtlb_miss_o), + + .req_port_i(dcache_req_ports_i[0]), + .req_port_o(dcache_req_ports_o[0]), .pmpcfg_i, - .pmpaddr_i, - .* + .pmpaddr_i ); end else begin : gen_no_mmu diff --git a/core/mmu_sv32/cva6_mmu_sv32.sv b/core/mmu_sv32/cva6_mmu_sv32.sv deleted file mode 100644 index 9c98793da4..0000000000 --- a/core/mmu_sv32/cva6_mmu_sv32.sv +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright (c) 2021 Thales. -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Sebastien Jacq Thales Research & Technology -// Date: 17/07/2021 -// -// Additional contributions by: -// Sebastien Jacq - sjthales on github.com -// -// Description: Memory Management Unit for CV32A6, contains TLB and -// address translation unit. Sv32 as defined in RISC-V -// privilege specification 1.11-WIP. -// This module is an adaptation of the MMU Sv39 developed -// by Florian Zaruba to the Sv32 standard. -// -// =========================================================================== // -// Revisions : -// Date Version Author Description -// 2020-02-17 0.1 S.Jacq MMU Sv32 for CV32A6 -// =========================================================================== // - -module cva6_mmu_sv32 - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int unsigned INSTR_TLB_ENTRIES = 2, - parameter int unsigned DATA_TLB_ENTRIES = 2, - parameter int unsigned ASID_WIDTH = 1 -) ( - input logic clk_i, - input logic rst_ni, - input logic flush_i, - input logic enable_translation_i, - input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores - // IF interface - input icache_arsp_t icache_areq_i, - output icache_areq_t icache_areq_o, - // LSU interface - // this is a more minimalistic interface because the actual addressing logic is handled - // in the LSU as we distinguish load and stores, what we do here is simple address translation - input exception_t misaligned_ex_i, - input logic lsu_req_i, // request address translation - input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in - input logic lsu_is_store_i, // the translation is requested by a store - // if we need to walk the page table we can't grant in the same cycle - // Cycle 0 - output logic lsu_dtlb_hit_o, // sent in the same cycle as the request if translation hits in the DTLB - output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) - // Cycle 1 - output logic lsu_valid_o, // translation is valid - output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address - output exception_t lsu_exception_o, // address translation threw an exception - // General control signals - input riscv::priv_lvl_t priv_lvl_i, - input riscv::priv_lvl_t ld_st_priv_lvl_i, - input logic sum_i, - input logic mxr_i, - // input logic flag_mprv_i, - input logic [riscv::PPNW-1:0] satp_ppn_i, - input logic [ASID_WIDTH-1:0] asid_i, - input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, - input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, - input logic flush_tlb_i, - // Performance counters - output logic itlb_miss_o, - output logic dtlb_miss_o, - // PTW memory interface - input dcache_req_o_t req_port_i, - output dcache_req_i_t req_port_o, - // PMP - input riscv::pmpcfg_t [15:0] pmpcfg_i, - input logic [15:0][riscv::PLEN-3:0] pmpaddr_i -); - - logic iaccess_err; // insufficient privilege to access this instruction page - logic daccess_err; // insufficient privilege to access this data page - logic ptw_active; // PTW is currently walking a page table - logic walking_instr; // PTW is walking because of an ITLB miss - logic ptw_error; // PTW threw an exception - logic ptw_access_exception; // PTW threw an access exception (PMPs) - logic [riscv::PLEN-1:0] ptw_bad_paddr; // PTW PMP exception bad physical addr - - logic [riscv::VLEN-1:0] update_vaddr; - tlb_update_sv32_t update_itlb, update_dtlb, update_shared_tlb; - - logic itlb_lu_access; - riscv::pte_sv32_t itlb_content; - logic itlb_is_4M; - logic itlb_lu_hit; - - logic dtlb_lu_access; - riscv::pte_sv32_t dtlb_content; - logic dtlb_is_4M; - logic dtlb_lu_hit; - - logic shared_tlb_access; - logic [riscv::VLEN-1:0] shared_tlb_vaddr; - logic shared_tlb_hit; - - logic itlb_req; - - - // Assignments - assign itlb_lu_access = icache_areq_i.fetch_req; - assign dtlb_lu_access = lsu_req_i; - - - cva6_tlb_sv32 #( - .CVA6Cfg (CVA6Cfg), - .TLB_ENTRIES(INSTR_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) - ) i_itlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_tlb_i), - - .update_i(update_itlb), - - .lu_access_i (itlb_lu_access), - .lu_asid_i (asid_i), - .asid_to_be_flushed_i (asid_to_be_flushed_i), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), - .lu_vaddr_i (icache_areq_i.fetch_vaddr), - .lu_content_o (itlb_content), - - .lu_is_4M_o(itlb_is_4M), - .lu_hit_o (itlb_lu_hit) - ); - - cva6_tlb_sv32 #( - .CVA6Cfg (CVA6Cfg), - .TLB_ENTRIES(DATA_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) - ) i_dtlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_tlb_i), - - .update_i(update_dtlb), - - .lu_access_i (dtlb_lu_access), - .lu_asid_i (asid_i), - .asid_to_be_flushed_i (asid_to_be_flushed_i), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), - .lu_vaddr_i (lsu_vaddr_i), - .lu_content_o (dtlb_content), - - .lu_is_4M_o(dtlb_is_4M), - .lu_hit_o (dtlb_lu_hit) - ); - - cva6_shared_tlb_sv32 #( - .CVA6Cfg (CVA6Cfg), - .SHARED_TLB_DEPTH(64), - .SHARED_TLB_WAYS (2), - .ASID_WIDTH (ASID_WIDTH) - ) i_shared_tlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_tlb_i), - - .enable_translation_i (enable_translation_i), - .en_ld_st_translation_i(en_ld_st_translation_i), - - .asid_i (asid_i), - // from TLBs - // did we miss? - .itlb_access_i(itlb_lu_access), - .itlb_hit_i (itlb_lu_hit), - .itlb_vaddr_i (icache_areq_i.fetch_vaddr), - - .dtlb_access_i(dtlb_lu_access), - .dtlb_hit_i (dtlb_lu_hit), - .dtlb_vaddr_i (lsu_vaddr_i), - - // to TLBs, update logic - .itlb_update_o(update_itlb), - .dtlb_update_o(update_dtlb), - - // Performance counters - .itlb_miss_o(itlb_miss_o), - .dtlb_miss_o(dtlb_miss_o), - - .shared_tlb_access_o(shared_tlb_access), - .shared_tlb_hit_o (shared_tlb_hit), - .shared_tlb_vaddr_o (shared_tlb_vaddr), - - .itlb_req_o (itlb_req), - // to update shared tlb - .shared_tlb_update_i(update_shared_tlb) - ); - - cva6_ptw_sv32 #( - .CVA6Cfg (CVA6Cfg), - .ASID_WIDTH(ASID_WIDTH) - ) i_ptw ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_i), - - .ptw_active_o (ptw_active), - .walking_instr_o (walking_instr), - .ptw_error_o (ptw_error), - .ptw_access_exception_o(ptw_access_exception), - - .lsu_is_store_i(lsu_is_store_i), - // PTW memory interface - .req_port_i (req_port_i), - .req_port_o (req_port_o), - - // to Shared TLB, update logic - .shared_tlb_update_o(update_shared_tlb), - - .update_vaddr_o(update_vaddr), - - .asid_i(asid_i), - - // from shared TLB - // did we miss? - .shared_tlb_access_i(shared_tlb_access), - .shared_tlb_hit_i (shared_tlb_hit), - .shared_tlb_vaddr_i (shared_tlb_vaddr), - - .itlb_req_i(itlb_req), - - // from CSR file - .satp_ppn_i(satp_ppn_i), // ppn from satp - .mxr_i (mxr_i), - - // Performance counters - .shared_tlb_miss_o(), //open for now - - // PMP - .pmpcfg_i (pmpcfg_i), - .pmpaddr_i (pmpaddr_i), - .bad_paddr_o(ptw_bad_paddr) - - ); - - // ila_1 i_ila_1 ( - // .clk(clk_i), // input wire clk - // .probe0({req_port_o.address_tag, req_port_o.address_index}), - // .probe1(req_port_o.data_req), // input wire [63:0] probe1 - // .probe2(req_port_i.data_gnt), // input wire [0:0] probe2 - // .probe3(req_port_i.data_rdata), // input wire [0:0] probe3 - // .probe4(req_port_i.data_rvalid), // input wire [0:0] probe4 - // .probe5(ptw_error), // input wire [1:0] probe5 - // .probe6(update_vaddr), // input wire [0:0] probe6 - // .probe7(update_itlb.valid), // input wire [0:0] probe7 - // .probe8(update_dtlb.valid), // input wire [0:0] probe8 - // .probe9(dtlb_lu_access), // input wire [0:0] probe9 - // .probe10(lsu_vaddr_i), // input wire [0:0] probe10 - // .probe11(dtlb_lu_hit), // input wire [0:0] probe11 - // .probe12(itlb_lu_access), // input wire [0:0] probe12 - // .probe13(icache_areq_i.fetch_vaddr), // input wire [0:0] probe13 - // .probe14(itlb_lu_hit) // input wire [0:0] probe13 - // ); - - //----------------------- - // Instruction Interface - //----------------------- - logic match_any_execute_region; - logic pmp_instr_allow; - - // The instruction interface is a simple request response interface - always_comb begin : instr_interface - // MMU disabled: just pass through - icache_areq_o.fetch_valid = icache_areq_i.fetch_req; - if (riscv::PLEN > riscv::VLEN) - icache_areq_o.fetch_paddr = { - {riscv::PLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr - }; // play through in case we disabled address translation - else - icache_areq_o.fetch_paddr = { - 2'b00, icache_areq_i.fetch_vaddr[riscv::VLEN-1:0] - }; // play through in case we disabled address translation - // two potential exception sources: - // 1. HPTW threw an exception -> signal with a page fault exception - // 2. We got an access error because of insufficient permissions -> throw an access exception - icache_areq_o.fetch_exception = '0; - // Check whether we are allowed to access this memory region from a fetch perspective - iaccess_err = icache_areq_i.fetch_req && (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content.u) - || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content.u)); - - // MMU enabled: address from TLB, request delayed until hit. Error when TLB - // hit and no access right or TLB hit and translated address not valid (e.g. - // AXI decode error), or when PTW performs walk due to ITLB miss and raises - // an error. - if (enable_translation_i) begin - // we work with SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal - if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = { - {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr - }; - end - - icache_areq_o.fetch_valid = 1'b0; - - // 4K page - icache_areq_o.fetch_paddr = {itlb_content.ppn, icache_areq_i.fetch_vaddr[11:0]}; - // Mega page - if (itlb_is_4M) begin - icache_areq_o.fetch_paddr[21:12] = icache_areq_i.fetch_vaddr[21:12]; - end - - - // --------- - // ITLB Hit - // -------- - // if we hit the ITLB output the request signal immediately - if (itlb_lu_hit) begin - icache_areq_o.fetch_valid = icache_areq_i.fetch_req; - // we got an access error - if (iaccess_err) begin - // throw a page fault - icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = { - {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr - }; - //to check on wave --> not connected - end else if (!pmp_instr_allow) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) icache_areq_o.fetch_exception.tval = icache_areq_i.fetch_vaddr; - //to check on wave --> not connected - end - end else - // --------- - // ITLB Miss - // --------- - // watch out for exceptions happening during walking the page table - if (ptw_active && walking_instr) begin - icache_areq_o.fetch_valid = ptw_error | ptw_access_exception; - if (ptw_error) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}; - end //to check on wave - // TODO(moschn,zarubaf): What should the value of tval be in this case? - else begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) icache_areq_o.fetch_exception.tval = ptw_bad_paddr[riscv::PLEN-1:2]; - end - end - end - // if it didn't match any execute region throw an `Instruction Access Fault` - // or: if we are not translating, check PMPs immediately on the paddr - if (!match_any_execute_region || (!enable_translation_i && !pmp_instr_allow)) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = icache_areq_o.fetch_paddr[riscv::PLEN-1:2]; - //to check on wave --> not connected - end - end - - // check for execute flag on memory - assign match_any_execute_region = config_pkg::is_inside_execute_regions( - CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} - ); - - // Instruction fetch - pmp #( - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_if ( - .addr_i (icache_areq_o.fetch_paddr), - .priv_lvl_i, - // we will always execute on the instruction fetch port - .access_type_i(riscv::ACCESS_EXEC), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (pmp_instr_allow) - ); - - //----------------------- - // Data Interface - //----------------------- - logic [riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; - riscv::pte_sv32_t dtlb_pte_n, dtlb_pte_q; - exception_t misaligned_ex_n, misaligned_ex_q; - logic lsu_req_n, lsu_req_q; - logic lsu_is_store_n, lsu_is_store_q; - logic dtlb_hit_n, dtlb_hit_q; - logic dtlb_is_4M_n, dtlb_is_4M_q; - - // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) - assign lsu_dtlb_hit_o = (en_ld_st_translation_i) ? dtlb_lu_hit : 1'b1; - - // Wires to PMP checks - riscv::pmp_access_t pmp_access_type; - logic pmp_data_allow; - localparam PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; - // The data interface is simpler and only consists of a request/response interface - always_comb begin : data_interface - // save request and DTLB response - lsu_vaddr_n = lsu_vaddr_i; - lsu_req_n = lsu_req_i; - misaligned_ex_n = misaligned_ex_i; - dtlb_pte_n = dtlb_content; - dtlb_hit_n = dtlb_lu_hit; - lsu_is_store_n = lsu_is_store_i; - dtlb_is_4M_n = dtlb_is_4M; - - if (riscv::PLEN > riscv::VLEN) begin - lsu_paddr_o = {{riscv::PLEN - riscv::VLEN{1'b0}}, lsu_vaddr_q}; - lsu_dtlb_ppn_o = {{riscv::PLEN - riscv::VLEN{1'b0}}, lsu_vaddr_n[riscv::VLEN-1:12]}; - end else begin - lsu_paddr_o = {2'b00, lsu_vaddr_q[riscv::VLEN-1:0]}; - lsu_dtlb_ppn_o = lsu_vaddr_n[riscv::PPNW-1:0]; - end - lsu_valid_o = lsu_req_q; - lsu_exception_o = misaligned_ex_q; - pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; - - // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions - misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; - - // Check if the User flag is set, then we may only access it in supervisor mode - // if SUM is enabled - daccess_err = (ld_st_priv_lvl_i == riscv::PRIV_LVL_S && !sum_i && dtlb_pte_q.u) || // SUM is not set and we are trying to access a user page in supervisor mode - (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q.u); // this is not a user page but we are in user mode and trying to access it - // translation is enabled and no misaligned exception occurred - if (en_ld_st_translation_i && !misaligned_ex_q.valid) begin - lsu_valid_o = 1'b0; - // 4K page - lsu_paddr_o = {dtlb_pte_q.ppn, lsu_vaddr_q[11:0]}; - lsu_dtlb_ppn_o = dtlb_content.ppn; - // Mega page - if (dtlb_is_4M_q) begin - lsu_paddr_o[21:12] = lsu_vaddr_q[21:12]; - lsu_dtlb_ppn_o[21:12] = lsu_vaddr_n[21:12]; - end - // --------- - // DTLB Hit - // -------- - if (dtlb_hit_q && lsu_req_q) begin - lsu_valid_o = 1'b1; - // exception priority: - // PAGE_FAULTS have higher priority than ACCESS_FAULTS - // virtual memory based exceptions are PAGE_FAULTS - // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) - - // this is a store - if (lsu_is_store_q) begin - // check if the page is write-able and we are not violating privileges - // also check if the dirty flag is set - if (!dtlb_pte_q.w || daccess_err || !dtlb_pte_q.d) begin - lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q - }; - // to check on wave - // Check if any PMPs are violated - end else if (!pmp_data_allow) begin - lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; - //only 32 bits on 34b of lsu_paddr_o are returned. - end - - // this is a load - end else begin - // check for sufficient access privileges - throw a page fault if necessary - if (daccess_err) begin - lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q - }; - // Check if any PMPs are violated - end else if (!pmp_data_allow) begin - lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; - end - end - end else - - // --------- - // DTLB Miss - // --------- - // watch out for exceptions - if (ptw_active && !walking_instr) begin - // page table walker threw an exception - if (ptw_error) begin - // an error makes the translation valid - lsu_valid_o = 1'b1; - // the page table walker can only throw page faults - if (lsu_is_store_q) begin - lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr - }; - end else begin - lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr - }; - end - end - - if (ptw_access_exception) begin - // an error makes the translation valid - lsu_valid_o = 1'b1; - // the page table walker can only throw page faults - lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = ptw_bad_paddr[riscv::PLEN-1:2]; - end - end - end // If translation is not enabled, check the paddr immediately against PMPs - else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin - if (lsu_is_store_q) begin - lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; - end else begin - lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = lsu_paddr_o[riscv::PLEN-1:2]; - end - end - end - - // Load/store PMP check - pmp #( - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_data ( - .addr_i (lsu_paddr_o), - .priv_lvl_i (ld_st_priv_lvl_i), - .access_type_i(pmp_access_type), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (pmp_data_allow) - ); - - // ---------- - // Registers - // ---------- - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - lsu_vaddr_q <= '0; - lsu_req_q <= '0; - misaligned_ex_q <= '0; - dtlb_pte_q <= '0; - dtlb_hit_q <= '0; - lsu_is_store_q <= '0; - dtlb_is_4M_q <= '0; - end else begin - lsu_vaddr_q <= lsu_vaddr_n; - lsu_req_q <= lsu_req_n; - misaligned_ex_q <= misaligned_ex_n; - dtlb_pte_q <= dtlb_pte_n; - dtlb_hit_q <= dtlb_hit_n; - lsu_is_store_q <= lsu_is_store_n; - dtlb_is_4M_q <= dtlb_is_4M_n; - end - end -endmodule diff --git a/core/mmu_sv32/cva6_ptw_sv32.sv b/core/mmu_sv32/cva6_ptw_sv32.sv deleted file mode 100644 index 4bd736bd30..0000000000 --- a/core/mmu_sv32/cva6_ptw_sv32.sv +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright (c) 2021 Thales. -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Sebastien Jacq Thales Research & Technology -// Date: 17/07/2021 -// -// Additional contributions by: -// Sebastien Jacq - sjthales on github.com -// -// Description: Hardware-PTW (Page-Table-Walker) for MMU Sv32. -// This module is an adaptation of the Sv39 PTW developed -// by Florian Zaruba and David Schaffenrath to the Sv32 standard. -// -// =========================================================================== // -// Revisions : -// Date Version Author Description -// 2020-02-17 0.1 S.Jacq PTW Sv32 for CV32A6 -// =========================================================================== // - -/* verilator lint_off WIDTH */ - -module cva6_ptw_sv32 - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int ASID_WIDTH = 1 -) ( - input logic clk_i, // Clock - input logic rst_ni, // Asynchronous reset active low - input logic flush_i, // flush everything, we need to do this because - // actually everything we do is speculative at this stage - // e.g.: there could be a CSR instruction that changes everything - output logic ptw_active_o, - output logic walking_instr_o, // set when walking for TLB - output logic ptw_error_o, // set when an error occurred - output logic ptw_access_exception_o, // set when an PMP access exception occured - - input logic lsu_is_store_i, // this translation was triggered by a store - // PTW memory interface - input dcache_req_o_t req_port_i, - output dcache_req_i_t req_port_o, - - // to Shared TLB, update logic - output tlb_update_sv32_t shared_tlb_update_o, - - output logic [riscv::VLEN-1:0] update_vaddr_o, - - input logic [ASID_WIDTH-1:0] asid_i, - - // from shared TLB - input logic shared_tlb_access_i, - input logic shared_tlb_hit_i, - input logic [riscv::VLEN-1:0] shared_tlb_vaddr_i, - - input logic itlb_req_i, - - // from CSR file - input logic [riscv::PPNW-1:0] satp_ppn_i, // ppn from satp - input logic mxr_i, - - // Performance counters - output logic shared_tlb_miss_o, - - // PMP - input riscv::pmpcfg_t [15:0] pmpcfg_i, - input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, - output logic [riscv::PLEN-1:0] bad_paddr_o - -); - - // input registers - logic data_rvalid_q; - riscv::xlen_t data_rdata_q; - - riscv::pte_sv32_t pte; - assign pte = riscv::pte_sv32_t'(data_rdata_q); - - - enum logic [2:0] { - IDLE, - WAIT_GRANT, - PTE_LOOKUP, - WAIT_RVALID, - PROPAGATE_ERROR, - PROPAGATE_ACCESS_ERROR, - LATENCY - } - state_q, state_d; - - // SV32 defines two levels of page tables - enum logic { - LVL1, - LVL2 - } - ptw_lvl_q, ptw_lvl_n; - - // is this an instruction page table walk? - logic is_instr_ptw_q, is_instr_ptw_n; - logic global_mapping_q, global_mapping_n; - // latched tag signal - logic tag_valid_n, tag_valid_q; - // register the ASID - logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_n; - // register the VPN we need to walk, SV32 defines a 32 bit virtual address - logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; - // 4 byte aligned physical pointer - logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; - - // Assignments - assign update_vaddr_o = vaddr_q; - - assign ptw_active_o = (state_q != IDLE); - //assign walking_instr_o = is_instr_ptw_q; - assign walking_instr_o = is_instr_ptw_q; - // directly output the correct physical address - assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; - assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; - // we are never going to kill this request - assign req_port_o.kill_req = '0; - // we are never going to write with the HPTW - assign req_port_o.data_wdata = '0; - // we only issue one single request at a time - assign req_port_o.data_id = '0; - - // ----------- - // Shared TLB Update - // ----------- - assign shared_tlb_update_o.vpn = vaddr_q[riscv::SV-1:12]; - // update the correct page table level - assign shared_tlb_update_o.is_4M = (ptw_lvl_q == LVL1); - // output the correct ASID - assign shared_tlb_update_o.asid = tlb_update_asid_q; - // set the global mapping bit - assign shared_tlb_update_o.content = pte | (global_mapping_q << 5); - - - assign req_port_o.tag_valid = tag_valid_q; - - logic allow_access; - - assign bad_paddr_o = ptw_access_exception_o ? ptw_pptr_q : 'b0; - - pmp #( - .CVA6Cfg (CVA6Cfg), - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_ptw ( - .addr_i (ptw_pptr_q), - // PTW access are always checked as if in S-Mode... - .priv_lvl_i (riscv::PRIV_LVL_S), - // ...and they are always loads - .access_type_i(riscv::ACCESS_READ), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (allow_access) - ); - - - assign req_port_o.data_be = be_gen_32(req_port_o.address_index[1:0], req_port_o.data_size); - - //------------------- - // Page table walker - //------------------- - // A virtual address va is translated into a physical address pa as follows: - // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, - // PAGESIZE=2^12 and LEVELS=3.) - // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For - // Sv32, PTESIZE=4.) - // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise an access - // exception. - // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. - // Otherwise, this PTE is a pointer to the next level of the page table. - // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let - // a = pte.ppn × PAGESIZE and go to step 2. - // 5. A leaf PTE has been found. Determine if the requested memory access - // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and - // raise an access exception. Otherwise, the translation is successful. - // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. - // The translated physical address is given as follows: - // - pa.pgoff = va.pgoff. - // - If i > 0, then this is a superpage translation and - // pa.ppn[i-1:0] = va.vpn[i-1:0]. - // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. - always_comb begin : ptw - // default assignments - // PTW memory interface - tag_valid_n = 1'b0; - req_port_o.data_req = 1'b0; - req_port_o.data_size = 2'b10; - req_port_o.data_we = 1'b0; - ptw_error_o = 1'b0; - ptw_access_exception_o = 1'b0; - shared_tlb_update_o.valid = 1'b0; - is_instr_ptw_n = is_instr_ptw_q; - ptw_lvl_n = ptw_lvl_q; - ptw_pptr_n = ptw_pptr_q; - state_d = state_q; - global_mapping_n = global_mapping_q; - // input registers - tlb_update_asid_n = tlb_update_asid_q; - vaddr_n = vaddr_q; - - shared_tlb_miss_o = 1'b0; - - case (state_q) - - IDLE: begin - // by default we start with the top-most page table - ptw_lvl_n = LVL1; - global_mapping_n = 1'b0; - is_instr_ptw_n = 1'b0; - // if we got a Shared TLB miss - if (shared_tlb_access_i & ~shared_tlb_hit_i) begin - ptw_pptr_n = { - satp_ppn_i, shared_tlb_vaddr_i[riscv::SV-1:22], 2'b0 - }; // SATP.PPN * PAGESIZE + VPN*PTESIZE = SATP.PPN * 2^(12) + VPN*4 - is_instr_ptw_n = itlb_req_i; - tlb_update_asid_n = asid_i; - vaddr_n = shared_tlb_vaddr_i; - state_d = WAIT_GRANT; - shared_tlb_miss_o = 1'b1; - end - end - - WAIT_GRANT: begin - // send a request out - req_port_o.data_req = 1'b1; - // wait for the WAIT_GRANT - if (req_port_i.data_gnt) begin - // send the tag valid signal one cycle later - tag_valid_n = 1'b1; - state_d = PTE_LOOKUP; - end - end - - PTE_LOOKUP: begin - // we wait for the valid signal - if (data_rvalid_q) begin - - // check if the global mapping bit is set - if (pte.g) global_mapping_n = 1'b1; - - // ------------- - // Invalid PTE - // ------------- - // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. - if (!pte.v || (!pte.r && pte.w)) state_d = PROPAGATE_ERROR; - // ----------- - // Valid PTE - // ----------- - else begin - //state_d = IDLE; - state_d = LATENCY; - // it is a valid PTE - // if pte.r = 1 or pte.x = 1 it is a valid PTE - if (pte.r || pte.x) begin - // Valid translation found (either 4M or 4K entry) - if (is_instr_ptw_q) begin - // ------------ - // Update ITLB - // ------------ - // If page is not executable, we can directly raise an error. This - // doesn't put a useless entry into the TLB. The same idea applies - // to the access flag since we let the access flag be managed by SW. - if (!pte.x || !pte.a) state_d = PROPAGATE_ERROR; - else shared_tlb_update_o.valid = 1'b1; - - end else begin - // ------------ - // Update DTLB - // ------------ - // Check if the access flag has been set, otherwise throw a page-fault - // and let the software handle those bits. - // If page is not readable (there are no write-only pages) - // we can directly raise an error. This doesn't put a useless - // entry into the TLB. - if (pte.a && (pte.r || (pte.x && mxr_i))) begin - shared_tlb_update_o.valid = 1'b1; - end else begin - state_d = PROPAGATE_ERROR; - end - // Request is a store: perform some additional checks - // If the request was a store and the page is not write-able, raise an error - // the same applies if the dirty flag is not set - if (lsu_is_store_i && (!pte.w || !pte.d)) begin - shared_tlb_update_o.valid = 1'b0; - state_d = PROPAGATE_ERROR; - end - end - // check if the ppn is correctly aligned: - // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault - // exception. - if (ptw_lvl_q == LVL1 && pte.ppn[9:0] != '0) begin - state_d = PROPAGATE_ERROR; - shared_tlb_update_o.valid = 1'b0; - end - // this is a pointer to the next TLB level - end else begin - // pointer to next level of page table - if (ptw_lvl_q == LVL1) begin - // we are in the second level now - ptw_lvl_n = LVL2; - ptw_pptr_n = {pte.ppn, vaddr_q[21:12], 2'b0}; - end - - state_d = WAIT_GRANT; - - if (ptw_lvl_q == LVL2) begin - // Should already be the last level page table => Error - ptw_lvl_n = LVL2; - state_d = PROPAGATE_ERROR; - end - end - end - - // Check if this access was actually allowed from a PMP perspective - if (!allow_access) begin - shared_tlb_update_o.valid = 1'b0; - // we have to return the failed address in bad_addr - ptw_pptr_n = ptw_pptr_q; - state_d = PROPAGATE_ACCESS_ERROR; - end - end - // we've got a data WAIT_GRANT so tell the cache that the tag is valid - end - // Propagate error to MMU/LSU - PROPAGATE_ERROR: begin - state_d = LATENCY; - ptw_error_o = 1'b1; - end - PROPAGATE_ACCESS_ERROR: begin - state_d = LATENCY; - ptw_access_exception_o = 1'b1; - end - // wait for the rvalid before going back to IDLE - WAIT_RVALID: begin - if (data_rvalid_q) state_d = IDLE; - end - LATENCY: begin - state_d = IDLE; - end - default: begin - state_d = IDLE; - end - endcase - - // ------- - // Flush - // ------- - // should we have flushed before we got an rvalid, wait for it until going back to IDLE - if (flush_i) begin - // on a flush check whether we are - // 1. in the PTE Lookup check whether we still need to wait for an rvalid - // 2. waiting for a grant, if so: wait for it - // if not, go back to idle - if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || - ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) - state_d = WAIT_RVALID; - else state_d = LATENCY; - end - end - - // sequential process - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - state_q <= IDLE; - is_instr_ptw_q <= 1'b0; - ptw_lvl_q <= LVL1; - tag_valid_q <= 1'b0; - tlb_update_asid_q <= '0; - vaddr_q <= '0; - ptw_pptr_q <= '0; - global_mapping_q <= 1'b0; - data_rdata_q <= '0; - data_rvalid_q <= 1'b0; - end else begin - state_q <= state_d; - ptw_pptr_q <= ptw_pptr_n; - is_instr_ptw_q <= is_instr_ptw_n; - ptw_lvl_q <= ptw_lvl_n; - tag_valid_q <= tag_valid_n; - tlb_update_asid_q <= tlb_update_asid_n; - vaddr_q <= vaddr_n; - global_mapping_q <= global_mapping_n; - data_rdata_q <= req_port_i.data_rdata; - data_rvalid_q <= req_port_i.data_rvalid; - end - end - -endmodule -/* verilator lint_on WIDTH */ diff --git a/core/mmu_sv32/cva6_tlb_sv32.sv b/core/mmu_sv32/cva6_tlb_sv32.sv deleted file mode 100644 index 79a7c98dc5..0000000000 --- a/core/mmu_sv32/cva6_tlb_sv32.sv +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright (c) 2021 Thales. -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Sebastien Jacq Thales Research & Technology -// Date: 17/07/2021 -// -// Additional contributions by: -// Sebastien Jacq - sjthales on github.com -// -// Description: Translation Lookaside Buffer, Sv32 , fully set-associative -// This module is an adaptation of the Sv39 TLB developed -// by Florian Zaruba and David Schaffenrath to the Sv32 standard. -// -// =========================================================================== // -// Revisions : -// Date Version Author Description -// 2020-02-17 0.1 S.Jacq TLB Sv32 for CV32A6 -// =========================================================================== // - -module cva6_tlb_sv32 - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int unsigned TLB_ENTRIES = 4, - parameter int unsigned ASID_WIDTH = 1 -) ( - input logic clk_i, // Clock - input logic rst_ni, // Asynchronous reset active low - input logic flush_i, // Flush signal - // Update TLB - input tlb_update_sv32_t update_i, - // Lookup signals - input logic lu_access_i, - input logic [ASID_WIDTH-1:0] lu_asid_i, - input logic [riscv::VLEN-1:0] lu_vaddr_i, - output riscv::pte_sv32_t lu_content_o, - input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, - input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, - output logic lu_is_4M_o, - output logic lu_hit_o -); - - // Sv32 defines two levels of page tables - struct packed { - logic [8:0] asid; //9 bits wide - logic [9:0] vpn1; //10 bits wide - logic [9:0] vpn0; //10 bits wide - logic is_4M; - logic valid; - } [TLB_ENTRIES-1:0] - tags_q, tags_n; - - riscv::pte_sv32_t [TLB_ENTRIES-1:0] content_q, content_n; - logic [9:0] vpn0, vpn1; - logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic - logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy - //------------- - // Translation - //------------- - always_comb begin : translation - vpn0 = lu_vaddr_i[21:12]; - vpn1 = lu_vaddr_i[31:22]; - - - // default assignment - lu_hit = '{default: 0}; - lu_hit_o = 1'b0; - lu_content_o = '{default: 0}; - lu_is_4M_o = 1'b0; - - for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - // first level match, this may be a mega page, check the ASID flags as well - // if the entry is associated to a global address, don't match the ASID (ASID is don't care) - if (tags_q[i].valid && ((lu_asid_i == tags_q[i].asid[ASID_WIDTH-1:0]) || content_q[i].g) && vpn1 == tags_q[i].vpn1) begin - if (tags_q[i].is_4M || vpn0 == tags_q[i].vpn0) begin - lu_is_4M_o = tags_q[i].is_4M; - lu_content_o = content_q[i]; - lu_hit_o = 1'b1; - lu_hit[i] = 1'b1; - end - end - end - end - - logic asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high - logic vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high - logic [TLB_ENTRIES-1:0] vaddr_vpn0_match; - logic [TLB_ENTRIES-1:0] vaddr_vpn1_match; - - - assign asid_to_be_flushed_is0 = ~(|asid_to_be_flushed_i); - assign vaddr_to_be_flushed_is0 = ~(|vaddr_to_be_flushed_i); - - // ------------------ - // Update and Flush - // ------------------ - always_comb begin : update_flush - tags_n = tags_q; - content_n = content_q; - - for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - - vaddr_vpn0_match[i] = (vaddr_to_be_flushed_i[21:12] == tags_q[i].vpn0); - vaddr_vpn1_match[i] = (vaddr_to_be_flushed_i[31:22] == tags_q[i].vpn1); - - if (flush_i) begin - // invalidate logic - // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) - if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; - // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages - else if (asid_to_be_flushed_is0 && ( (vaddr_vpn0_match[i] && vaddr_vpn1_match[i]) || (vaddr_vpn1_match[i] && tags_q[i].is_4M) ) && (~vaddr_to_be_flushed_is0)) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) - else if ((!content_q[i].g) && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i]) || (vaddr_vpn1_match[i] && tags_q[i].is_4M)) && (asid_to_be_flushed_i == tags_q[i].asid[ASID_WIDTH-1:0]) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) - else if ((!content_q[i].g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid[ASID_WIDTH-1:0]) && (!asid_to_be_flushed_is0)) - tags_n[i].valid = 1'b0; - // normal replacement - end else if (update_i.valid & replace_en[i]) begin - // update tag array - tags_n[i] = '{ - asid: update_i.asid, - vpn1: update_i.vpn[19:10], - vpn0: update_i.vpn[9:0], - is_4M: update_i.is_4M, - valid: 1'b1 - }; - // and content as well - content_n[i] = update_i.content; - end - end - end - - // ----------------------------------------------- - // PLRU - Pseudo Least Recently Used Replacement - // ----------------------------------------------- - logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; - logic en; - int unsigned idx_base, shift, new_index; - always_comb begin : plru_replacement - plru_tree_n = plru_tree_q; - en = '0; - idx_base = '0; - shift = '0; - new_index = '0; - // The PLRU-tree indexing: - // lvl0 0 - // / \ - // / \ - // lvl1 1 2 - // / \ / \ - // lvl2 3 4 5 6 - // / \ /\/\ /\ - // ... ... ... ... - // Just predefine which nodes will be set/cleared - // E.g. for a TLB with 8 entries, the for-loop is semantically - // equivalent to the following pseudo-code: - // unique case (1'b1) - // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; - // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; - // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; - // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; - // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; - // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; - // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; - // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; - // default: begin /* No hit */ end - // endcase - for ( - int unsigned i = 0; i < TLB_ENTRIES; i++ - ) begin - // we got a hit so update the pointer as it was least recently used - if (lu_hit[i] & lu_access_i) begin - // Set the nodes to the values we would expect - for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin - idx_base = $unsigned((2 ** lvl) - 1); - // lvl0 <=> MSB, lvl1 <=> MSB-1, ... - shift = $clog2(TLB_ENTRIES) - lvl; - // to circumvent the 32 bit integer arithmetic assignment - new_index = ~((i >> (shift - 1)) & 32'b1); - plru_tree_n[idx_base+(i>>shift)] = new_index[0]; - end - end - end - // Decode tree to write enable signals - // Next for-loop basically creates the following logic for e.g. an 8 entry - // TLB (note: pseudo-code obviously): - // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} - // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} - // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} - // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} - // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} - // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} - // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} - // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} - // For each entry traverse the tree. If every tree-node matches, - // the corresponding bit of the entry's index, this is - // the next entry to replace. - for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin - en = 1'b1; - for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin - idx_base = $unsigned((2 ** lvl) - 1); - // lvl0 <=> MSB, lvl1 <=> MSB-1, ... - shift = $clog2(TLB_ENTRIES) - lvl; - - // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); - new_index = (i >> (shift - 1)) & 32'b1; - if (new_index[0]) begin - en &= plru_tree_q[idx_base+(i>>shift)]; - end else begin - en &= ~plru_tree_q[idx_base+(i>>shift)]; - end - end - replace_en[i] = en; - end - end - - // sequential process - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - tags_q <= '{default: 0}; - content_q <= '{default: 0}; - plru_tree_q <= '{default: 0}; - end else begin - tags_q <= tags_n; - content_q <= content_n; - plru_tree_q <= plru_tree_n; - end - end - //-------------- - // Sanity checks - //-------------- - - //pragma translate_off -`ifndef VERILATOR - - initial begin : p_assertions - assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) - else begin - $error("TLB size must be a multiple of 2 and greater than 1"); - $stop(); - end - assert (ASID_WIDTH >= 1) - else begin - $error("ASID width must be at least 1"); - $stop(); - end - end - - // Just for checking - function int countSetBits(logic [TLB_ENTRIES-1:0] vector); - automatic int count = 0; - foreach (vector[idx]) begin - count += vector[idx]; - end - return count; - endfunction - - assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) - else begin - $error("More then one hit in TLB!"); - $stop(); - end - assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) - else begin - $error("More then one TLB entry selected for next replace!"); - $stop(); - end - -`endif - //pragma translate_on - -endmodule diff --git a/core/mmu_sv39/mmu.sv b/core/mmu_sv39/mmu.sv deleted file mode 100644 index 56994f260d..0000000000 --- a/core/mmu_sv39/mmu.sv +++ /dev/null @@ -1,534 +0,0 @@ -// Copyright 2018 ETH Zurich and University of Bologna. -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: Florian Zaruba, ETH Zurich -// Date: 19/04/2017 -// Description: Memory Management Unit for Ariane, contains TLB and -// address translation unit. SV39 as defined in RISC-V -// privilege specification 1.11-WIP - - -module mmu - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int unsigned INSTR_TLB_ENTRIES = 4, - parameter int unsigned DATA_TLB_ENTRIES = 4, - parameter int unsigned ASID_WIDTH = 1 -) ( - input logic clk_i, - input logic rst_ni, - input logic flush_i, - input logic enable_translation_i, - input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores - // IF interface - input icache_arsp_t icache_areq_i, - output icache_areq_t icache_areq_o, - // LSU interface - // this is a more minimalistic interface because the actual addressing logic is handled - // in the LSU as we distinguish load and stores, what we do here is simple address translation - input exception_t misaligned_ex_i, - input logic lsu_req_i, // request address translation - input logic [riscv::VLEN-1:0] lsu_vaddr_i, // virtual address in - input logic lsu_is_store_i, // the translation is requested by a store - // if we need to walk the page table we can't grant in the same cycle - // Cycle 0 - output logic lsu_dtlb_hit_o, // sent in the same cycle as the request if translation hits in the DTLB - output logic [riscv::PPNW-1:0] lsu_dtlb_ppn_o, // ppn (send same cycle as hit) - // Cycle 1 - output logic lsu_valid_o, // translation is valid - output logic [riscv::PLEN-1:0] lsu_paddr_o, // translated address - output exception_t lsu_exception_o, // address translation threw an exception - // General control signals - input riscv::priv_lvl_t priv_lvl_i, - input riscv::priv_lvl_t ld_st_priv_lvl_i, - input logic sum_i, - input logic mxr_i, - // input logic flag_mprv_i, - input logic [riscv::PPNW-1:0] satp_ppn_i, - input logic [ASID_WIDTH-1:0] asid_i, - input logic [ASID_WIDTH-1:0] asid_to_be_flushed_i, - input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, - input logic flush_tlb_i, - // Performance counters - output logic itlb_miss_o, - output logic dtlb_miss_o, - // PTW memory interface - input dcache_req_o_t req_port_i, - output dcache_req_i_t req_port_o, - // PMP - input riscv::pmpcfg_t [15:0] pmpcfg_i, - input logic [15:0][riscv::PLEN-3:0] pmpaddr_i -); - - logic iaccess_err; // insufficient privilege to access this instruction page - logic daccess_err; // insufficient privilege to access this data page - logic ptw_active; // PTW is currently walking a page table - logic walking_instr; // PTW is walking because of an ITLB miss - logic ptw_error; // PTW threw an exception - logic ptw_access_exception; // PTW threw an access exception (PMPs) - logic [riscv::PLEN-1:0] ptw_bad_paddr; // PTW PMP exception bad physical addr - - logic [riscv::VLEN-1:0] update_vaddr; - tlb_update_t update_ptw_itlb, update_ptw_dtlb; - - logic itlb_lu_access; - riscv::pte_t itlb_content; - logic itlb_is_2M; - logic itlb_is_1G; - logic itlb_lu_hit; - - logic dtlb_lu_access; - riscv::pte_t dtlb_content; - logic dtlb_is_2M; - logic dtlb_is_1G; - logic dtlb_lu_hit; - - - // Assignments - assign itlb_lu_access = icache_areq_i.fetch_req; - assign dtlb_lu_access = lsu_req_i; - - - tlb #( - .CVA6Cfg (CVA6Cfg), - .TLB_ENTRIES(INSTR_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) - ) i_itlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_tlb_i), - - .update_i(update_ptw_itlb), - - .lu_access_i (itlb_lu_access), - .lu_asid_i (asid_i), - .asid_to_be_flushed_i (asid_to_be_flushed_i), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), - .lu_vaddr_i (icache_areq_i.fetch_vaddr), - .lu_content_o (itlb_content), - - .lu_is_2M_o(itlb_is_2M), - .lu_is_1G_o(itlb_is_1G), - .lu_hit_o (itlb_lu_hit) - ); - - tlb #( - .CVA6Cfg (CVA6Cfg), - .TLB_ENTRIES(DATA_TLB_ENTRIES), - .ASID_WIDTH (ASID_WIDTH) - ) i_dtlb ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .flush_i(flush_tlb_i), - - .update_i(update_ptw_dtlb), - - .lu_access_i (dtlb_lu_access), - .lu_asid_i (asid_i), - .asid_to_be_flushed_i (asid_to_be_flushed_i), - .vaddr_to_be_flushed_i(vaddr_to_be_flushed_i), - .lu_vaddr_i (lsu_vaddr_i), - .lu_content_o (dtlb_content), - - .lu_is_2M_o(dtlb_is_2M), - .lu_is_1G_o(dtlb_is_1G), - .lu_hit_o (dtlb_lu_hit) - ); - - - ptw #( - .CVA6Cfg (CVA6Cfg), - .ASID_WIDTH(ASID_WIDTH) - ) i_ptw ( - .clk_i (clk_i), - .rst_ni (rst_ni), - .ptw_active_o (ptw_active), - .walking_instr_o (walking_instr), - .ptw_error_o (ptw_error), - .ptw_access_exception_o(ptw_access_exception), - .enable_translation_i (enable_translation_i), - - .update_vaddr_o(update_vaddr), - .itlb_update_o (update_ptw_itlb), - .dtlb_update_o (update_ptw_dtlb), - - .itlb_access_i(itlb_lu_access), - .itlb_hit_i (itlb_lu_hit), - .itlb_vaddr_i (icache_areq_i.fetch_vaddr), - - .dtlb_access_i(dtlb_lu_access), - .dtlb_hit_i (dtlb_lu_hit), - .dtlb_vaddr_i (lsu_vaddr_i), - - .req_port_i (req_port_i), - .req_port_o (req_port_o), - .pmpcfg_i, - .pmpaddr_i, - .bad_paddr_o(ptw_bad_paddr), - .* - ); - - // ila_1 i_ila_1 ( - // .clk(clk_i), // input wire clk - // .probe0({req_port_o.address_tag, req_port_o.address_index}), - // .probe1(req_port_o.data_req), // input wire [63:0] probe1 - // .probe2(req_port_i.data_gnt), // input wire [0:0] probe2 - // .probe3(req_port_i.data_rdata), // input wire [0:0] probe3 - // .probe4(req_port_i.data_rvalid), // input wire [0:0] probe4 - // .probe5(ptw_error), // input wire [1:0] probe5 - // .probe6(update_vaddr), // input wire [0:0] probe6 - // .probe7(update_ptw_itlb.valid), // input wire [0:0] probe7 - // .probe8(update_ptw_dtlb.valid), // input wire [0:0] probe8 - // .probe9(dtlb_lu_access), // input wire [0:0] probe9 - // .probe10(lsu_vaddr_i), // input wire [0:0] probe10 - // .probe11(dtlb_lu_hit), // input wire [0:0] probe11 - // .probe12(itlb_lu_access), // input wire [0:0] probe12 - // .probe13(icache_areq_i.fetch_vaddr), // input wire [0:0] probe13 - // .probe14(itlb_lu_hit) // input wire [0:0] probe13 - // ); - - //----------------------- - // Instruction Interface - //----------------------- - logic match_any_execute_region; - logic pmp_instr_allow; - - // The instruction interface is a simple request response interface - always_comb begin : instr_interface - // MMU disabled: just pass through - icache_areq_o.fetch_valid = icache_areq_i.fetch_req; - icache_areq_o.fetch_paddr = icache_areq_i.fetch_vaddr[riscv::PLEN-1:0]; // play through in case we disabled address translation - // two potential exception sources: - // 1. HPTW threw an exception -> signal with a page fault exception - // 2. We got an access error because of insufficient permissions -> throw an access exception - icache_areq_o.fetch_exception = '0; - // Check whether we are allowed to access this memory region from a fetch perspective - iaccess_err = icache_areq_i.fetch_req && enable_translation_i - && (((priv_lvl_i == riscv::PRIV_LVL_U) && ~itlb_content.u) - || ((priv_lvl_i == riscv::PRIV_LVL_S) && itlb_content.u)); - - // MMU enabled: address from TLB, request delayed until hit. Error when TLB - // hit and no access right or TLB hit and translated address not valid (e.g. - // AXI decode error), or when PTW performs walk due to ITLB miss and raises - // an error. - if (enable_translation_i) begin - // we work with SV39 or SV32, so if VM is enabled, check that all bits [riscv::VLEN-1:riscv::SV-1] are equal - if (icache_areq_i.fetch_req && !((&icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b1 || (|icache_areq_i.fetch_vaddr[riscv::VLEN-1:riscv::SV-1]) == 1'b0)) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = { - {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr - }; - end - - icache_areq_o.fetch_valid = 1'b0; - - // 4K page - icache_areq_o.fetch_paddr = {itlb_content.ppn, icache_areq_i.fetch_vaddr[11:0]}; - // Mega page - if (itlb_is_2M) begin - icache_areq_o.fetch_paddr[20:12] = icache_areq_i.fetch_vaddr[20:12]; - end - // Giga page - if (itlb_is_1G) begin - icache_areq_o.fetch_paddr[29:12] = icache_areq_i.fetch_vaddr[29:12]; - end - - // --------- - // ITLB Hit - // -------- - // if we hit the ITLB output the request signal immediately - if (itlb_lu_hit) begin - icache_areq_o.fetch_valid = icache_areq_i.fetch_req; - // we got an access error - if (iaccess_err) begin - // throw a page fault - icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = { - {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr - }; - end else if (!pmp_instr_allow) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = { - {riscv::XLEN - riscv::VLEN{1'b0}}, icache_areq_i.fetch_vaddr - }; - end - end else - // --------- - // ITLB Miss - // --------- - // watch out for exceptions happening during walking the page table - if (ptw_active && walking_instr) begin - icache_areq_o.fetch_valid = ptw_error | ptw_access_exception; - if (ptw_error) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_PAGE_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}; - end else begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, update_vaddr}; - end - end - end - // if it didn't match any execute region throw an `Instruction Access Fault` - // or: if we are not translating, check PMPs immediately on the paddr - if ((!match_any_execute_region && !ptw_error) || (!enable_translation_i && !pmp_instr_allow)) begin - icache_areq_o.fetch_exception.cause = riscv::INSTR_ACCESS_FAULT; - icache_areq_o.fetch_exception.valid = 1'b1; - if (CVA6Cfg.TvalEn) - icache_areq_o.fetch_exception.tval = { - {riscv::XLEN - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr - }; - end - end - - // check for execute flag on memory - assign match_any_execute_region = config_pkg::is_inside_execute_regions( - CVA6Cfg, {{64 - riscv::PLEN{1'b0}}, icache_areq_o.fetch_paddr} - ); - - // Instruction fetch - pmp #( - .CVA6Cfg (CVA6Cfg), - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_if ( - .addr_i (icache_areq_o.fetch_paddr), - .priv_lvl_i, - // we will always execute on the instruction fetch port - .access_type_i(riscv::ACCESS_EXEC), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (pmp_instr_allow) - ); - - //----------------------- - // Data Interface - //----------------------- - logic [riscv::VLEN-1:0] lsu_vaddr_n, lsu_vaddr_q; - riscv::pte_t dtlb_pte_n, dtlb_pte_q; - exception_t misaligned_ex_n, misaligned_ex_q; - logic lsu_req_n, lsu_req_q; - logic lsu_is_store_n, lsu_is_store_q; - logic dtlb_hit_n, dtlb_hit_q; - logic dtlb_is_2M_n, dtlb_is_2M_q; - logic dtlb_is_1G_n, dtlb_is_1G_q; - - // check if we need to do translation or if we are always ready (e.g.: we are not translating anything) - assign lsu_dtlb_hit_o = (en_ld_st_translation_i) ? dtlb_lu_hit : 1'b1; - - // Wires to PMP checks - riscv::pmp_access_t pmp_access_type; - logic pmp_data_allow; - localparam PPNWMin = (riscv::PPNW - 1 > 29) ? 29 : riscv::PPNW - 1; - // The data interface is simpler and only consists of a request/response interface - always_comb begin : data_interface - // save request and DTLB response - lsu_vaddr_n = lsu_vaddr_i; - lsu_req_n = lsu_req_i; - misaligned_ex_n = misaligned_ex_i; - dtlb_pte_n = dtlb_content; - dtlb_hit_n = dtlb_lu_hit; - lsu_is_store_n = lsu_is_store_i; - dtlb_is_2M_n = dtlb_is_2M; - dtlb_is_1G_n = dtlb_is_1G; - - lsu_paddr_o = lsu_vaddr_q[riscv::PLEN-1:0]; - lsu_dtlb_ppn_o = lsu_vaddr_n[riscv::PLEN-1:12]; - lsu_valid_o = lsu_req_q; - lsu_exception_o = misaligned_ex_q; - pmp_access_type = lsu_is_store_q ? riscv::ACCESS_WRITE : riscv::ACCESS_READ; - - // mute misaligned exceptions if there is no request otherwise they will throw accidental exceptions - misaligned_ex_n.valid = misaligned_ex_i.valid & lsu_req_i; - - // Check if the User flag is set, then we may only access it in supervisor mode - // if SUM is enabled - daccess_err = en_ld_st_translation_i && ((ld_st_priv_lvl_i == riscv::PRIV_LVL_S && !sum_i && dtlb_pte_q.u) || // SUM is not set and we are trying to access a user page in supervisor mode - (ld_st_priv_lvl_i == riscv::PRIV_LVL_U && !dtlb_pte_q.u)); // this is not a user page but we are in user mode and trying to access it - // translation is enabled and no misaligned exception occurred - if (en_ld_st_translation_i && !misaligned_ex_q.valid) begin - lsu_valid_o = 1'b0; - // 4K page - lsu_paddr_o = {dtlb_pte_q.ppn, lsu_vaddr_q[11:0]}; - lsu_dtlb_ppn_o = dtlb_content.ppn; - // Mega page - if (dtlb_is_2M_q) begin - lsu_paddr_o[20:12] = lsu_vaddr_q[20:12]; - lsu_dtlb_ppn_o[20:12] = lsu_vaddr_n[20:12]; - end - // Giga page - if (dtlb_is_1G_q) begin - lsu_paddr_o[PPNWMin:12] = lsu_vaddr_q[PPNWMin:12]; - lsu_dtlb_ppn_o[PPNWMin:12] = lsu_vaddr_n[PPNWMin:12]; - end - // --------- - // DTLB Hit - // -------- - if (dtlb_hit_q && lsu_req_q) begin - lsu_valid_o = 1'b1; - // exception priority: - // PAGE_FAULTS have higher priority than ACCESS_FAULTS - // virtual memory based exceptions are PAGE_FAULTS - // physical memory based exceptions are ACCESS_FAULTS (PMA/PMP) - - // this is a store - if (lsu_is_store_q) begin - // check if the page is write-able and we are not violating privileges - // also check if the dirty flag is set - if (!dtlb_pte_q.w || daccess_err || !dtlb_pte_q.d) begin - lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q - }; - // Check if any PMPs are violated - end else if (!pmp_data_allow) begin - lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q - }; - end - - // this is a load - end else begin - // check for sufficient access privileges - throw a page fault if necessary - if (daccess_err) begin - lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q - }; - // Check if any PMPs are violated - end else if (!pmp_data_allow) begin - lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, lsu_vaddr_q - }; - end - end - end else - - // --------- - // DTLB Miss - // --------- - // watch out for exceptions - if (ptw_active && !walking_instr) begin - // page table walker threw an exception - if (ptw_error) begin - // an error makes the translation valid - lsu_valid_o = 1'b1; - // the page table walker can only throw page faults - if (lsu_is_store_q) begin - lsu_exception_o.cause = riscv::STORE_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr - }; - end else begin - lsu_exception_o.cause = riscv::LOAD_PAGE_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = { - {riscv::XLEN - riscv::VLEN{lsu_vaddr_q[riscv::VLEN-1]}}, update_vaddr - }; - end - end - - if (ptw_access_exception) begin - // an error makes the translation valid - lsu_valid_o = 1'b1; - // Any fault of the page table walk should be based of the original access type - if (lsu_is_store_q) begin - lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, lsu_vaddr_n}; - end else begin - lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) - lsu_exception_o.tval = {{riscv::XLEN - riscv::VLEN{1'b0}}, lsu_vaddr_n}; - end - end - end - end // If translation is not enabled, check the paddr immediately against PMPs - else if (lsu_req_q && !misaligned_ex_q.valid && !pmp_data_allow) begin - if (lsu_is_store_q) begin - lsu_exception_o.cause = riscv::ST_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = {{riscv::XLEN - riscv::PLEN{1'b0}}, lsu_paddr_o}; - end else begin - lsu_exception_o.cause = riscv::LD_ACCESS_FAULT; - lsu_exception_o.valid = 1'b1; - if (CVA6Cfg.TvalEn) lsu_exception_o.tval = {{riscv::XLEN - riscv::PLEN{1'b0}}, lsu_paddr_o}; - end - end - end - - // Load/store PMP check - pmp #( - .CVA6Cfg (CVA6Cfg), - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_data ( - .addr_i (lsu_paddr_o), - .priv_lvl_i (ld_st_priv_lvl_i), - .access_type_i(pmp_access_type), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (pmp_data_allow) - ); - - // ---------- - // Registers - // ---------- - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - lsu_vaddr_q <= '0; - lsu_req_q <= '0; - misaligned_ex_q <= '0; - dtlb_pte_q <= '0; - dtlb_hit_q <= '0; - lsu_is_store_q <= '0; - dtlb_is_2M_q <= '0; - dtlb_is_1G_q <= '0; - end else begin - lsu_vaddr_q <= lsu_vaddr_n; - lsu_req_q <= lsu_req_n; - misaligned_ex_q <= misaligned_ex_n; - dtlb_pte_q <= dtlb_pte_n; - dtlb_hit_q <= dtlb_hit_n; - lsu_is_store_q <= lsu_is_store_n; - dtlb_is_2M_q <= dtlb_is_2M_n; - dtlb_is_1G_q <= dtlb_is_1G_n; - end - end -endmodule diff --git a/core/mmu_sv39/ptw.sv b/core/mmu_sv39/ptw.sv deleted file mode 100644 index 2d0e3780ac..0000000000 --- a/core/mmu_sv39/ptw.sv +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2018 ETH Zurich and University of Bologna. -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: David Schaffenrath, TU Graz -// Author: Florian Zaruba, ETH Zurich -// Date: 24.4.2017 -// Description: Hardware-PTW - -/* verilator lint_off WIDTH */ - -module ptw - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int ASID_WIDTH = 1 -) ( - input logic clk_i, // Clock - input logic rst_ni, // Asynchronous reset active low - input logic flush_i, // flush everything, we need to do this because - // actually everything we do is speculative at this stage - // e.g.: there could be a CSR instruction that changes everything - output logic ptw_active_o, - output logic walking_instr_o, // set when walking for TLB - output logic ptw_error_o, // set when an error occurred - output logic ptw_access_exception_o, // set when an PMP access exception occured - input logic enable_translation_i, // CSRs indicate to enable SV39 - input logic en_ld_st_translation_i, // enable virtual memory translation for load/stores - - input logic lsu_is_store_i, // this translation was triggered by a store - // PTW memory interface - input dcache_req_o_t req_port_i, - output dcache_req_i_t req_port_o, - - - // to TLBs, update logic - output tlb_update_t itlb_update_o, - output tlb_update_t dtlb_update_o, - - output logic [riscv::VLEN-1:0] update_vaddr_o, - - input logic [ ASID_WIDTH-1:0] asid_i, - // from TLBs - // did we miss? - input logic itlb_access_i, - input logic itlb_hit_i, - input logic [riscv::VLEN-1:0] itlb_vaddr_i, - - input logic dtlb_access_i, - input logic dtlb_hit_i, - input logic [riscv::VLEN-1:0] dtlb_vaddr_i, - // from CSR file - input logic [riscv::PPNW-1:0] satp_ppn_i, // ppn from satp - input logic mxr_i, - // Performance counters - output logic itlb_miss_o, - output logic dtlb_miss_o, - // PMP - - input riscv::pmpcfg_t [15:0] pmpcfg_i, - input logic [15:0][riscv::PLEN-3:0] pmpaddr_i, - output logic [riscv::PLEN-1:0] bad_paddr_o - -); - - // input registers - logic data_rvalid_q; - logic [63:0] data_rdata_q; - - riscv::pte_t pte; - assign pte = riscv::pte_t'(data_rdata_q); - - enum logic [2:0] { - IDLE, - WAIT_GRANT, - PTE_LOOKUP, - WAIT_RVALID, - PROPAGATE_ERROR, - PROPAGATE_ACCESS_ERROR - } - state_q, state_d; - - // SV39 defines three levels of page tables - enum logic [1:0] { - LVL1, - LVL2, - LVL3 - } - ptw_lvl_q, ptw_lvl_n; - - // is this an instruction page table walk? - logic is_instr_ptw_q, is_instr_ptw_n; - logic global_mapping_q, global_mapping_n; - // latched tag signal - logic tag_valid_n, tag_valid_q; - // register the ASID - logic [ASID_WIDTH-1:0] tlb_update_asid_q, tlb_update_asid_n; - // register the VPN we need to walk, SV39 defines a 39 bit virtual address - logic [riscv::VLEN-1:0] vaddr_q, vaddr_n; - // 4 byte aligned physical pointer - logic [riscv::PLEN-1:0] ptw_pptr_q, ptw_pptr_n; - - // Assignments - assign update_vaddr_o = vaddr_q; - - assign ptw_active_o = (state_q != IDLE); - assign walking_instr_o = is_instr_ptw_q; - // directly output the correct physical address - assign req_port_o.address_index = ptw_pptr_q[DCACHE_INDEX_WIDTH-1:0]; - assign req_port_o.address_tag = ptw_pptr_q[DCACHE_INDEX_WIDTH+DCACHE_TAG_WIDTH-1:DCACHE_INDEX_WIDTH]; - // we are never going to kill this request - assign req_port_o.kill_req = '0; - // we are never going to write with the HPTW - assign req_port_o.data_wdata = 64'b0; - // we only issue one single request at a time - assign req_port_o.data_id = '0; - // ----------- - // TLB Update - // ----------- - assign itlb_update_o.vpn = {{39 - riscv::SV{1'b0}}, vaddr_q[riscv::SV-1:12]}; - assign dtlb_update_o.vpn = {{39 - riscv::SV{1'b0}}, vaddr_q[riscv::SV-1:12]}; - // update the correct page table level - assign itlb_update_o.is_2M = (ptw_lvl_q == LVL2); - assign itlb_update_o.is_1G = (ptw_lvl_q == LVL1); - assign dtlb_update_o.is_2M = (ptw_lvl_q == LVL2); - assign dtlb_update_o.is_1G = (ptw_lvl_q == LVL1); - // output the correct ASID - assign itlb_update_o.asid = tlb_update_asid_q; - assign dtlb_update_o.asid = tlb_update_asid_q; - // set the global mapping bit - assign itlb_update_o.content = pte | (global_mapping_q << 5); - assign dtlb_update_o.content = pte | (global_mapping_q << 5); - - assign req_port_o.tag_valid = tag_valid_q; - - logic allow_access; - - assign bad_paddr_o = ptw_access_exception_o ? ptw_pptr_q : 'b0; - - pmp #( - .CVA6Cfg (CVA6Cfg), - .PLEN (riscv::PLEN), - .PMP_LEN (riscv::PLEN - 2), - .NR_ENTRIES(CVA6Cfg.NrPMPEntries) - ) i_pmp_ptw ( - .addr_i (ptw_pptr_q), - // PTW access are always checked as if in S-Mode... - .priv_lvl_i (riscv::PRIV_LVL_S), - // ...and they are always loads - .access_type_i(riscv::ACCESS_READ), - // Configuration - .conf_addr_i (pmpaddr_i), - .conf_i (pmpcfg_i), - .allow_o (allow_access) - ); - - //------------------- - // Page table walker - //------------------- - // A virtual address va is translated into a physical address pa as follows: - // 1. Let a be sptbr.ppn × PAGESIZE, and let i = LEVELS-1. (For Sv39, - // PAGESIZE=2^12 and LEVELS=3.) - // 2. Let pte be the value of the PTE at address a+va.vpn[i]×PTESIZE. (For - // Sv32, PTESIZE=4.) - // 3. If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise an access - // exception. - // 4. Otherwise, the PTE is valid. If pte.r = 1 or pte.x = 1, go to step 5. - // Otherwise, this PTE is a pointer to the next level of the page table. - // Let i=i-1. If i < 0, stop and raise an access exception. Otherwise, let - // a = pte.ppn × PAGESIZE and go to step 2. - // 5. A leaf PTE has been found. Determine if the requested memory access - // is allowed by the pte.r, pte.w, and pte.x bits. If not, stop and - // raise an access exception. Otherwise, the translation is successful. - // Set pte.a to 1, and, if the memory access is a store, set pte.d to 1. - // The translated physical address is given as follows: - // - pa.pgoff = va.pgoff. - // - If i > 0, then this is a superpage translation and - // pa.ppn[i-1:0] = va.vpn[i-1:0]. - // - pa.ppn[LEVELS-1:i] = pte.ppn[LEVELS-1:i]. - always_comb begin : ptw - // default assignments - // PTW memory interface - tag_valid_n = 1'b0; - req_port_o.data_req = 1'b0; - req_port_o.data_be = 8'hFF; - req_port_o.data_size = 2'b11; - req_port_o.data_we = 1'b0; - ptw_error_o = 1'b0; - ptw_access_exception_o = 1'b0; - itlb_update_o.valid = 1'b0; - dtlb_update_o.valid = 1'b0; - is_instr_ptw_n = is_instr_ptw_q; - ptw_lvl_n = ptw_lvl_q; - ptw_pptr_n = ptw_pptr_q; - state_d = state_q; - global_mapping_n = global_mapping_q; - // input registers - tlb_update_asid_n = tlb_update_asid_q; - vaddr_n = vaddr_q; - - itlb_miss_o = 1'b0; - dtlb_miss_o = 1'b0; - - case (state_q) - - IDLE: begin - // by default we start with the top-most page table - ptw_lvl_n = LVL1; - global_mapping_n = 1'b0; - is_instr_ptw_n = 1'b0; - // if we got an ITLB miss - if (enable_translation_i & itlb_access_i & ~itlb_hit_i & ~dtlb_access_i) begin - ptw_pptr_n = {satp_ppn_i, itlb_vaddr_i[riscv::SV-1:30], 3'b0}; - is_instr_ptw_n = 1'b1; - tlb_update_asid_n = asid_i; - vaddr_n = itlb_vaddr_i; - state_d = WAIT_GRANT; - itlb_miss_o = 1'b1; - // we got an DTLB miss - end else if (en_ld_st_translation_i & dtlb_access_i & ~dtlb_hit_i) begin - ptw_pptr_n = {satp_ppn_i, dtlb_vaddr_i[riscv::SV-1:30], 3'b0}; - tlb_update_asid_n = asid_i; - vaddr_n = dtlb_vaddr_i; - state_d = WAIT_GRANT; - dtlb_miss_o = 1'b1; - end - end - - WAIT_GRANT: begin - // send a request out - req_port_o.data_req = 1'b1; - // wait for the WAIT_GRANT - if (req_port_i.data_gnt) begin - // send the tag valid signal one cycle later - tag_valid_n = 1'b1; - state_d = PTE_LOOKUP; - end - end - - PTE_LOOKUP: begin - // we wait for the valid signal - if (data_rvalid_q) begin - - // check if the global mapping bit is set - if (pte.g) global_mapping_n = 1'b1; - - // ------------- - // Invalid PTE - // ------------- - // If pte.v = 0, or if pte.r = 0 and pte.w = 1, stop and raise a page-fault exception. - if (!pte.v || (!pte.r && pte.w)) state_d = PROPAGATE_ERROR; - // ----------- - // Valid PTE - // ----------- - else begin - state_d = IDLE; - // it is a valid PTE - // if pte.r = 1 or pte.x = 1 it is a valid PTE - if (pte.r || pte.x) begin - // Valid translation found (either 1G, 2M or 4K entry) - if (is_instr_ptw_q) begin - // ------------ - // Update ITLB - // ------------ - // If page is not executable, we can directly raise an error. This - // doesn't put a useless entry into the TLB. The same idea applies - // to the access flag since we let the access flag be managed by SW. - if (!pte.x || !pte.a) state_d = PROPAGATE_ERROR; - else itlb_update_o.valid = 1'b1; - - end else begin - // ------------ - // Update DTLB - // ------------ - // Check if the access flag has been set, otherwise throw a page-fault - // and let the software handle those bits. - // If page is not readable (there are no write-only pages) - // we can directly raise an error. This doesn't put a useless - // entry into the TLB. - if (pte.a && (pte.r || (pte.x && mxr_i))) begin - dtlb_update_o.valid = 1'b1; - end else begin - state_d = PROPAGATE_ERROR; - end - // Request is a store: perform some additional checks - // If the request was a store and the page is not write-able, raise an error - // the same applies if the dirty flag is not set - if (lsu_is_store_i && (!pte.w || !pte.d)) begin - dtlb_update_o.valid = 1'b0; - state_d = PROPAGATE_ERROR; - end - end - // check if the ppn is correctly aligned: - // 6. If i > 0 and pa.ppn[i − 1 : 0] != 0, this is a misaligned superpage; stop and raise a page-fault - // exception. - if (ptw_lvl_q == LVL1 && pte.ppn[17:0] != '0) begin - state_d = PROPAGATE_ERROR; - dtlb_update_o.valid = 1'b0; - itlb_update_o.valid = 1'b0; - end else if (ptw_lvl_q == LVL2 && pte.ppn[8:0] != '0) begin - state_d = PROPAGATE_ERROR; - dtlb_update_o.valid = 1'b0; - itlb_update_o.valid = 1'b0; - end - // this is a pointer to the next TLB level - end else begin - // pointer to next level of page table - if (ptw_lvl_q == LVL1) begin - // we are in the second level now - ptw_lvl_n = LVL2; - ptw_pptr_n = {pte.ppn, vaddr_q[29:21], 3'b0}; - end - - if (ptw_lvl_q == LVL2) begin - // here we received a pointer to the third level - ptw_lvl_n = LVL3; - ptw_pptr_n = {pte.ppn, vaddr_q[20:12], 3'b0}; - end - - state_d = WAIT_GRANT; - - if (ptw_lvl_q == LVL3) begin - // Should already be the last level page table => Error - ptw_lvl_n = LVL3; - state_d = PROPAGATE_ERROR; - end - end - end - - // Check if this access was actually allowed from a PMP perspective - if (!allow_access) begin - itlb_update_o.valid = 1'b0; - dtlb_update_o.valid = 1'b0; - // we have to return the failed address in bad_addr - ptw_pptr_n = ptw_pptr_q; - state_d = PROPAGATE_ACCESS_ERROR; - end - end - // we've got a data WAIT_GRANT so tell the cache that the tag is valid - end - // Propagate error to MMU/LSU - PROPAGATE_ERROR: begin - state_d = IDLE; - ptw_error_o = 1'b1; - end - PROPAGATE_ACCESS_ERROR: begin - state_d = IDLE; - ptw_access_exception_o = 1'b1; - end - // wait for the rvalid before going back to IDLE - WAIT_RVALID: begin - if (data_rvalid_q) state_d = IDLE; - end - default: begin - state_d = IDLE; - end - endcase - - // ------- - // Flush - // ------- - // should we have flushed before we got an rvalid, wait for it until going back to IDLE - if (flush_i) begin - // on a flush check whether we are - // 1. in the PTE Lookup check whether we still need to wait for an rvalid - // 2. waiting for a grant, if so: wait for it - // if not, go back to idle - if (((state_q inside {PTE_LOOKUP, WAIT_RVALID}) && !data_rvalid_q) || - ((state_q == WAIT_GRANT) && req_port_i.data_gnt)) - state_d = WAIT_RVALID; - else state_d = IDLE; - end - end - - // sequential process - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - state_q <= IDLE; - is_instr_ptw_q <= 1'b0; - ptw_lvl_q <= LVL1; - tag_valid_q <= 1'b0; - tlb_update_asid_q <= '0; - vaddr_q <= '0; - ptw_pptr_q <= '0; - global_mapping_q <= 1'b0; - data_rdata_q <= '0; - data_rvalid_q <= 1'b0; - end else begin - state_q <= state_d; - ptw_pptr_q <= ptw_pptr_n; - is_instr_ptw_q <= is_instr_ptw_n; - ptw_lvl_q <= ptw_lvl_n; - tag_valid_q <= tag_valid_n; - tlb_update_asid_q <= tlb_update_asid_n; - vaddr_q <= vaddr_n; - global_mapping_q <= global_mapping_n; - data_rdata_q <= req_port_i.data_rdata; - data_rvalid_q <= req_port_i.data_rvalid; - end - end - -endmodule -/* verilator lint_on WIDTH */ diff --git a/core/mmu_sv39/tlb.sv b/core/mmu_sv39/tlb.sv deleted file mode 100644 index 3df2cb0173..0000000000 --- a/core/mmu_sv39/tlb.sv +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2018 ETH Zurich and University of Bologna. -// Copyright and related rights are licensed under the Solderpad Hardware -// License, Version 0.51 (the "License"); you may not use this file except in -// compliance with the License. You may obtain a copy of the License at -// http://solderpad.org/licenses/SHL-0.51. Unless required by applicable law -// or agreed to in writing, software, hardware and materials distributed under -// this License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// -// Author: David Schaffenrath, TU Graz -// Author: Florian Zaruba, ETH Zurich -// Date: 21.4.2017 -// Description: Translation Lookaside Buffer, SV39 -// fully set-associative - - -module tlb - import ariane_pkg::*; -#( - parameter config_pkg::cva6_cfg_t CVA6Cfg = config_pkg::cva6_cfg_empty, - parameter int unsigned TLB_ENTRIES = 4, - parameter int unsigned ASID_WIDTH = 1 -) ( - input logic clk_i, // Clock - input logic rst_ni, // Asynchronous reset active low - input logic flush_i, // Flush signal - // Update TLB - input tlb_update_t update_i, - // Lookup signals - input logic lu_access_i, - input logic [ ASID_WIDTH-1:0] lu_asid_i, - input logic [riscv::VLEN-1:0] lu_vaddr_i, - output riscv::pte_t lu_content_o, - input logic [ ASID_WIDTH-1:0] asid_to_be_flushed_i, - input logic [riscv::VLEN-1:0] vaddr_to_be_flushed_i, - output logic lu_is_2M_o, - output logic lu_is_1G_o, - output logic lu_hit_o -); - - // SV39 defines three levels of page tables - struct packed { - logic [ASID_WIDTH-1:0] asid; - logic [riscv::VPN2:0] vpn2; - logic [8:0] vpn1; - logic [8:0] vpn0; - logic is_2M; - logic is_1G; - logic valid; - } [TLB_ENTRIES-1:0] - tags_q, tags_n; - - riscv::pte_t [TLB_ENTRIES-1:0] content_q, content_n; - logic [8:0] vpn0, vpn1; - logic [ riscv::VPN2:0] vpn2; - logic [TLB_ENTRIES-1:0] lu_hit; // to replacement logic - logic [TLB_ENTRIES-1:0] replace_en; // replace the following entry, set by replacement strategy - //------------- - // Translation - //------------- - always_comb begin : translation - vpn0 = lu_vaddr_i[20:12]; - vpn1 = lu_vaddr_i[29:21]; - vpn2 = lu_vaddr_i[30+riscv::VPN2:30]; - - // default assignment - lu_hit = '{default: 0}; - lu_hit_o = 1'b0; - lu_content_o = '{default: 0}; - lu_is_1G_o = 1'b0; - lu_is_2M_o = 1'b0; - - for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - // first level match, this may be a giga page, check the ASID flags as well - // if the entry is associated to a global address, don't match the ASID (ASID is don't care) - if (tags_q[i].valid && ((lu_asid_i == tags_q[i].asid) || content_q[i].g) && vpn2 == tags_q[i].vpn2) begin - // second level - if (tags_q[i].is_1G) begin - lu_is_1G_o = 1'b1; - lu_content_o = content_q[i]; - lu_hit_o = 1'b1; - lu_hit[i] = 1'b1; - // not a giga page hit so check further - end else if (vpn1 == tags_q[i].vpn1) begin - // this could be a 2 mega page hit or a 4 kB hit - // output accordingly - if (tags_q[i].is_2M || vpn0 == tags_q[i].vpn0) begin - lu_is_2M_o = tags_q[i].is_2M; - lu_content_o = content_q[i]; - lu_hit_o = 1'b1; - lu_hit[i] = 1'b1; - end - end - end - end - end - - - - logic asid_to_be_flushed_is0; // indicates that the ASID provided by SFENCE.VMA (rs2)is 0, active high - logic vaddr_to_be_flushed_is0; // indicates that the VADDR provided by SFENCE.VMA (rs1)is 0, active high - logic [TLB_ENTRIES-1:0] vaddr_vpn0_match; - logic [TLB_ENTRIES-1:0] vaddr_vpn1_match; - logic [TLB_ENTRIES-1:0] vaddr_vpn2_match; - - assign asid_to_be_flushed_is0 = ~(|asid_to_be_flushed_i); - assign vaddr_to_be_flushed_is0 = ~(|vaddr_to_be_flushed_i); - - // ------------------ - // Update and Flush - // ------------------ - always_comb begin : update_flush - tags_n = tags_q; - content_n = content_q; - - for (int unsigned i = 0; i < TLB_ENTRIES; i++) begin - - vaddr_vpn0_match[i] = (vaddr_to_be_flushed_i[20:12] == tags_q[i].vpn0); - vaddr_vpn1_match[i] = (vaddr_to_be_flushed_i[29:21] == tags_q[i].vpn1); - vaddr_vpn2_match[i] = (vaddr_to_be_flushed_i[30+riscv::VPN2:30] == tags_q[i].vpn2); - - if (flush_i) begin - // invalidate logic - // flush everything if ASID is 0 and vaddr is 0 ("SFENCE.VMA x0 x0" case) - if (asid_to_be_flushed_is0 && vaddr_to_be_flushed_is0) tags_n[i].valid = 1'b0; - // flush vaddr in all addressing space ("SFENCE.VMA vaddr x0" case), it should happen only for leaf pages - else if (asid_to_be_flushed_is0 && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_2M) ) && (~vaddr_to_be_flushed_is0)) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global and asid and vaddr both matches with the entry to be flushed ("SFENCE.VMA vaddr asid" case) - else if ((!content_q[i].g) && ((vaddr_vpn0_match[i] && vaddr_vpn1_match[i] && vaddr_vpn2_match[i]) || (vaddr_vpn2_match[i] && tags_q[i].is_1G) || (vaddr_vpn1_match[i] && vaddr_vpn2_match[i] && tags_q[i].is_2M)) && (asid_to_be_flushed_i == tags_q[i].asid) && (!vaddr_to_be_flushed_is0) && (!asid_to_be_flushed_is0)) - tags_n[i].valid = 1'b0; - // the entry is flushed if it's not global, and the asid matches and vaddr is 0. ("SFENCE.VMA 0 asid" case) - else if ((!content_q[i].g) && (vaddr_to_be_flushed_is0) && (asid_to_be_flushed_i == tags_q[i].asid) && (!asid_to_be_flushed_is0)) - tags_n[i].valid = 1'b0; - // normal replacement - end else if (update_i.valid & replace_en[i]) begin - // update tag array - tags_n[i] = '{ - asid: update_i.asid, - vpn2: update_i.vpn[18+riscv::VPN2:18], - vpn1: update_i.vpn[17:9], - vpn0: update_i.vpn[8:0], - is_1G: update_i.is_1G, - is_2M: update_i.is_2M, - valid: 1'b1 - }; - // and content as well - content_n[i] = update_i.content; - end - end - end - - // ----------------------------------------------- - // PLRU - Pseudo Least Recently Used Replacement - // ----------------------------------------------- - logic [2*(TLB_ENTRIES-1)-1:0] plru_tree_q, plru_tree_n; - always_comb begin : plru_replacement - plru_tree_n = plru_tree_q; - // The PLRU-tree indexing: - // lvl0 0 - // / \ - // / \ - // lvl1 1 2 - // / \ / \ - // lvl2 3 4 5 6 - // / \ /\/\ /\ - // ... ... ... ... - // Just predefine which nodes will be set/cleared - // E.g. for a TLB with 8 entries, the for-loop is semantically - // equivalent to the following pseudo-code: - // unique case (1'b1) - // lu_hit[7]: plru_tree_n[0, 2, 6] = {1, 1, 1}; - // lu_hit[6]: plru_tree_n[0, 2, 6] = {1, 1, 0}; - // lu_hit[5]: plru_tree_n[0, 2, 5] = {1, 0, 1}; - // lu_hit[4]: plru_tree_n[0, 2, 5] = {1, 0, 0}; - // lu_hit[3]: plru_tree_n[0, 1, 4] = {0, 1, 1}; - // lu_hit[2]: plru_tree_n[0, 1, 4] = {0, 1, 0}; - // lu_hit[1]: plru_tree_n[0, 1, 3] = {0, 0, 1}; - // lu_hit[0]: plru_tree_n[0, 1, 3] = {0, 0, 0}; - // default: begin /* No hit */ end - // endcase - for ( - int unsigned i = 0; i < TLB_ENTRIES; i++ - ) begin - automatic int unsigned idx_base, shift, new_index; - // we got a hit so update the pointer as it was least recently used - if (lu_hit[i] & lu_access_i) begin - // Set the nodes to the values we would expect - for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin - idx_base = $unsigned((2 ** lvl) - 1); - // lvl0 <=> MSB, lvl1 <=> MSB-1, ... - shift = $clog2(TLB_ENTRIES) - lvl; - // to circumvent the 32 bit integer arithmetic assignment - new_index = ~((i >> (shift - 1)) & 32'b1); - plru_tree_n[idx_base+(i>>shift)] = new_index[0]; - end - end - end - // Decode tree to write enable signals - // Next for-loop basically creates the following logic for e.g. an 8 entry - // TLB (note: pseudo-code obviously): - // replace_en[7] = &plru_tree_q[ 6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,1} - // replace_en[6] = &plru_tree_q[~6, 2, 0]; //plru_tree_q[0,2,6]=={1,1,0} - // replace_en[5] = &plru_tree_q[ 5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,1} - // replace_en[4] = &plru_tree_q[~5,~2, 0]; //plru_tree_q[0,2,5]=={1,0,0} - // replace_en[3] = &plru_tree_q[ 4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,1} - // replace_en[2] = &plru_tree_q[~4, 1,~0]; //plru_tree_q[0,1,4]=={0,1,0} - // replace_en[1] = &plru_tree_q[ 3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,1} - // replace_en[0] = &plru_tree_q[~3,~1,~0]; //plru_tree_q[0,1,3]=={0,0,0} - // For each entry traverse the tree. If every tree-node matches, - // the corresponding bit of the entry's index, this is - // the next entry to replace. - for (int unsigned i = 0; i < TLB_ENTRIES; i += 1) begin - automatic logic en; - automatic int unsigned idx_base, shift, new_index; - en = 1'b1; - for (int unsigned lvl = 0; lvl < $clog2(TLB_ENTRIES); lvl++) begin - idx_base = $unsigned((2 ** lvl) - 1); - // lvl0 <=> MSB, lvl1 <=> MSB-1, ... - shift = $clog2(TLB_ENTRIES) - lvl; - - // en &= plru_tree_q[idx_base + (i>>shift)] == ((i >> (shift-1)) & 1'b1); - new_index = (i >> (shift - 1)) & 32'b1; - if (new_index[0]) begin - en &= plru_tree_q[idx_base+(i>>shift)]; - end else begin - en &= ~plru_tree_q[idx_base+(i>>shift)]; - end - end - replace_en[i] = en; - end - end - - // sequential process - always_ff @(posedge clk_i or negedge rst_ni) begin - if (~rst_ni) begin - tags_q <= '{default: 0}; - content_q <= '{default: 0}; - plru_tree_q <= '{default: 0}; - end else begin - tags_q <= tags_n; - content_q <= content_n; - plru_tree_q <= plru_tree_n; - end - end - //-------------- - // Sanity checks - //-------------- - - //pragma translate_off -`ifndef VERILATOR - - initial begin : p_assertions - assert ((TLB_ENTRIES % 2 == 0) && (TLB_ENTRIES > 1)) - else begin - $error("TLB size must be a multiple of 2 and greater than 1"); - $stop(); - end - assert (ASID_WIDTH >= 1) - else begin - $error("ASID width must be at least 1"); - $stop(); - end - end - - // Just for checking - function int countSetBits(logic [TLB_ENTRIES-1:0] vector); - automatic int count = 0; - foreach (vector[idx]) begin - count += vector[idx]; - end - return count; - endfunction - - assert property (@(posedge clk_i) (countSetBits(lu_hit) <= 1)) - else begin - $error("More then one hit in TLB!"); - $stop(); - end - assert property (@(posedge clk_i) (countSetBits(replace_en) <= 1)) - else begin - $error("More then one TLB entry selected for next replace!"); - $stop(); - end - -`endif - //pragma translate_on - -endmodule diff --git a/src_files.yml b/src_files.yml index 84173c67ca..1a8a206d0b 100644 --- a/src_files.yml +++ b/src_files.yml @@ -34,8 +34,7 @@ ariane: src/load_unit.sv, src/load_store_unit.sv, src/miss_handler.sv, - src/mmu_sv39/mmu.sv, - src/mmu_sv32/cva6_mmu_sv32.sv, + src/cva6_mmu/cva6_mmu.sv, src/mult.sv, src/nbdcache.sv, src/vdregs.sv, @@ -43,14 +42,13 @@ ariane: src/sram_wrapper.sv, src/pcgen_stage.sv, src/perf_counters.sv, - src/mmu_sv39/ptw.sv, - src/mmu_sv32/cva6_ptw_sv32.sv, + src/cva6_mmu/cva6_ptw.sv, src/re_name.sv, src/scoreboard.sv, src/store_buffer.sv, src/store_unit.sv, - src/mmu_sv39/tlb.sv, - src/mmu_sv32/cva6_tlb_sv32.sv, + src/cva6_mmu/cva6_tlb.sv, + src/cva6_mmu/cva6_shared_tlb.sv, src/acc_dispatcher.sv, src/debug/dm_csrs.sv, src/debug/dm_mem.sv,