/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2026 Ruslan Bukin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _RISCV_IOMMU_IOMMU_H_ #define _RISCV_IOMMU_IOMMU_H_ #define RISCV_IOMMU_CAPABILITIES 0x0000 #define CAPABILITIES_VERSION_S 0 #define CAPABILITIES_VERSION_M (0xff << CAPABILITIES_VERSION_S) #define CAPABILITIES_SV32 (1 << 8) #define CAPABILITIES_SV39 (1 << 9) #define CAPABILITIES_SV48 (1 << 10) #define CAPABILITIES_SV57 (1 << 11) #define CAPABILITIES_SVPBMT (1 << 15) #define CAPABILITIES_SV32X4 (1 << 16) #define CAPABILITIES_SV39X4 (1 << 17) #define CAPABILITIES_SV48X4 (1 << 18) #define CAPABILITIES_SV57X4 (1 << 19) #define CAPABILITIES_AMO_MRIF (1 << 21) #define CAPABILITIES_MSI_FLAT (1 << 22) #define CAPABILITIES_MSI_MRIF (1 << 23) #define CAPABILITIES_AMO_HWAD (1 << 24) #define CAPABILITIES_ATS (1 << 25) #define CAPABILITIES_T2GPA (1 << 26) #define CAPABILITIES_END (1 << 27) #define CAPABILITIES_IGS_S 28 #define CAPABILITIES_IGS_M (0x3 << CAPABILITIES_IGS_S) #define CAPABILITIES_HPM (1 << 30) #define CAPABILITIES_DBG (1 << 31) #define CAPABILITIES_PAS_S 32ULL #define CAPABILITIES_PAS_M (0x3f << CAPABILITIES_PAS_S) #define CAPABILITIES_PD8 (1ULL << 38) #define CAPABILITIES_PD17 (1ULL << 39) #define CAPABILITIES_PD20 (1ULL << 40) #define RISCV_IOMMU_FCTL 0x0008 #define FCTL_BE (1 << 0) /* Big-endian */ #define FCTL_WSI (1 << 1) /* Wire-signalled Ints. */ #define FCTL_GXL (1 << 2) /* Guest physical addresses */ #define RISCV_IOMMU_DDTP 0x0010 #define DDTP_IOMMU_MODE_S 0 #define DDTP_IOMMU_MODE_OFF (0 << DDTP_IOMMU_MODE_S) #define DDTP_IOMMU_MODE_BARE (1 << DDTP_IOMMU_MODE_S) #define DDTP_IOMMU_MODE_1LVL (2 << DDTP_IOMMU_MODE_S) #define DDTP_IOMMU_MODE_2LVL (3 << DDTP_IOMMU_MODE_S) #define DDTP_IOMMU_MODE_3LVL (4 << DDTP_IOMMU_MODE_S) #define DDTP_BUSY (1 << 4) #define DDTP_PPN_S 10 #define DDTP_PPN_M (0xfffffffffffULL << DDTP_PPN_S) #define RISCV_IOMMU_CQB 0x18 /* Command queue base. */ #define CQB_LOG2SZ_1_S 0 #define CQB_LOG2SZ_1_M (0x3f << CQB_LOG2SZ_1_S) #define CQB_PPN_S 10 #define CQB_PPN_M (0xfffffffffffULL << CQB_PPN_S) #define RISCV_IOMMU_CQH 0x20 #define RISCV_IOMMU_CQT 0x24 #define RISCV_IOMMU_FQB 0x28 /* Fault queue base. */ #define RISCV_IOMMU_FQH 0x30 #define RISCV_IOMMU_FQT 0x34 #define RISCV_IOMMU_PQB 0x38 /* Page queue base. */ #define RISCV_IOMMU_PQH 0x40 #define RISCV_IOMMU_PQT 0x44 #define RISCV_IOMMU_CQCSR 0x48 #define CQCSR_BUSY (1 << 17) /* Write is observed */ #define CQCSR_CQON (1 << 16) /* Active */ #define CQCSR_FENCE_W_IP (1 << 11) /* iofence.c completed */ #define CQCSR_CMD_ILL (1 << 10) /* Illegal command */ #define CQCSR_CMD_TO (1 << 9) /* Timeout */ #define CQCSR_CQMF (1 << 8) /* Memory Fault */ #define CQCSR_CIE (1 << 1) /* Interrupt Enable */ #define CQCSR_CQEN (1 << 0) /* Enable */ #define RISCV_IOMMU_FQCSR 0x4C #define FQCSR_BUSY (1 << 17) /* Write is observed */ #define FQCSR_FQON (1 << 16) /* Active */ #define FQCSR_FQOF (1 << 9) /* Overflow */ #define FQCSR_FQMF (1 << 8) /* Memory Fault */ #define FQCSR_FIE (1 << 1) /* Interrupt Enable */ #define FQCSR_FQEN (1 << 0) /* Enable */ #define RISCV_IOMMU_PQCSR 0x50 #define PQCSR_BUSY (1 << 17) /* Write is observed */ #define PQCSR_PQON (1 << 16) /* Active */ #define PQCSR_PQOF (1 << 9) /* Overflow */ #define PQCSR_PQMF (1 << 8) /* Memory Fault */ #define PQCSR_PIE (1 << 1) /* Interrupt Enable */ #define PQCSR_PQEN (1 << 0) /* Enable */ #define RISCV_IOMMU_IPSR 0x54 #define IPSR_CIP (1 << 0) /* Command queue interrupt pending */ #define IPSR_FIP (1 << 1) /* Fault queue interrupt pending */ #define IPSR_PMIP (1 << 2) /* Performance monitoring int pend */ #define IPSR_PIP (1 << 3) /* Page queue interrupt pending */ #define RISCV_IOMMU_IOCOUNTOVF 0x0058 #define RISCV_IOMMU_IOCOUNTINH 0x005C #define RISCV_IOMMU_IOHPMCYCLES 0x0060 #define RISCV_IOMMU_IOHPMCTR_BASE 0x0068 #define RISCV_IOMMU_IOHPMCTR(_n) (RISCV_IOMMU_IOHPMCTR_BASE + ((_n) * 0x8)) #define RISCV_IOMMU_IOHPMEVT_BASE 0x0160 #define RISCV_IOMMU_IOHPMEVT(_n) (RISCV_IOMMU_IOHPMEVT_BASE + ((_n) * 0x8)) #define RISCV_IOMMU_TR_REQ_IOVA 0x0258 #define RISCV_IOMMU_TR_REQ_CTL 0x0260 #define RISCV_IOMMU_TR_RESPONSE 0x0268 #define RISCV_IOMMU_ICVEC 0x02F8 #define RISCV_IOMMU_LOCK(_sc) mtx_lock(&(_sc)->mtx) #define RISCV_IOMMU_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) DECLARE_CLASS(riscv_iommu_driver); MALLOC_DECLARE(M_IOMMU); struct riscv_iommu_unit { struct iommu_unit iommu; LIST_HEAD(, riscv_iommu_domain) domain_list; LIST_ENTRY(riscv_iommu_unit) next; device_t dev; intptr_t xref; }; struct riscv_iommu_domain { struct iommu_domain iodom; LIST_HEAD(, riscv_iommu_ctx) ctx_list; LIST_ENTRY(riscv_iommu_domain) next; u_int entries_cnt; struct riscv_iommu_cd *cd; struct riscv_iommu_pmap p; uint16_t pscid; }; struct riscv_iommu_ctx { struct iommu_ctx ioctx; struct riscv_iommu_domain *domain; LIST_ENTRY(riscv_iommu_ctx) next; device_t dev; bool bypass; int did; uint16_t vendor; uint16_t device; u_int refcnt; }; struct riscv_iommu_queue_local_copy { union { uint64_t val; struct { uint32_t head; uint32_t tail; }; }; }; struct riscv_iommu_queue { struct riscv_iommu_queue_local_copy lc; vm_paddr_t paddr; void *vaddr; uint64_t mask; uint32_t head_off; uint32_t tail_off; int size_log2; uint64_t base; uint64_t csr; int idx; uint8_t entry_size; }; struct l1_desc { uint8_t span; void *va; vm_paddr_t pa; }; /* Base-format device-context. */ struct riscv_iommu_dc_base { uint64_t tc; /* Translation control */ #define DC_TC_V (1 << 0) #define DC_TC_EN_ATS (1 << 1) #define DC_TC_EN_PRI (1 << 2) #define DC_TC_T2GPA (1 << 3) #define DC_TC_DTF (1 << 4) #define DC_TC_PDTV (1 << 5) #define DC_TC_PRPR (1 << 6) #define DC_TC_GADE (1 << 7) #define DC_TC_SADE (1 << 8) #define DC_TC_DPE (1 << 9) #define DC_TC_SBE (1 << 10) #define DC_TC_SXL (1 << 11) uint64_t iohgatp; /* IO Hyp guest address translation */ uint64_t ta; /* Translation attributes */ #define DC_TA_V (1 << 0) #define DC_TA_ENS (1 << 1) #define DC_TA_SUM (1 << 2) #define DC_TA_PSCID_S 12 #define DC_TA_PSCID_M (0xfffff << DC_TA_PSCID_S) uint64_t fsc; /* First-stage-context */ }; /* Extended-format device-context. */ struct riscv_iommu_dc { struct riscv_iommu_dc_base base; uint64_t msiptp; /* MSI page table pointer */ uint64_t msi_addr_mask; uint64_t msi_addr_pattern; uint64_t _reserved; }; #define DC_NON_LEAF_ENTRY_PPN_S 10 #define DC_NON_LEAF_ENTRY_VALID (1 << 0) struct riscv_iommu_ddt { void *vaddr; uint64_t base; uint32_t base_cfg; uint32_t num_top_entries; struct l1_desc *l1; struct riscv_iommu_dc *dc; }; struct riscv_iommu_softc { device_t dev; intptr_t xref; struct riscv_iommu_unit unit; struct resource *res[5]; void *intr_cookie[4]; struct riscv_iommu_queue cq; struct riscv_iommu_queue fq; struct riscv_iommu_queue pq; struct riscv_iommu_ddt ddt; struct mtx mtx; uint32_t l0_did_bits; uint32_t dc_dwords; /* PSCID management. */ bitstr_t *pscid_set; int pscid_set_size; struct mtx pscid_set_mutex; uint32_t pscid_bits; enum pmap_mode pm_mode; int iommu_mode; }; /* * Command queue request. */ struct riscv_iommu_command { uint64_t dword0; uint64_t dword1; }; enum riscv_iommu_fq_causes { FQ_CAUSE_INST_FAULT = 1, FQ_CAUSE_RD_ADDR_MISALIGNED = 4, FQ_CAUSE_RD_FAULT = 5, FQ_CAUSE_WR_ADDR_MISALIGNED = 6, FQ_CAUSE_WR_FAULT = 7, FQ_CAUSE_INST_FAULT_S = 12, FQ_CAUSE_RD_FAULT_S = 13, FQ_CAUSE_WR_FAULT_S = 15, FQ_CAUSE_INST_FAULT_VS = 20, FQ_CAUSE_RD_FAULT_VS = 21, FQ_CAUSE_WR_FAULT_VS = 23, FQ_CAUSE_DMA_DISABLED = 256, FQ_CAUSE_DDT_LOAD_FAULT = 257, FQ_CAUSE_DDT_INVALID = 258, FQ_CAUSE_DDT_MISCONFIGURED = 259, FQ_CAUSE_TR_TYPE_DISALLOWED = 260, FQ_CAUSE_MSI_LOAD_FAULT = 261, FQ_CAUSE_MSI_INVALID = 262, FQ_CAUSE_MSI_MISCONFIGURED = 263, FQ_CAUSE_MRIF_FAULT = 264, FQ_CAUSE_PDT_LOAD_FAULT = 265, FQ_CAUSE_PDT_INVALID = 266, FQ_CAUSE_PDT_MISCONFIGURED = 267, FQ_CAUSE_DDT_CORRUPTED = 268, FQ_CAUSE_PDT_CORRUPTED = 269, FQ_CAUSE_MSI_PT_CORRUPTED = 270, FQ_CAUSE_MRIF_CORRUPTED = 271, FQ_CAUSE_INTERNAL_DP_ERROR = 272, FQ_CAUSE_MSI_WR_FAULT = 273, FQ_CAUSE_PT_CORRUPTED = 274, }; /* * Fault queue record. */ struct riscv_iommu_fq_record { uint64_t hdr; #define FQR_HDR_CAUSE_S 0 #define FQR_HDR_CAUSE_M (0xfff << FQR_HDR_CAUSE_S) #define FQR_HDR_PID_S 12 #define FQR_HDR_PID_M (0xfffffULL << FQR_HDR_PID_S) #define FQR_HDR_PV (1ULL << 32) #define FQR_HDR_PRIV (1ULL << 33) #define FQR_HDR_TTYP_S 34ULL #define FQR_HDR_TTYP_M (0x3fULL << FQR_HDR_TTYP_S) #define FQR_HDR_DID_S 40ULL #define FQR_HDR_DID_M (0xffffffULL << FQR_HDR_DID_S) uint32_t custom; uint32_t reserved; uint64_t iotval; uint64_t iotval2; }; #define COMMAND_OPCODE_S 0 #define COMMAND_OPCODE_IOTINVAL (1 << COMMAND_OPCODE_S) #define COMMAND_OPCODE_IOFENCE (2 << COMMAND_OPCODE_S) #define COMMAND_OPCODE_IODIR (3 << COMMAND_OPCODE_S) #define COMMAND_OPCODE_ATS (4 << COMMAND_OPCODE_S) #define COMMAND_OPCODE_FUNC_S 7 #define COMMAND_OPCODE_FUNC_M (0x3 << COMMAND_OPCODE_FUNC_S) #define FUNC_IODIR_INVAL_DDT (0 << COMMAND_OPCODE_FUNC_S) #define FUNC_IODIR_INVAL_PDT (1 << COMMAND_OPCODE_FUNC_S) #define FUNC_IODIR_PID_S 12 #define FUNC_IODIR_DV (1ULL << 33) /* DID Valid */ #define FUNC_IODIR_DID_S 40ULL /* dword0 */ #define FUNC_IOTINVAL_VMA (0 << COMMAND_OPCODE_FUNC_S) #define FUNC_IOTINVAL_GVMA (1 << COMMAND_OPCODE_FUNC_S) #define FUNC_IOTINVAL_AV (1 << 10) /* Address Valid */ #define FUNC_IOTINVAL_PSCID_S 12 /* Process-Soft-Context ID */ #define FUNC_IOTINVAL_PSCV (1ULL << 32) /* PSCID Valid */ #define FUNC_IOTINVAL_GV (1ULL << 33) /* GSCID Valid */ #define FUNC_IOTINVAL_GSCID_S 44 /* Guest-Soft-Context ID */ /* dword1 */ #define FUNC_IOTINVAL_ADDR_S 10 #define FUNC_IOFENCE_FUNC_C (0 << 7) #define FUNC_IOFENCE_AV (1 << 10) #define FUNC_IOFENCE_WSI (1 << 11) #define FUNC_IOFENCE_PR (1 << 12) #define FUNC_IOFENCE_PW (1 << 13) #define FUNC_IOFENCE_DATA_S 32ULL int riscv_iommu_attach(device_t dev); struct riscv_iommu_ctx *riscv_iommu_ctx_lookup_by_did(device_t dev, u_int did); #endif /* _RISCV_IOMMU_IOMMU_H_ */