FreeBSD/src
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2026-06-02 11:24:32 +00:00
Code Issues Projects Releases Wiki Activity

riscv: IOMMU support

Browse Source 
Support for RISC-V IOMMU spec v1.0.1 (ratified)
  https://github.com/riscv-non-isa/riscv-iommu

Supports translation for PCI devices only.
Supports 1 or 2-level device-directory-table (DDT).
Supports SV39 and SV48 virtual memory system (on per-device basis).
Supports both "standard" and "extended" device-context (DC) structure.
Supports "bypass" mode to disable translation for a particular device.
Supports WSI (Wire-Signalled Interrupts) only.

This includes both PCI-bus and FDT attachment drivers.

Note in case of PCI-bus attachment, interrupts are not available. In this
case no error report is provided in case of translation fault. Otherwise
interrupts are not needed.

Differential Revision:	https://reviews.freebsd.org/D55922
This commit is contained in:
Ruslan Bukin
2026-05-12 11:09:04 +01:00
parent 6f451c6091
commit bcecad2c24
13 changed files with 3418 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sys/conf/files.riscv
+9
View File
@@ -6,6 +6,7 @@ cddl/dev/fbt/riscv/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${
crypto/des/des_enc.c optional netsmb
dev/cpufreq/cpufreq_dt.c optional cpufreq fdt
dev/ofw/ofw_cpu.c optional fdt
dev/ofw/ofw_pci.c optional pci fdt
dev/ofw/ofw_pcib.c optional pci fdt
dev/pci/pci_dw.c optional pci fdt
dev/pci/pci_dw_if.m optional pci fdt
@@ -42,6 +43,14 @@ libkern/memset.c standard
libkern/strcmp.c standard
libkern/strlen.c standard
libkern/strncmp.c standard
riscv/iommu/iommu_frontend.c standard
riscv/iommu/iommu_if.m standard
riscv/iommu/iommu.c standard
riscv/iommu/iommu_fdt.c optional fdt
riscv/iommu/iommu_pci.c optional pci
riscv/iommu/iommu_pmap.c optional iommu
dev/iommu/busdma_iommu.c optional iommu
dev/iommu/iommu_gas.c optional iommu
riscv/riscv/aplic.c standard
riscv/riscv/autoconf.c standard
riscv/riscv/bus_machdep.c standard
sys/riscv/conf/GENERIC
+1
View File
@@ -77,6 +77,7 @@ options RACCT # Resource accounting framework
options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default
options RCTL # Resource limits
options SMP
options IOMMU
# RISC-V SBI console
device rcons
sys/riscv/include/bus_dma_impl.h
+3
View File
@@ -41,6 +41,7 @@ struct bus_dma_tag_common {
int flags;
bus_dma_lock_t *lockfunc;
void *lockfuncarg;
int domain;
};
struct bus_dma_impl {
@@ -52,6 +53,8 @@ struct bus_dma_impl {
int (*tag_destroy)(bus_dma_tag_t dmat);
int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp);
int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map);
int (*tag_set_domain)(bus_dma_tag_t);
bool (*id_mapped)(bus_dma_tag_t, vm_paddr_t, bus_size_t);
int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags,
bus_dmamap_t *mapp);
void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map);
sys/riscv/include/iommu.h
+10
View File
@@ -0,0 +1,10 @@
/*-
* This file is in the public domain.
*/
#ifndef _MACHINE_IOMMU_H_
#define _MACHINE_IOMMU_H_
#include <riscv/iommu/iommu_frontend.h>
#endif /* !_MACHINE_IOMMU_H_ */
sys/riscv/iommu/iommu.c
+1351
View File
File diff suppressed because it is too large Load Diff
sys/riscv/iommu/iommu.h
+359
View File
@@ -0,0 +1,359 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _RISCV_IOMMU_IOMMU_H_
#define _RISCV_IOMMU_IOMMU_H_
#define RISCV_IOMMU_CAPABILITIES 0x0000
#define CAPABILITIES_VERSION_S 0
#define CAPABILITIES_VERSION_M (0xff << CAPABILITIES_VERSION_S)
#define CAPABILITIES_SV32 (1 << 8)
#define CAPABILITIES_SV39 (1 << 9)
#define CAPABILITIES_SV48 (1 << 10)
#define CAPABILITIES_SV57 (1 << 11)
#define CAPABILITIES_SVPBMT (1 << 15)
#define CAPABILITIES_SV32X4 (1 << 16)
#define CAPABILITIES_SV39X4 (1 << 17)
#define CAPABILITIES_SV48X4 (1 << 18)
#define CAPABILITIES_SV57X4 (1 << 19)
#define CAPABILITIES_AMO_MRIF (1 << 21)
#define CAPABILITIES_MSI_FLAT (1 << 22)
#define CAPABILITIES_MSI_MRIF (1 << 23)
#define CAPABILITIES_AMO_HWAD (1 << 24)
#define CAPABILITIES_ATS (1 << 25)
#define CAPABILITIES_T2GPA (1 << 26)
#define CAPABILITIES_END (1 << 27)
#define CAPABILITIES_IGS_S 28
#define CAPABILITIES_IGS_M (0x3 << CAPABILITIES_IGS_S)
#define CAPABILITIES_HPM (1 << 30)
#define CAPABILITIES_DBG (1 << 31)
#define CAPABILITIES_PAS_S 32ULL
#define CAPABILITIES_PAS_M (0x3f << CAPABILITIES_PAS_S)
#define CAPABILITIES_PD8 (1ULL << 38)
#define CAPABILITIES_PD17 (1ULL << 39)
#define CAPABILITIES_PD20 (1ULL << 40)
#define RISCV_IOMMU_FCTL 0x0008
#define FCTL_BE (1 << 0) /* Big-endian */
#define FCTL_WSI (1 << 1) /* Wire-signalled Ints. */
#define FCTL_GXL (1 << 2) /* Guest physical addresses */
#define RISCV_IOMMU_DDTP 0x0010
#define DDTP_IOMMU_MODE_S 0
#define DDTP_IOMMU_MODE_OFF (0 << DDTP_IOMMU_MODE_S)
#define DDTP_IOMMU_MODE_BARE (1 << DDTP_IOMMU_MODE_S)
#define DDTP_IOMMU_MODE_1LVL (2 << DDTP_IOMMU_MODE_S)
#define DDTP_IOMMU_MODE_2LVL (3 << DDTP_IOMMU_MODE_S)
#define DDTP_IOMMU_MODE_3LVL (4 << DDTP_IOMMU_MODE_S)
#define DDTP_BUSY (1 << 4)
#define DDTP_PPN_S 10
#define DDTP_PPN_M (0xfffffffffffULL << DDTP_PPN_S)
#define RISCV_IOMMU_CQB 0x18 /* Command queue base. */
#define CQB_LOG2SZ_1_S 0
#define CQB_LOG2SZ_1_M (0x3f << CQB_LOG2SZ_1_S)
#define CQB_PPN_S 10
#define CQB_PPN_M (0xfffffffffffULL << CQB_PPN_S)
#define RISCV_IOMMU_CQH 0x20
#define RISCV_IOMMU_CQT 0x24
#define RISCV_IOMMU_FQB 0x28 /* Fault queue base. */
#define RISCV_IOMMU_FQH 0x30
#define RISCV_IOMMU_FQT 0x34
#define RISCV_IOMMU_PQB 0x38 /* Page queue base. */
#define RISCV_IOMMU_PQH 0x40
#define RISCV_IOMMU_PQT 0x44
#define RISCV_IOMMU_CQCSR 0x48
#define CQCSR_BUSY (1 << 17) /* Write is observed */
#define CQCSR_CQON (1 << 16) /* Active */
#define CQCSR_FENCE_W_IP (1 << 11) /* iofence.c completed */
#define CQCSR_CMD_ILL (1 << 10) /* Illegal command */
#define CQCSR_CMD_TO (1 << 9) /* Timeout */
#define CQCSR_CQMF (1 << 8) /* Memory Fault */
#define CQCSR_CIE (1 << 1) /* Interrupt Enable */
#define CQCSR_CQEN (1 << 0) /* Enable */
#define RISCV_IOMMU_FQCSR 0x4C
#define FQCSR_BUSY (1 << 17) /* Write is observed */
#define FQCSR_FQON (1 << 16) /* Active */
#define FQCSR_FQOF (1 << 9) /* Overflow */
#define FQCSR_FQMF (1 << 8) /* Memory Fault */
#define FQCSR_FIE (1 << 1) /* Interrupt Enable */
#define FQCSR_FQEN (1 << 0) /* Enable */
#define RISCV_IOMMU_PQCSR 0x50
#define PQCSR_BUSY (1 << 17) /* Write is observed */
#define PQCSR_PQON (1 << 16) /* Active */
#define PQCSR_PQOF (1 << 9) /* Overflow */
#define PQCSR_PQMF (1 << 8) /* Memory Fault */
#define PQCSR_PIE (1 << 1) /* Interrupt Enable */
#define PQCSR_PQEN (1 << 0) /* Enable */
#define RISCV_IOMMU_IPSR 0x54
#define IPSR_CIP (1 << 0) /* Command queue interrupt pending */
#define IPSR_FIP (1 << 1) /* Fault queue interrupt pending */
#define IPSR_PMIP (1 << 2) /* Performance monitoring int pend */
#define IPSR_PIP (1 << 3) /* Page queue interrupt pending */
#define RISCV_IOMMU_IOCOUNTOVF 0x0058
#define RISCV_IOMMU_IOCOUNTINH 0x005C
#define RISCV_IOMMU_IOHPMCYCLES 0x0060
#define RISCV_IOMMU_IOHPMCTR_BASE 0x0068
#define RISCV_IOMMU_IOHPMCTR(_n) (RISCV_IOMMU_IOHPMCTR_BASE + ((_n) * 0x8))
#define RISCV_IOMMU_IOHPMEVT_BASE 0x0160
#define RISCV_IOMMU_IOHPMEVT(_n) (RISCV_IOMMU_IOHPMEVT_BASE + ((_n) * 0x8))
#define RISCV_IOMMU_TR_REQ_IOVA 0x0258
#define RISCV_IOMMU_TR_REQ_CTL 0x0260
#define RISCV_IOMMU_TR_RESPONSE 0x0268
#define RISCV_IOMMU_ICVEC 0x02F8
#define RISCV_IOMMU_LOCK(_sc) mtx_lock(&(_sc)->mtx)
#define RISCV_IOMMU_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx)
DECLARE_CLASS(riscv_iommu_driver);
MALLOC_DECLARE(M_IOMMU);
struct riscv_iommu_unit {
struct iommu_unit iommu;
LIST_HEAD(, riscv_iommu_domain) domain_list;
LIST_ENTRY(riscv_iommu_unit) next;
device_t dev;
intptr_t xref;
};
struct riscv_iommu_domain {
struct iommu_domain iodom;
LIST_HEAD(, riscv_iommu_ctx) ctx_list;
LIST_ENTRY(riscv_iommu_domain) next;
u_int entries_cnt;
struct riscv_iommu_cd *cd;
struct riscv_iommu_pmap p;
uint16_t pscid;
};
struct riscv_iommu_ctx {
struct iommu_ctx ioctx;
struct riscv_iommu_domain *domain;
LIST_ENTRY(riscv_iommu_ctx) next;
device_t dev;
bool bypass;
int did;
uint16_t vendor;
uint16_t device;
u_int refcnt;
};
struct riscv_iommu_queue_local_copy {
union {
uint64_t val;
struct {
uint32_t head;
uint32_t tail;
};
};
};
struct riscv_iommu_queue {
struct riscv_iommu_queue_local_copy lc;
vm_paddr_t paddr;
void *vaddr;
uint64_t mask;
uint32_t head_off;
uint32_t tail_off;
int size_log2;
uint64_t base;
uint64_t csr;
int idx;
uint8_t entry_size;
};
struct l1_desc {
uint8_t span;
void *va;
vm_paddr_t pa;
};
/* Base-format device-context. */
struct riscv_iommu_dc_base {
uint64_t tc; /* Translation control */
#define DC_TC_V (1 << 0)
#define DC_TC_EN_ATS (1 << 1)
#define DC_TC_EN_PRI (1 << 2)
#define DC_TC_T2GPA (1 << 3)
#define DC_TC_DTF (1 << 4)
#define DC_TC_PDTV (1 << 5)
#define DC_TC_PRPR (1 << 6)
#define DC_TC_GADE (1 << 7)
#define DC_TC_SADE (1 << 8)
#define DC_TC_DPE (1 << 9)
#define DC_TC_SBE (1 << 10)
#define DC_TC_SXL (1 << 11)
uint64_t iohgatp; /* IO Hyp guest address translation */
uint64_t ta; /* Translation attributes */
#define DC_TA_V (1 << 0)
#define DC_TA_ENS (1 << 1)
#define DC_TA_SUM (1 << 2)
#define DC_TA_PSCID_S 12
#define DC_TA_PSCID_M (0xfffff << DC_TA_PSCID_S)
uint64_t fsc; /* First-stage-context */
};
/* Extended-format device-context. */
struct riscv_iommu_dc {
struct riscv_iommu_dc_base base;
uint64_t msiptp; /* MSI page table pointer */
uint64_t msi_addr_mask;
uint64_t msi_addr_pattern;
uint64_t _reserved;
};
#define DC_NON_LEAF_ENTRY_PPN_S 10
#define DC_NON_LEAF_ENTRY_VALID (1 << 0)
struct riscv_iommu_ddt {
void *vaddr;
uint64_t base;
uint32_t base_cfg;
uint32_t num_top_entries;
struct l1_desc *l1;
struct riscv_iommu_dc *dc;
};
struct riscv_iommu_softc {
device_t dev;
intptr_t xref;
struct riscv_iommu_unit unit;
struct resource *res[5];
void *intr_cookie[4];
struct riscv_iommu_queue cq;
struct riscv_iommu_queue fq;
struct riscv_iommu_queue pq;
struct riscv_iommu_ddt ddt;
struct mtx mtx;
uint32_t l0_did_bits;
uint32_t dc_dwords;
/* PSCID management. */
bitstr_t *pscid_set;
int pscid_set_size;
struct mtx pscid_set_mutex;
uint32_t pscid_bits;
enum pmap_mode pm_mode;
int iommu_mode;
};
/*
* Command queue request.
*/
struct riscv_iommu_command {
uint64_t dword0;
uint64_t dword1;
};
enum riscv_iommu_fq_causes {
FQ_CAUSE_INST_FAULT = 1,
FQ_CAUSE_RD_ADDR_MISALIGNED = 4,
FQ_CAUSE_RD_FAULT = 5,
FQ_CAUSE_WR_ADDR_MISALIGNED = 6,
FQ_CAUSE_WR_FAULT = 7,
FQ_CAUSE_INST_FAULT_S = 12,
FQ_CAUSE_RD_FAULT_S = 13,
FQ_CAUSE_WR_FAULT_S = 15,
FQ_CAUSE_INST_FAULT_VS = 20,
FQ_CAUSE_RD_FAULT_VS = 21,
FQ_CAUSE_WR_FAULT_VS = 23,
FQ_CAUSE_DMA_DISABLED = 256,
FQ_CAUSE_DDT_LOAD_FAULT = 257,
FQ_CAUSE_DDT_INVALID = 258,
FQ_CAUSE_DDT_MISCONFIGURED = 259,
FQ_CAUSE_TR_TYPE_DISALLOWED = 260,
FQ_CAUSE_MSI_LOAD_FAULT = 261,
FQ_CAUSE_MSI_INVALID = 262,
FQ_CAUSE_MSI_MISCONFIGURED = 263,
FQ_CAUSE_MRIF_FAULT = 264,
FQ_CAUSE_PDT_LOAD_FAULT = 265,
FQ_CAUSE_PDT_INVALID = 266,
FQ_CAUSE_PDT_MISCONFIGURED = 267,
FQ_CAUSE_DDT_CORRUPTED = 268,
FQ_CAUSE_PDT_CORRUPTED = 269,
FQ_CAUSE_MSI_PT_CORRUPTED = 270,
FQ_CAUSE_MRIF_CORRUPTED = 271,
FQ_CAUSE_INTERNAL_DP_ERROR = 272,
FQ_CAUSE_MSI_WR_FAULT = 273,
FQ_CAUSE_PT_CORRUPTED = 274,
};
/*
* Fault queue record.
*/
struct riscv_iommu_fq_record {
uint64_t hdr;
#define FQR_HDR_CAUSE_S 0
#define FQR_HDR_CAUSE_M (0xfff << FQR_HDR_CAUSE_S)
#define FQR_HDR_PID_S 12
#define FQR_HDR_PID_M (0xfffffULL << FQR_HDR_PID_S)
#define FQR_HDR_PV (1ULL << 32)
#define FQR_HDR_PRIV (1ULL << 33)
#define FQR_HDR_TTYP_S 34ULL
#define FQR_HDR_TTYP_M (0x3fULL << FQR_HDR_TTYP_S)
#define FQR_HDR_DID_S 40ULL
#define FQR_HDR_DID_M (0xffffffULL << FQR_HDR_DID_S)
uint32_t custom;
uint32_t reserved;
uint64_t iotval;
uint64_t iotval2;
};
#define COMMAND_OPCODE_S 0
#define COMMAND_OPCODE_IOTINVAL (1 << COMMAND_OPCODE_S)
#define COMMAND_OPCODE_IOFENCE (2 << COMMAND_OPCODE_S)
#define COMMAND_OPCODE_IODIR (3 << COMMAND_OPCODE_S)
#define COMMAND_OPCODE_ATS (4 << COMMAND_OPCODE_S)
#define COMMAND_OPCODE_FUNC_S 7
#define COMMAND_OPCODE_FUNC_M (0x3 << COMMAND_OPCODE_FUNC_S)
#define FUNC_IODIR_INVAL_DDT (0 << COMMAND_OPCODE_FUNC_S)
#define FUNC_IODIR_INVAL_PDT (1 << COMMAND_OPCODE_FUNC_S)
#define FUNC_IODIR_PID_S 12
#define FUNC_IODIR_DV (1ULL << 33) /* DID Valid */
#define FUNC_IODIR_DID_S 40ULL
/* dword0 */
#define FUNC_IOTINVAL_VMA (0 << COMMAND_OPCODE_FUNC_S)
#define FUNC_IOTINVAL_GVMA (1 << COMMAND_OPCODE_FUNC_S)
#define FUNC_IOTINVAL_AV (1 << 10) /* Address Valid */
#define FUNC_IOTINVAL_PSCID_S 12 /* Process-Soft-Context ID */
#define FUNC_IOTINVAL_PSCV (1ULL << 32) /* PSCID Valid */
#define FUNC_IOTINVAL_GV (1ULL << 33) /* GSCID Valid */
#define FUNC_IOTINVAL_GSCID_S 44 /* Guest-Soft-Context ID */
/* dword1 */
#define FUNC_IOTINVAL_ADDR_S 10
#define FUNC_IOFENCE_FUNC_C (0 << 7)
#define FUNC_IOFENCE_AV (1 << 10)
#define FUNC_IOFENCE_WSI (1 << 11)
#define FUNC_IOFENCE_PR (1 << 12)
#define FUNC_IOFENCE_PW (1 << 13)
#define FUNC_IOFENCE_DATA_S 32ULL
int riscv_iommu_attach(device_t dev);
struct riscv_iommu_ctx *riscv_iommu_ctx_lookup_by_did(device_t dev, u_int did);
#endif /* _RISCV_IOMMU_IOMMU_H_ */
sys/riscv/iommu/iommu_fdt.c
+145
View File
@@ -0,0 +1,145 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/bitstring.h>
#include <sys/kernel.h>
#include <sys/rman.h>
#include <sys/sysctl.h>
#include <sys/tree.h>
#include <sys/taskqueue.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/rman.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include <dev/iommu/iommu.h>
#include <riscv/iommu/iommu_pmap.h>
#include <riscv/iommu/iommu.h>
#include <riscv/iommu/iommu_frontend.h>
static struct ofw_compat_data compat_data[] = {
{ "riscv,iommu", 1 },
{ NULL, 0 }
};
static struct resource_spec riscv_iommu_spec[] = {
{ SYS_RES_MEMORY, 0, RF_ACTIVE },
{ SYS_RES_IRQ, 0, RF_ACTIVE }, /* CQ */
{ SYS_RES_IRQ, 1, RF_ACTIVE }, /* FQ */
{ SYS_RES_IRQ, 2, RF_ACTIVE }, /* PM */
{ SYS_RES_IRQ, 3, RF_ACTIVE }, /* PQ */
RESOURCE_SPEC_END
};
static int
riscv_iommu_fdt_probe(device_t dev)
{
if (!ofw_bus_status_okay(dev))
return (ENXIO);
if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
return (ENXIO);
device_set_desc(dev, "RISC-V IOMMU");
return (BUS_PROBE_DEFAULT);
}
static int
riscv_iommu_fdt_attach(device_t dev)
{
struct riscv_iommu_softc *sc;
struct riscv_iommu_unit *unit;
struct iommu_unit *iommu;
phandle_t node;
int error;
sc = device_get_softc(dev);
sc->dev = dev;
node = ofw_bus_get_node(dev);
error = bus_alloc_resources(dev, riscv_iommu_spec, sc->res);
if (error) {
device_printf(dev, "could not allocate resources\n");
goto error;
}
error = riscv_iommu_attach(dev);
if (error != 0) {
device_printf(dev, "Failed to attach. Error %d\n", error);
goto error;
}
unit = &sc->unit;
unit->dev = dev;
iommu = &unit->iommu;
iommu->dev = dev;
LIST_INIT(&unit->domain_list);
sc->xref = OF_xref_from_node(node);
error = iommu_register(iommu);
if (error) {
device_printf(dev, "Failed to register RISC-V IOMMU.\n");
goto error;
}
return (0);
error:
bus_release_resources(dev, riscv_iommu_spec, sc->res);
return (error);
}
static device_method_t riscv_iommu_fdt_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, riscv_iommu_fdt_probe),
DEVMETHOD(device_attach, riscv_iommu_fdt_attach),
DEVMETHOD_END
};
DEFINE_CLASS_1(riscv_iommu, riscv_iommu_fdt_driver, riscv_iommu_fdt_methods,
sizeof(struct riscv_iommu_softc), riscv_iommu_driver);
EARLY_DRIVER_MODULE(riscv_iommu, simplebus, riscv_iommu_fdt_driver, 0, 0,
BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE);
sys/riscv/iommu/iommu_frontend.c
+505
View File
@@ -0,0 +1,505 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include "opt_platform.h"
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/memdesc.h>
#include <sys/tree.h>
#include <sys/taskqueue.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <machine/bus.h>
#include <dev/iommu/busdma_iommu.h>
#include <machine/vmparam.h>
#ifdef FDT
#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#endif
#include "iommu_frontend.h"
#include "iommu_if.h"
static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework");
#define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx)
#define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx)
#define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED)
#define dprintf(fmt, ...)
static struct sx iommu_sx;
struct iommu_entry {
struct iommu_unit *iommu;
LIST_ENTRY(iommu_entry) next;
};
static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list);
static int
iommu_domain_unmap_buf(struct iommu_domain *iodom,
struct iommu_map_entry *entry, int flags)
{
struct iommu_unit *iommu;
int error;
iommu = iodom->iommu;
error = IOMMU_UNMAP(iommu->dev, iodom, entry->start, entry->end -
entry->start);
return (error);
}
static int
iommu_domain_map_buf(struct iommu_domain *iodom, struct iommu_map_entry *entry,
vm_page_t *ma, uint64_t eflags, int flags)
{
struct iommu_unit *iommu;
vm_prot_t prot;
vm_offset_t va;
int error;
dprintf("%s: base %lx, size %lx\n", __func__, base, size);
prot = 0;
if (eflags & IOMMU_MAP_ENTRY_READ)
prot |= VM_PROT_READ;
if (eflags & IOMMU_MAP_ENTRY_WRITE)
prot |= VM_PROT_WRITE;
va = entry->start;
iommu = iodom->iommu;
error = IOMMU_MAP(iommu->dev, iodom, va, ma, entry->end -
entry->start, prot);
return (error);
}
static const struct iommu_domain_map_ops domain_map_ops = {
.map = iommu_domain_map_buf,
.unmap = iommu_domain_unmap_buf,
};
static struct iommu_domain *
iommu_domain_alloc(struct iommu_unit *iommu)
{
struct iommu_domain *iodom;
iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu);
if (iodom == NULL)
return (NULL);
KASSERT(iodom->end != 0, ("domain end is not set"));
iommu_domain_init(iommu, iodom, &domain_map_ops);
iodom->iommu = iommu;
iommu_gas_init_domain(iodom);
return (iodom);
}
static int
iommu_domain_free(struct iommu_domain *iodom)
{
struct iommu_unit *iommu;
iommu = iodom->iommu;
IOMMU_LOCK(iommu);
if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
IOMMU_DOMAIN_LOCK(iodom);
iommu_gas_fini_domain(iodom);
IOMMU_DOMAIN_UNLOCK(iodom);
}
iommu_domain_fini(iodom);
IOMMU_DOMAIN_FREE(iommu->dev, iodom);
IOMMU_UNLOCK(iommu);
return (0);
}
static void
iommu_tag_init(struct iommu_domain *iodom, struct bus_dma_tag_iommu *t)
{
bus_addr_t maxaddr;
maxaddr = MIN(iodom->end, BUS_SPACE_MAXADDR);
t->common.impl = &bus_dma_iommu_impl;
t->common.alignment = 1;
t->common.boundary = 0;
t->common.lowaddr = maxaddr;
t->common.highaddr = maxaddr;
t->common.maxsize = maxaddr;
t->common.nsegments = BUS_SPACE_UNRESTRICTED;
t->common.maxsegsz = maxaddr;
}
static struct iommu_ctx *
iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled)
{
struct iommu_unit *iommu;
struct iommu_ctx *ioctx;
iommu = iodom->iommu;
ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled);
if (ioctx == NULL)
return (NULL);
ioctx->domain = iodom;
return (ioctx);
}
static int
iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx)
{
struct bus_dma_tag_iommu *tag;
struct iommu_domain *iodom;
struct iommu_unit *iommu;
int error;
iodom = ioctx->domain;
iommu = iodom->iommu;
error = IOMMU_CTX_INIT(iommu->dev, ioctx);
if (error)
return (error);
tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu),
M_DEVBUF, M_WAITOK | M_ZERO);
tag->owner = requester;
tag->ctx = ioctx;
tag->ctx->domain = iodom;
iommu_tag_init(iodom, tag);
return (error);
}
static struct iommu_unit *
iommu_lookup(device_t dev)
{
struct iommu_entry *entry;
struct iommu_unit *iommu;
IOMMU_LIST_LOCK();
LIST_FOREACH(entry, &iommu_list, next) {
iommu = entry->iommu;
if (iommu->dev == dev) {
IOMMU_LIST_UNLOCK();
return (iommu);
}
}
IOMMU_LIST_UNLOCK();
return (NULL);
}
#ifdef FDT
struct iommu_ctx *
iommu_get_ctx_ofw(device_t dev, int channel)
{
struct iommu_domain *iodom;
struct iommu_unit *iommu;
struct iommu_ctx *ioctx;
phandle_t node, parent;
device_t iommu_dev;
pcell_t *cells;
int niommus;
int ncells;
int error;
node = ofw_bus_get_node(dev);
if (node <= 0) {
device_printf(dev,
"%s called on not ofw based device.\n", __func__);
return (NULL);
}
error = ofw_bus_parse_xref_list_get_length(node,
"iommus", "#iommu-cells", &niommus);
if (error) {
device_printf(dev, "%s can't get iommu list.\n", __func__);
return (NULL);
}
if (niommus == 0) {
device_printf(dev, "%s iommu list is empty.\n", __func__);
return (NULL);
}
error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells",
channel, &parent, &ncells, &cells);
if (error != 0) {
device_printf(dev, "%s can't get iommu device xref.\n",
__func__);
return (NULL);
}
iommu_dev = OF_device_from_xref(parent);
if (iommu_dev == NULL) {
device_printf(dev, "%s can't get iommu device.\n", __func__);
return (NULL);
}
iommu = iommu_lookup(iommu_dev);
if (iommu == NULL) {
device_printf(dev, "%s can't lookup iommu.\n", __func__);
return (NULL);
}
/*
* In our current configuration we have a domain per each ctx,
* so allocate a domain first.
*/
iodom = iommu_domain_alloc(iommu);
if (iodom == NULL) {
device_printf(dev, "%s can't allocate domain.\n", __func__);
return (NULL);
}
ioctx = iommu_ctx_alloc(dev, iodom, false);
if (ioctx == NULL) {
iommu_domain_free(iodom);
return (NULL);
}
ioctx->domain = iodom;
error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells);
if (error) {
device_printf(dev, "%s can't set MD data\n", __func__);
return (NULL);
}
error = iommu_ctx_init(dev, ioctx);
if (error) {
IOMMU_CTX_FREE(iommu->dev, ioctx);
iommu_domain_free(iodom);
return (NULL);
}
return (ioctx);
}
#endif
struct iommu_ctx *
iommu_get_ctx(struct iommu_unit *iommu, device_t requester,
uint16_t rid, bool disabled, bool rmrr)
{
struct iommu_domain *iodom;
struct iommu_ctx *ioctx;
int error;
IOMMU_LOCK(iommu);
ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester);
if (ioctx) {
IOMMU_UNLOCK(iommu);
return (ioctx);
}
IOMMU_UNLOCK(iommu);
/*
* In our current configuration we have a domain per each ctx.
* So allocate a domain first.
*/
iodom = iommu_domain_alloc(iommu);
if (iodom == NULL)
return (NULL);
ioctx = iommu_ctx_alloc(requester, iodom, disabled);
if (ioctx == NULL) {
iommu_domain_free(iodom);
return (NULL);
}
error = iommu_ctx_init(requester, ioctx);
if (error) {
IOMMU_CTX_FREE(iommu->dev, ioctx);
iommu_domain_free(iodom);
return (NULL);
}
return (ioctx);
}
void
iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx)
{
struct iommu_domain *domain;
bool released;
int error;
IOMMU_ASSERT_LOCKED(iommu);
domain = ioctx->domain;
released = IOMMU_CTX_FREE(iommu->dev, ioctx);
IOMMU_UNLOCK(iommu);
if (released) {
/* Since we have a domain per each ctx, remove it too. */
error = iommu_domain_free(domain);
if (error)
device_printf(iommu->dev, "Could not free a domain\n");
}
}
static void
iommu_domain_free_entry(struct iommu_map_entry *entry, bool free)
{
iommu_gas_free_space(entry);
if (free)
iommu_gas_free_entry(entry);
else
entry->flags = 0;
}
void
iommu_domain_unload(struct iommu_domain *iodom,
struct iommu_map_entries_tailq *entries, bool cansleep)
{
struct iommu_map_entry *entry, *entry1;
int error __diagused;
TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
("not mapped entry %p %p", iodom, entry));
error = iodom->ops->unmap(iodom, entry,
cansleep ? IOMMU_PGF_WAITOK : 0);
KASSERT(error == 0, ("unmap %p error %d", iodom, error));
TAILQ_REMOVE(entries, entry, dmamap_link);
iommu_domain_free_entry(entry, true);
}
if (TAILQ_EMPTY(entries))
return;
panic("entries map is not empty");
}
int
iommu_register(struct iommu_unit *iommu)
{
struct iommu_entry *entry;
mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF);
entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO);
entry->iommu = iommu;
IOMMU_LIST_LOCK();
LIST_INSERT_HEAD(&iommu_list, entry, next);
IOMMU_LIST_UNLOCK();
sysctl_ctx_init(&iommu->sysctl_ctx);
iommu_init_busdma(iommu);
return (0);
}
int
iommu_unregister(struct iommu_unit *iommu)
{
struct iommu_entry *entry, *tmp;
IOMMU_LIST_LOCK();
LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) {
if (entry->iommu == iommu) {
LIST_REMOVE(entry, next);
free(entry, M_IOMMU);
}
}
IOMMU_LIST_UNLOCK();
iommu_fini_busdma(iommu);
sysctl_ctx_free(&iommu->sysctl_ctx);
mtx_destroy(&iommu->lock);
return (0);
}
struct iommu_unit *
iommu_find(device_t dev, bool verbose)
{
struct iommu_entry *entry;
struct iommu_unit *iommu;
int error;
IOMMU_LIST_LOCK();
LIST_FOREACH(entry, &iommu_list, next) {
iommu = entry->iommu;
error = IOMMU_FIND(iommu->dev, dev);
if (error == 0) {
IOMMU_LIST_UNLOCK();
return (entry->iommu);
}
}
IOMMU_LIST_UNLOCK();
return (NULL);
}
void
iommu_unit_pre_instantiate_ctx(struct iommu_unit *unit)
{
}
void
iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
bool cansleep __unused)
{
dprintf("%s\n", __func__);
iommu_domain_free_entry(entry, free);
}
static void
iommu_init(void)
{
sx_init(&iommu_sx, "IOMMU list");
}
SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL);
sys/riscv/iommu/iommu_frontend.h
+38
View File
@@ -0,0 +1,38 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _ARM64_IOMMU_IOMMU_H_
#define _ARM64_IOMMU_IOMMU_H_
#define IOMMU_PAGE_SIZE 4096
#define IOMMU_PAGE_MASK (IOMMU_PAGE_SIZE - 1)
int iommu_unregister(struct iommu_unit *unit);
int iommu_register(struct iommu_unit *unit);
struct iommu_ctx * iommu_get_ctx_ofw(device_t dev, int channel);
#endif /* _ARM64_IOMMU_IOMMU_H_ */
sys/riscv/iommu/iommu_if.m
+147
View File
@@ -0,0 +1,147 @@
#-
# SPDX-License-Identifier: BSD-2-Clause
#:
# Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
#
#include "opt_platform.h"
#include <sys/types.h>
#include <sys/taskqueue.h>
#include <sys/bus.h>
#include <sys/sysctl.h>
#include <sys/tree.h>
#include <vm/vm.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/iommu/iommu.h>
#ifdef FDT
#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#endif
INTERFACE iommu;
#
# Check if the iommu controller dev is responsible to serve traffic
# for a given child.
#
METHOD int find {
device_t dev;
device_t child;
};
#
# Map a virtual address VA to a physical address PA.
#
METHOD int map {
device_t dev;
struct iommu_domain *iodom;
vm_offset_t va;
vm_page_t *ma;
bus_size_t size;
vm_prot_t prot;
};
#
# Unmap a virtual address VA.
#
METHOD int unmap {
device_t dev;
struct iommu_domain *iodom;
vm_offset_t va;
bus_size_t size;
};
#
# Allocate an IOMMU domain.
#
METHOD struct iommu_domain * domain_alloc {
device_t dev;
struct iommu_unit *iommu;
};
#
# Release all the resources held by IOMMU domain.
#
METHOD void domain_free {
device_t dev;
struct iommu_domain *iodom;
};
#
# Find a domain allocated for a dev.
#
METHOD struct iommu_domain * domain_lookup {
device_t dev;
};
#
# Find an allocated context for a device.
#
METHOD struct iommu_ctx * ctx_lookup {
device_t dev;
device_t child;
};
#
# Allocate a new iommu context.
#
METHOD struct iommu_ctx * ctx_alloc {
device_t dev;
struct iommu_domain *iodom;
device_t child;
bool disabled;
};
#
# Initialize the new iommu context.
#
METHOD int ctx_init {
device_t dev;
struct iommu_ctx *ioctx;
};
#
# Free the iommu context.
#
METHOD bool ctx_free {
device_t dev;
struct iommu_ctx *ioctx;
};
#ifdef FDT
#
# Notify controller we have machine-dependent data.
#
METHOD int ofw_md_data {
device_t dev;
struct iommu_ctx *ioctx;
pcell_t *cells;
int ncells;
};
#endif
sys/riscv/iommu/iommu_pci.c
+172
View File
@@ -0,0 +1,172 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/bitstring.h>
#include <sys/kernel.h>
#include <sys/rman.h>
#include <sys/sysctl.h>
#include <sys/tree.h>
#include <sys/taskqueue.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <machine/bus.h>
#include <machine/resource.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>
#include <dev/iommu/iommu.h>
#include <riscv/iommu/iommu_pmap.h>
#include <riscv/iommu/iommu.h>
#include <riscv/iommu/iommu_frontend.h>
#define PCI_DEVICE_ID_REDHAT_RISCV_IOMMU 0x0014
#define PCI_VENDOR_ID_REDHAT 0x1b36
#define PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA 0x0008
#define PCI_VENDOR_ID_RIVOS 0x1efd
static int
iommu_pci_probe(device_t dev)
{
uint16_t vendor_id, device_id;
vendor_id = pci_get_vendor(dev);
device_id = pci_get_device(dev);
if (vendor_id == PCI_VENDOR_ID_REDHAT &&
device_id == PCI_DEVICE_ID_REDHAT_RISCV_IOMMU) {
device_set_desc(dev, "RedHat IOMMU");
return (BUS_PROBE_DEFAULT);
}
if (vendor_id == PCI_VENDOR_ID_RIVOS &&
device_id == PCI_DEVICE_ID_RIVOS_RISCV_IOMMU_GA) {
device_set_desc(dev, "Rivos IOMMU");
return (BUS_PROBE_DEFAULT);
}
return (ENXIO);
}
static int
iommu_pci_attach(device_t dev)
{
struct riscv_iommu_unit *unit;
struct riscv_iommu_softc *sc;
struct iommu_unit *iommu;
phandle_t node;
int count;
int error;
int rid;
int i;
sc = device_get_softc(dev);
sc->dev = dev;
node = ofw_bus_get_node(dev);
pci_enable_busmaster(dev);
rid = PCIR_BAR(0);
sc->res[0] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
RF_ACTIVE);
if (sc->res[0] == NULL) {
device_printf(dev, "couldn't map memory\n");
goto error;
}
count = 4;
if (pci_alloc_msix(dev, &count) == 0) {
for (i = 0; i < 4; i++) {
rid = i;
sc->res[1 + i] = bus_alloc_resource_any(dev,
SYS_RES_IRQ, &rid, RF_ACTIVE);
if (sc->res[i + 1] == NULL) {
device_printf(dev, "Can't allocate IRQ "
" resource.\n");
goto error;
}
}
} else
device_printf(dev, "Can't allocate MSI-X interrupts."
" Ignoring.\n");
error = riscv_iommu_attach(dev);
if (error)
goto error;
unit = &sc->unit;
unit->dev = dev;
iommu = &unit->iommu;
iommu->dev = dev;
LIST_INIT(&unit->domain_list);
sc->xref = OF_xref_from_node(node);
error = iommu_register(iommu);
if (error) {
device_printf(dev, "Failed to register RISC-V IOMMU.\n");
goto error;
}
return (0);
error:
if (sc->res[0])
bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0),
sc->res[0]);
for (i = 0; i < 4; i++)
if (sc->res[i + 1])
bus_release_resource(dev, SYS_RES_IRQ, i,
sc->res[i + 1]);
return (error);
}
static device_method_t riscv_iommu_pci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, iommu_pci_probe),
DEVMETHOD(device_attach, iommu_pci_attach),
DEVMETHOD_END
};
DEFINE_CLASS_1(riscv_iommu, riscv_iommu_pci_driver,
riscv_iommu_pci_methods, sizeof(struct riscv_iommu_softc),
riscv_iommu_driver);
DRIVER_MODULE(riscv_iommu, pci, riscv_iommu_pci_driver, NULL, NULL);
sys/riscv/iommu/iommu_pmap.c
+629
View File
@@ -0,0 +1,629 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_page.h>
#include <riscv/iommu/iommu_pmap.h>
/*
* Boundary values for the page table page index space:
*
* L3 pages: [0, NUL2E)
* L2 pages: [NUL2E, NUL2E + NUL1E)
* L1 pages: [NUL2E + NUL1E, NUL2E + NUL1E + NUL0E)
*
* Note that these ranges are used in both SV39 and SV48 mode. In SV39 mode the
* ranges are not fully populated since there are at most Ln_ENTRIES^2 L3 pages
* in a set of page tables.
*/
#define NUL0E Ln_ENTRIES
#define NUL1E (Ln_ENTRIES * NUL0E)
#define NUL2E (Ln_ENTRIES * NUL1E)
#define pmap_l1_pindex(v) (NUL2E + ((v) >> L1_SHIFT))
#define pmap_l2_pindex(v) ((v) >> L2_SHIFT)
#define pmap_clear(pte) pmap_store(pte, 0)
#define pmap_clear_bits(pte, bits) atomic_clear_64(pte, bits)
#define pmap_load_store(pte, entry) atomic_swap_64(pte, entry)
#define pmap_load_clear(pte) pmap_load_store(pte, 0)
#define pmap_load(pte) atomic_load_64(pte)
#define pmap_store(pte, entry) atomic_store_64(pte, entry)
#define pmap_store_bits(pte, bits) atomic_set_64(pte, bits)
#define pmap_l0_index(va) (((va) >> L0_SHIFT) & Ln_ADDR_MASK)
#define pmap_l1_index(va) (((va) >> L1_SHIFT) & Ln_ADDR_MASK)
#define pmap_l2_index(va) (((va) >> L2_SHIFT) & Ln_ADDR_MASK)
#define pmap_l3_index(va) (((va) >> L3_SHIFT) & Ln_ADDR_MASK)
#define PTE_TO_PHYS(pte) \
((((pte) & ~PTE_HI_MASK) >> PTE_PPN0_S) * PAGE_SIZE)
#define L2PTE_TO_PHYS(l2) \
((((l2) & ~PTE_HI_MASK) >> PTE_PPN1_S) << L2_SHIFT)
#define L1PTE_TO_PHYS(l1) \
((((l1) & ~PTE_HI_MASK) >> PTE_PPN2_S) << L1_SHIFT)
#define PTE_TO_VM_PAGE(pte) PHYS_TO_VM_PAGE(PTE_TO_PHYS(pte))
/********************/
/* Inline functions */
/********************/
static __inline pd_entry_t *
pmap_l0(struct riscv_iommu_pmap *pmap, vm_offset_t va)
{
KASSERT(pmap->pm_mode != PMAP_MODE_SV39,
("%s: in SV39 mode", __func__));
KASSERT(VIRT_IS_VALID(va),
("%s: malformed virtual address %#lx", __func__, va));
return (&pmap->pm_top[pmap_l0_index(va)]);
}
static __inline pd_entry_t *
pmap_l0_to_l1(struct riscv_iommu_pmap *pmap, pd_entry_t *l0, vm_offset_t va)
{
vm_paddr_t phys;
pd_entry_t *l1;
KASSERT(pmap->pm_mode != PMAP_MODE_SV39,
("%s: in SV39 mode", __func__));
phys = PTE_TO_PHYS(pmap_load(l0));
l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
return (&l1[pmap_l1_index(va)]);
}
static __inline pd_entry_t *
pmap_l1(struct riscv_iommu_pmap *pmap, vm_offset_t va)
{
pd_entry_t *l0;
KASSERT(VIRT_IS_VALID(va),
("%s: malformed virtual address %#lx", __func__, va));
if (pmap->pm_mode == PMAP_MODE_SV39) {
return (&pmap->pm_top[pmap_l1_index(va)]);
} else {
l0 = pmap_l0(pmap, va);
if ((pmap_load(l0) & PTE_V) == 0)
return (NULL);
if ((pmap_load(l0) & PTE_RX) != 0)
return (NULL);
return (pmap_l0_to_l1(pmap, l0, va));
}
}
static __inline pd_entry_t *
pmap_l1_to_l2(pd_entry_t *l1, vm_offset_t va)
{
vm_paddr_t phys;
pd_entry_t *l2;
phys = PTE_TO_PHYS(pmap_load(l1));
l2 = (pd_entry_t *)PHYS_TO_DMAP(phys);
return (&l2[pmap_l2_index(va)]);
}
static __inline pd_entry_t *
pmap_l2(struct riscv_iommu_pmap *pmap, vm_offset_t va)
{
pd_entry_t *l1;
l1 = pmap_l1(pmap, va);
if (l1 == NULL)
return (NULL);
if ((pmap_load(l1) & PTE_V) == 0)
return (NULL);
if ((pmap_load(l1) & PTE_RX) != 0)
return (NULL);
return (pmap_l1_to_l2(l1, va));
}
static __inline pt_entry_t *
pmap_l2_to_l3(pd_entry_t *l2, vm_offset_t va)
{
vm_paddr_t phys;
pt_entry_t *l3;
phys = PTE_TO_PHYS(pmap_load(l2));
l3 = (pd_entry_t *)PHYS_TO_DMAP(phys);
return (&l3[pmap_l3_index(va)]);
}
static __inline pt_entry_t *
pmap_l3(struct riscv_iommu_pmap *pmap, vm_offset_t va)
{
pd_entry_t *l2;
l2 = pmap_l2(pmap, va);
if (l2 == NULL)
return (NULL);
if ((pmap_load(l2) & PTE_V) == 0)
return (NULL);
if ((pmap_load(l2) & PTE_RX) != 0)
return (NULL);
return (pmap_l2_to_l3(l2, va));
}
static __inline void
pmap_resident_count_inc(struct riscv_iommu_pmap *pmap, int count)
{
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
pmap->sp_resident_count += count;
}
static __inline void
pmap_resident_count_dec(struct riscv_iommu_pmap *pmap, int count)
{
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
KASSERT(pmap->sp_resident_count >= count,
("pmap %p resident count underflow %ld %d", pmap,
pmap->sp_resident_count, count));
pmap->sp_resident_count -= count;
}
/***************************************************
* Page table page management routines.....
***************************************************/
int
iommu_pmap_pinit(struct riscv_iommu_pmap *pmap, enum pmap_mode pm_mode)
{
vm_paddr_t topphys;
vm_page_t m;
m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO |
VM_ALLOC_WAITOK);
topphys = VM_PAGE_TO_PHYS(m);
pmap->pm_top = (pd_entry_t *)PHYS_TO_DMAP(topphys);
pmap->pm_mode = pm_mode;
switch (pm_mode) {
case PMAP_MODE_SV39:
pmap->pm_satp = SATP_MODE_SV39;
break;
case PMAP_MODE_SV48:
pmap->pm_satp = SATP_MODE_SV48;
break;
default:
panic("Unknown virtual memory system");
};
pmap->pm_satp |= (topphys >> PAGE_SHIFT);
#ifdef INVARIANTS
pmap->sp_resident_count = 0;
#endif
mtx_init(&pmap->pm_mtx, "iommu pmap", NULL, MTX_DEF);
return (1);
}
/*
* Release any resources held by the given physical map.
* Called when a pmap initialized by pmap_pinit is being released.
* Should only be called if the map contains no valid mappings.
*/
void
iommu_pmap_release(struct riscv_iommu_pmap *pmap)
{
vm_page_t m;
KASSERT(pmap->sp_resident_count == 0,
("pmap_release: pmap resident count %ld != 0",
pmap->sp_resident_count));
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_top));
vm_page_unwire_noq(m);
vm_page_free_zero(m);
mtx_destroy(&pmap->pm_mtx);
}
/*
* This routine is called if the desired page table page does not exist.
*
* If page table page allocation fails, this routine may sleep before
* returning NULL. It sleeps only if a lock pointer was given.
*
* Note: If a page allocation fails at page table level two or three,
* one or two pages may be held during the wait, only to be released
* afterwards. This conservative approach is easily argued to avoid
* race conditions.
*/
static vm_page_t
_pmap_alloc_l3(struct riscv_iommu_pmap *pmap, vm_pindex_t ptepindex)
{
vm_page_t m, pdpg;
pt_entry_t entry;
vm_paddr_t phys;
pn_t pn;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
/*
* Allocate a page table page.
*/
m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO);
if (m == NULL) {
/*
* Indicate the need to retry. While waiting, the page table
* page may have been allocated.
*/
return (NULL);
}
m->pindex = ptepindex;
/*
* Map the pagetable page into the process address space, if
* it isn't already there.
*/
pn = VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT;
if (ptepindex >= NUL2E + NUL1E) {
pd_entry_t *l0;
vm_pindex_t l0index;
KASSERT(pmap->pm_mode != PMAP_MODE_SV39,
("%s: pindex %#lx in SV39 mode", __func__, ptepindex));
KASSERT(ptepindex < NUL2E + NUL1E + NUL0E,
("%s: pindex %#lx out of range", __func__, ptepindex));
l0index = ptepindex - (NUL2E + NUL1E);
l0 = &pmap->pm_top[l0index];
KASSERT((pmap_load(l0) & PTE_V) == 0,
("%s: L0 entry %#lx is valid", __func__, pmap_load(l0)));
entry = PTE_V | (pn << PTE_PPN0_S);
pmap_store(l0, entry);
} else if (ptepindex >= NUL2E) {
pd_entry_t *l0, *l1;
vm_pindex_t l0index, l1index;
l1index = ptepindex - NUL2E;
if (pmap->pm_mode == PMAP_MODE_SV39) {
l1 = &pmap->pm_top[l1index];
} else {
l0index = l1index >> Ln_ENTRIES_SHIFT;
l0 = &pmap->pm_top[l0index];
if (pmap_load(l0) == 0) {
/* Recurse to allocate the L1 page. */
if (_pmap_alloc_l3(pmap,
NUL2E + NUL1E + l0index) == NULL)
goto fail;
phys = PTE_TO_PHYS(pmap_load(l0));
} else {
phys = PTE_TO_PHYS(pmap_load(l0));
pdpg = PHYS_TO_VM_PAGE(phys);
pdpg->ref_count++;
}
l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
l1 = &l1[ptepindex & Ln_ADDR_MASK];
}
KASSERT((pmap_load(l1) & PTE_V) == 0,
("%s: L1 entry %#lx is valid", __func__, pmap_load(l1)));
entry = PTE_V | (pn << PTE_PPN0_S);
pmap_store(l1, entry);
} else {
vm_pindex_t l0index, l1index;
pd_entry_t *l0, *l1, *l2;
l1index = ptepindex >> (L1_SHIFT - L2_SHIFT);
if (pmap->pm_mode == PMAP_MODE_SV39) {
l1 = &pmap->pm_top[l1index];
if (pmap_load(l1) == 0) {
/* recurse for allocating page dir */
if (_pmap_alloc_l3(pmap, NUL2E + l1index)
== NULL)
goto fail;
} else {
pdpg = PTE_TO_VM_PAGE(pmap_load(l1));
pdpg->ref_count++;
}
} else {
l0index = l1index >> Ln_ENTRIES_SHIFT;
l0 = &pmap->pm_top[l0index];
if (pmap_load(l0) == 0) {
/* Recurse to allocate the L1 entry. */
if (_pmap_alloc_l3(pmap, NUL2E + l1index)
== NULL)
goto fail;
phys = PTE_TO_PHYS(pmap_load(l0));
l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
l1 = &l1[l1index & Ln_ADDR_MASK];
} else {
phys = PTE_TO_PHYS(pmap_load(l0));
l1 = (pd_entry_t *)PHYS_TO_DMAP(phys);
l1 = &l1[l1index & Ln_ADDR_MASK];
if (pmap_load(l1) == 0) {
/* Recurse to allocate the L2 page. */
if (_pmap_alloc_l3(pmap,
NUL2E + l1index) == NULL)
goto fail;
} else {
pdpg = PTE_TO_VM_PAGE(pmap_load(l1));
pdpg->ref_count++;
}
}
}
phys = PTE_TO_PHYS(pmap_load(l1));
l2 = (pd_entry_t *)PHYS_TO_DMAP(phys);
l2 = &l2[ptepindex & Ln_ADDR_MASK];
KASSERT((pmap_load(l2) & PTE_V) == 0,
("%s: L2 entry %#lx is valid", __func__, pmap_load(l2)));
entry = PTE_V | (pn << PTE_PPN0_S);
pmap_store(l2, entry);
}
pmap_resident_count_inc(pmap, 1);
return (m);
fail:
vm_page_unwire_noq(m);
vm_page_free_zero(m);
return (NULL);
}
/*
* Remove a single IOMMU entry.
*/
int
iommu_pmap_remove(struct riscv_iommu_pmap *pmap, vm_offset_t va)
{
pt_entry_t *l3;
int rc;
PMAP_LOCK(pmap);
l3 = pmap_l3(pmap, va);
if (l3 != NULL) {
pmap_resident_count_dec(pmap, 1);
pmap_clear(l3);
rc = KERN_SUCCESS;
} else
rc = KERN_FAILURE;
PMAP_UNLOCK(pmap);
return (rc);
}
/* Add a single IOMMU entry. This function does not sleep. */
int
iommu_pmap_enter(struct riscv_iommu_pmap *pmap, vm_offset_t va, vm_paddr_t pa,
vm_prot_t prot, u_int flags)
{
pd_entry_t *l2, l2e;
pt_entry_t new_l3;
pt_entry_t *l3;
vm_page_t mpte;
pn_t pn;
int rv;
pn = (pa / PAGE_SIZE);
new_l3 = PTE_V | PTE_R | PTE_A;
if (prot & VM_PROT_EXECUTE)
new_l3 |= PTE_X;
if (flags & VM_PROT_WRITE)
new_l3 |= PTE_D;
if (prot & VM_PROT_WRITE)
new_l3 |= PTE_W;
if (va < VM_MAX_USER_ADDRESS)
new_l3 |= PTE_U;
new_l3 |= (pn << PTE_PPN0_S);
new_l3 |= PTE_MA_IO;
/*
* Set modified bit gratuitously for writeable mappings if
* the page is unmanaged. We do not want to take a fault
* to do the dirty bit accounting for these mappings.
*/
if (prot & VM_PROT_WRITE)
new_l3 |= PTE_D;
CTR2(KTR_PMAP, "pmap_enter: %.16lx -> %.16lx", va, pa);
mpte = NULL;
PMAP_LOCK(pmap);
l2 = pmap_l2(pmap, va);
if (l2 != NULL && ((l2e = pmap_load(l2)) & PTE_V) != 0 &&
((l2e & PTE_RWX) == 0)) {
l3 = pmap_l2_to_l3(l2, va);
} else if (va < VM_MAXUSER_ADDRESS) {
mpte = _pmap_alloc_l3(pmap, pmap_l2_pindex(va));
if (mpte == NULL) {
CTR0(KTR_PMAP, "pmap_enter: mpte == NULL");
rv = KERN_RESOURCE_SHORTAGE;
goto out;
}
l3 = pmap_l3(pmap, va);
} else
panic("pmap_enter: missing L3 table for kernel va %#lx", va);
KASSERT((pmap_load(l3) & PTE_V) == 0, ("l3 is valid"));
pmap_store(l3, new_l3);
pmap_resident_count_inc(pmap, 1);
rv = KERN_SUCCESS;
out:
PMAP_UNLOCK(pmap);
return (rv);
}
static void
iommu_pmap_remove_pages_sv48(struct riscv_iommu_pmap *pmap)
{
pd_entry_t l0e, *l1, l1e, *l2, l2e, *l3, l3e;
vm_paddr_t pa0, pa1, pa;
vm_page_t m0, m1, m;
int i, j, k, l;
PMAP_LOCK(pmap);
for (i = 0; i < Ln_ENTRIES; i++) {
l0e = pmap->pm_top[i];
if ((l0e & PTE_V) == 0)
continue;
pa0 = PTE_TO_PHYS(l0e);
m0 = PHYS_TO_VM_PAGE(pa0);
l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0);
for (j = 0; j < Ln_ENTRIES; j++) {
l1e = l1[j];
if ((l1e & PTE_V) == 0)
continue;
pa1 = PTE_TO_PHYS(l1e);
m1 = PHYS_TO_VM_PAGE(pa1);
l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1);
for (k = 0; k < Ln_ENTRIES; k++) {
l2e = l2[k];
if ((l2e & PTE_V) == 0)
continue;
pa = PTE_TO_PHYS(l2e);
m = PHYS_TO_VM_PAGE(pa);
l3 = (pt_entry_t *)PHYS_TO_DMAP(pa);
for (l = 0; l < Ln_ENTRIES; l++) {
l3e = l3[l];
if ((l3e & PTE_V) == 0)
continue;
panic("%s: l3e found (idx %d %d %d %d)",
__func__, i, j, k, l);
}
vm_page_unwire_noq(m1);
vm_page_unwire_noq(m);
pmap_resident_count_dec(pmap, 1);
vm_page_free(m);
pmap_clear(&l2[k]);
}
vm_page_unwire_noq(m0);
pmap_resident_count_dec(pmap, 1);
vm_page_free(m1);
pmap_clear(&l1[j]);
}
pmap_resident_count_dec(pmap, 1);
vm_page_free(m0);
pmap_clear(&pmap->pm_top[i]);
}
KASSERT(pmap->sp_resident_count == 0,
("Invalid resident count %jd", pmap->sp_resident_count));
PMAP_UNLOCK(pmap);
}
static void
iommu_pmap_remove_pages_sv39(struct riscv_iommu_pmap *pmap)
{
pd_entry_t l1e, *l2, l2e, *l3, l3e;
vm_paddr_t pa1, pa;
vm_page_t m1, m;
int j, k, l;
PMAP_LOCK(pmap);
for (j = 0; j < Ln_ENTRIES; j++) {
l1e = pmap->pm_top[j];
if ((l1e & PTE_V) == 0)
continue;
pa1 = PTE_TO_PHYS(l1e);
m1 = PHYS_TO_VM_PAGE(pa1);
l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1);
for (k = 0; k < Ln_ENTRIES; k++) {
l2e = l2[k];
if ((l2e & PTE_V) == 0)
continue;
pa = PTE_TO_PHYS(l2e);
m = PHYS_TO_VM_PAGE(pa);
l3 = (pt_entry_t *)PHYS_TO_DMAP(pa);
for (l = 0; l < Ln_ENTRIES; l++) {
l3e = l3[l];
if ((l3e & PTE_V) == 0)
continue;
panic("%s: l3e found (idx %d %d %d)",
__func__, j, k, l);
}
vm_page_unwire_noq(m1);
vm_page_unwire_noq(m);
pmap_resident_count_dec(pmap, 1);
vm_page_free(m);
pmap_clear(&l2[k]);
}
pmap_resident_count_dec(pmap, 1);
vm_page_free(m1);
pmap_clear(&pmap->pm_top[j]);
}
KASSERT(pmap->sp_resident_count == 0,
("Invalid resident count %jd", pmap->sp_resident_count));
PMAP_UNLOCK(pmap);
}
void
iommu_pmap_remove_pages(struct riscv_iommu_pmap *pmap)
{
switch (pmap->pm_mode) {
case PMAP_MODE_SV39:
iommu_pmap_remove_pages_sv39(pmap);
break;
case PMAP_MODE_SV48:
iommu_pmap_remove_pages_sv48(pmap);
break;
default:
panic("Unknown virtual memory system");
}
}
sys/riscv/iommu/iommu_pmap.h
+49
View File
@@ -0,0 +1,49 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _RISCV_IOMMU_IOMMU_PMAP_H_
#define _RISCV_IOMMU_IOMMU_PMAP_H_
struct riscv_iommu_pmap {
pd_entry_t *pm_top;
enum pmap_mode pm_mode;
uint64_t pm_satp;
struct mtx pm_mtx;
#ifdef INVARIANTS
long sp_resident_count;
#endif
};
int iommu_pmap_enter(struct riscv_iommu_pmap *pmap, vm_offset_t va,
vm_paddr_t pa, vm_prot_t prot, u_int flags);
int iommu_pmap_remove(struct riscv_iommu_pmap *pmap, vm_offset_t va);
void iommu_pmap_remove_pages(struct riscv_iommu_pmap *pmap);
int iommu_pmap_pinit(struct riscv_iommu_pmap *pmap, enum pmap_mode pm_mode);
void iommu_pmap_release(struct riscv_iommu_pmap *pmap);
void iommu_pmap_remove_pages(struct riscv_iommu_pmap *pmap);
#endif /* !_RISCV_IOMMU_IOMMU_PMAP_H_ */