From 7aee7aff1226c5d2cce2cbcbfacc5f1a21ddfef7 Mon Sep 17 00:00:00 2001 From: Elliott Mitchell Date: Fri, 4 Aug 2023 11:34:20 -0700 Subject: [PATCH] intrng: break PIC functionality off of main INTRNG file This should serve to truly split things into layers. Perhaps this could then be used on x86 and serve to continue unifying interrupt frameworks. --- sys/conf/files.arm | 1 + sys/conf/files.arm64 | 1 + sys/conf/files.riscv | 1 + sys/kern/subr_intr.c | 1117 ------------------------------------- sys/kern/subr_intrpic.c | 1172 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 1175 insertions(+), 1117 deletions(-) create mode 100644 sys/kern/subr_intrpic.c diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 5ada97ccad3006..5701c525167ec7 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -35,6 +35,7 @@ arm/arm/identcpu-v6.c standard arm/arm/in_cksum_arm.S optional inet | inet6 arm/arm/in_cksum_machdep.c optional inet | inet6 kern/subr_intr.c standard +kern/subr_intrpic.c standard arm/arm/locore.S standard no-obj arm/arm/hypervisor-stub.S standard arm/arm/machdep.c standard diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 3335dfe6cab176..4b22d21f481e13 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -7,6 +7,7 @@ kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_devmap.c standard kern/subr_intr.c optional intrng +kern/subr_intrpic.c optional intrng kern/subr_physmem.c standard libkern/strlen.c standard libkern/arm64/crc32c_armv8.S standard diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 6186ae9b3371a9..93c09a9c5e3083 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -23,6 +23,7 @@ kern/pic_if.m standard kern/subr_devmap.c standard kern/subr_dummy_vdso_tc.c standard kern/subr_intr.c standard +kern/subr_intrpic.c standard kern/subr_physmem.c standard libkern/bcopy.c standard libkern/memcmp.c standard diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c index ecdd0674bd3bdb..d3d31877524c12 100644 --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -40,39 +40,21 @@ */ #include "opt_ddb.h" -#include "opt_hwpmc_hooks.h" -#include "opt_iommu.h" -#include -#include -#include #include #include #include #include #include #include -#include #include #include -#include #include #include -#include -#include -#include #include #include -#include -#include -#include -#include -#ifdef HWPMC_HOOKS -#include -#endif #include -#include #include #include @@ -80,57 +62,13 @@ #include #endif -#ifdef IOMMU -#include -#endif - #include "pic_if.h" -#include "msi_if.h" #define INTRNAME_LEN (2*MAXCOMLEN + 1) -#ifdef DEBUG -#define debugf(fmt, args...) do { printf("%s(): ", __func__); \ - printf(fmt,##args); } while (0) -#else -#define debugf(fmt, args...) -#endif - MALLOC_DECLARE(M_INTRNG); MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); -/* Root interrupt controller stuff. */ -struct intr_irq_root { - device_t dev; - intr_irq_filter_t *filter; - void *arg; -}; - -static struct intr_irq_root intr_irq_roots[INTR_ROOT_COUNT]; - -struct intr_pic_child { - SLIST_ENTRY(intr_pic_child) pc_next; - struct intr_pic *pc_pic; - intr_child_irq_filter_t *pc_filter; - void *pc_filter_arg; - uintptr_t pc_start; - uintptr_t pc_length; -}; - -/* Interrupt controller definition. */ -struct intr_pic { - SLIST_ENTRY(intr_pic) pic_next; - intptr_t pic_xref; /* hardware identification */ - device_t pic_dev; -/* Only one of FLAG_PIC or FLAG_MSI may be set */ -#define FLAG_PIC (1 << 0) -#define FLAG_MSI (1 << 1) -#define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) - u_int pic_flags; - struct mtx pic_child_lock; - SLIST_HEAD(, intr_pic_child) pic_children; -}; - #ifdef SMP #define INTR_IPI_NAMELEN (MAXCOMLEN + 1) @@ -147,11 +85,6 @@ static u_int intr_ipi_dev_priority; static bool intr_ipi_dev_frozen; #endif -static struct mtx pic_list_lock; -static SLIST_HEAD(, intr_pic) pic_list; - -static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, u_int flags); - /* Interrupt source definition. */ static struct mtx isrc_table_lock; static struct intr_irqsrc **irq_sources; @@ -179,12 +112,6 @@ size_t sintrnames; int nintrcnt; static bitstr_t *intrcnt_bitmap; -static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); -static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); -static struct intr_map_data * intr_map_get_map_data(u_int res_id); -static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, - struct intr_map_data **data); - /* * Interrupt framework initialization routine. */ @@ -192,9 +119,6 @@ static void intr_irq_init(void *dummy __unused) { - SLIST_INIT(&pic_list); - mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); - mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); /* @@ -323,64 +247,6 @@ isrc_release_counters(struct intr_irqsrc *isrc) bit_nclear(intrcnt_bitmap, idx, idx + 1); } -/* - * Main interrupt dispatch handler. It's called straight - * from the assembler, where CPU interrupt is served. - */ -void -intr_irq_handler(struct trapframe *tf, uint32_t rootnum) -{ - struct trapframe * oldframe; - struct thread * td; - struct intr_irq_root *root; - - KASSERT(rootnum < INTR_ROOT_COUNT, - ("%s: invalid interrupt root %d", __func__, rootnum)); - - root = &intr_irq_roots[rootnum]; - KASSERT(root->filter != NULL, ("%s: no filter", __func__)); - - kasan_mark(tf, sizeof(*tf), sizeof(*tf), 0); - kmsan_mark(tf, sizeof(*tf), KMSAN_STATE_INITED); - - VM_CNT_INC(v_intr); - critical_enter(); - td = curthread; - oldframe = td->td_intr_frame; - td->td_intr_frame = tf; - (root->filter)(root->arg); - td->td_intr_frame = oldframe; - critical_exit(); -#ifdef HWPMC_HOOKS - if (pmc_hook && TRAPF_USERMODE(tf) && - (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) - pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); -#endif -} - -int -intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) -{ - struct intr_pic_child *child; - bool found; - - found = false; - mtx_lock_spin(&parent->pic_child_lock); - SLIST_FOREACH(child, &parent->pic_children, pc_next) { - if (child->pc_start <= irq && - irq < (child->pc_start + child->pc_length)) { - found = true; - break; - } - } - mtx_unlock_spin(&parent->pic_child_lock); - - if (found) - return (child->pc_filter(child->pc_filter_arg, irq)); - - return (FILTER_STRAY); -} - /* * interrupt controller dispatch function for interrupts. It should * be called straight from the interrupt controller, when associated interrupt @@ -485,14 +351,6 @@ isrc_free_irq(struct intr_irqsrc *isrc) return (0); } -device_t -intr_irq_root_device(uint32_t rootnum) -{ - KASSERT(rootnum < INTR_ROOT_COUNT, - ("%s: invalid interrupt root %d", __func__, rootnum)); - return (intr_irq_roots[rootnum].dev); -} - /* * Initialize interrupt source and register it into global interrupt table. */ @@ -545,32 +403,6 @@ intr_isrc_deregister(struct intr_irqsrc *isrc) return (error); } -#ifdef SMP -/* - * A support function for a PIC to decide if provided ISRC should be inited - * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of - * struct intr_irqsrc is the following: - * - * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus - * set in isrc_cpu. If not, the ISRC should be inited on every cpu and - * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. - */ -bool -intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) -{ - - if (isrc->isrc_handlers == 0) - return (false); - if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) - return (false); - if (isrc->isrc_flags & INTR_ISRCF_BOUND) - return (CPU_ISSET(cpu, &isrc->isrc_cpu)); - - CPU_SET(cpu, &isrc->isrc_cpu); - return (true); -} -#endif - #ifdef INTR_SOLO /* * Setup filter into interrupt source. @@ -635,42 +467,6 @@ intr_isrc_post_filter(void *arg) PIC_POST_FILTER(isrc->isrc_dev, isrc); } -int -intr_assign_irq(struct intr_irqsrc *isrc, int cpu, bool do_assignment) -{ -#ifdef SMP - int error; - - mtx_lock(&pic_list_lock); - if (cpu == NOCPU) { - CPU_ZERO(&isrc->isrc_cpu); - isrc->isrc_flags &= ~INTR_ISRCF_BOUND; - } else { - CPU_SETOF(cpu, &isrc->isrc_cpu); - isrc->isrc_flags |= INTR_ISRCF_BOUND; - } - - /* - * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or - * re-balance it to another CPU or enable it on more CPUs. However, - * PIC is expected to change isrc_cpu appropriately to keep us well - * informed if the call is successful. - */ - if (do_assignment) { - error = PIC_BIND_INTR(isrc->isrc_dev, isrc); - if (error) { - CPU_ZERO(&isrc->isrc_cpu); - mtx_unlock(&pic_list_lock); - return (error); - } - } - mtx_unlock(&pic_list_lock); - return (0); -#else - return (EOPNOTSUPP); -#endif -} - /* * Interrupt source assign_cpu method for MI interrupt framework. */ @@ -761,443 +557,6 @@ intr_add_handler(struct intr_irqsrc *isrc, const char *name, return (error); } -/* - * Lookup interrupt controller locked. - */ -static inline struct intr_pic * -pic_lookup_locked(device_t dev, intptr_t xref, u_int flags) -{ - struct intr_pic *pic; - - mtx_assert(&pic_list_lock, MA_OWNED); - - if (dev == NULL && xref == 0) - return (NULL); - - /* Note that pic->pic_dev is never NULL on registered PIC. */ - SLIST_FOREACH(pic, &pic_list, pic_next) { - if ((pic->pic_flags & FLAG_TYPE_MASK) != - (flags & FLAG_TYPE_MASK)) - continue; - - if (dev == NULL) { - if (xref == pic->pic_xref) - return (pic); - } else if (xref == 0 || pic->pic_xref == 0) { - if (dev == pic->pic_dev) - return (pic); - } else if (xref == pic->pic_xref && dev == pic->pic_dev) - return (pic); - } - return (NULL); -} - -/* - * Lookup interrupt controller. - */ -static struct intr_pic * -pic_lookup(device_t dev, intptr_t xref, u_int flags) -{ - struct intr_pic *pic; - - mtx_lock(&pic_list_lock); - pic = pic_lookup_locked(dev, xref, flags); - mtx_unlock(&pic_list_lock); - return (pic); -} - -/* - * Create interrupt controller. - */ -static struct intr_pic * -pic_create(device_t dev, intptr_t xref, u_int flags) -{ - struct intr_pic *pic; - - mtx_lock(&pic_list_lock); - pic = pic_lookup_locked(dev, xref, flags); - if (pic != NULL) { - mtx_unlock(&pic_list_lock); - return (pic); - } - pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); - if (pic == NULL) { - mtx_unlock(&pic_list_lock); - return (NULL); - } - pic->pic_xref = xref; - pic->pic_dev = dev; - pic->pic_flags = flags; - mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); - SLIST_INSERT_HEAD(&pic_list, pic, pic_next); - mtx_unlock(&pic_list_lock); - - return (pic); -} -#ifdef notyet -/* - * Destroy interrupt controller. - */ -static void -pic_destroy(device_t dev, intptr_t xref, u_int flags) -{ - struct intr_pic *pic; - - mtx_lock(&pic_list_lock); - pic = pic_lookup_locked(dev, xref, flags); - if (pic == NULL) { - mtx_unlock(&pic_list_lock); - return; - } - SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); - mtx_unlock(&pic_list_lock); - - free(pic, M_INTRNG); -} -#endif -/* - * Register interrupt controller. - */ -struct intr_pic * -intr_pic_register(device_t dev, intptr_t xref) -{ - struct intr_pic *pic; - - if (dev == NULL) - return (NULL); - pic = pic_create(dev, xref, FLAG_PIC); - if (pic == NULL) - return (NULL); - - debugf("PIC %p registered for %s \n", pic, - device_get_nameunit(dev), dev, (uintmax_t)xref); - return (pic); -} - -/* - * Unregister interrupt controller. - */ -int -intr_pic_deregister(device_t dev, intptr_t xref) -{ - - panic("%s: not implemented", __func__); -} - -/* - * Mark interrupt controller (itself) as a root one. - * - * Note that only an interrupt controller can really know its position - * in interrupt controller's tree. So root PIC must claim itself as a root. - * - * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, - * page 30: - * "The root of the interrupt tree is determined when traversal - * of the interrupt tree reaches an interrupt controller node without - * an interrupts property and thus no explicit interrupt parent." - */ -int -intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, - void *arg, uint32_t rootnum) -{ - struct intr_pic *pic; - struct intr_irq_root *root; - - pic = pic_lookup(dev, xref, FLAG_PIC); - if (pic == NULL) { - device_printf(dev, "not registered\n"); - return (EINVAL); - } - - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, - ("%s: Found a non-PIC controller: %s", __func__, - device_get_name(pic->pic_dev))); - - if (filter == NULL) { - device_printf(dev, "filter missing\n"); - return (EINVAL); - } - - /* - * Only one interrupt controllers could be on the root for now. - * Note that we further suppose that there is not threaded interrupt - * routine (handler) on the root. See intr_irq_handler(). - */ - KASSERT(rootnum < INTR_ROOT_COUNT, - ("%s: invalid interrupt root %d", __func__, rootnum)); - root = &intr_irq_roots[rootnum]; - if (root->dev != NULL) { - device_printf(dev, "another root already set\n"); - return (EBUSY); - } - - root->dev = dev; - root->filter = filter; - root->arg = arg; - - debugf("irq root set to %s\n", device_get_nameunit(dev)); - return (0); -} - -/* - * Add a handler to manage a sub range of a parents interrupts. - */ -int -intr_pic_add_handler(device_t parent, struct intr_pic *pic, - intr_child_irq_filter_t *filter, void *arg, uintptr_t start, - uintptr_t length) -{ - struct intr_pic *parent_pic; - struct intr_pic_child *newchild; -#ifdef INVARIANTS - struct intr_pic_child *child; -#endif - - /* Find the parent PIC */ - parent_pic = pic_lookup(parent, 0, FLAG_PIC); - if (parent_pic == NULL) - return (ENXIO); - - newchild = malloc(sizeof(*newchild), M_INTRNG, M_WAITOK | M_ZERO); - newchild->pc_pic = pic; - newchild->pc_filter = filter; - newchild->pc_filter_arg = arg; - newchild->pc_start = start; - newchild->pc_length = length; - - mtx_lock_spin(&parent_pic->pic_child_lock); -#ifdef INVARIANTS - SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { - KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", - __func__)); - } -#endif - SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); - mtx_unlock_spin(&parent_pic->pic_child_lock); - - return (0); -} - -static int -intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, - struct intr_irqsrc **isrc) -{ - struct intr_pic *pic; - struct intr_map_data_msi *msi; - - if (data == NULL) - return (EINVAL); - - pic = pic_lookup(dev, xref, - (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); - if (pic == NULL) - return (ESRCH); - - switch (data->type) { - case INTR_MAP_DATA_MSI: - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, - ("%s: Found a non-MSI controller: %s", __func__, - device_get_name(pic->pic_dev))); - msi = (struct intr_map_data_msi *)data; - *isrc = msi->isrc; - return (0); - - default: - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, - ("%s: Found a non-PIC controller: %s", __func__, - device_get_name(pic->pic_dev))); - return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); - } -} - -bool -intr_is_per_cpu(struct resource *res) -{ - u_int res_id; - struct intr_irqsrc *isrc; - - res_id = (u_int)rman_get_start(res); - isrc = intr_map_get_isrc(res_id); - - if (isrc == NULL) - panic("Attempt to get isrc for non-active resource id: %u\n", - res_id); - return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); -} - -int -intr_activate_irq(device_t dev, struct resource *res) -{ - device_t map_dev; - intptr_t map_xref; - struct intr_map_data *data; - struct intr_irqsrc *isrc; - u_int res_id; - int error; - - KASSERT(rman_get_start(res) == rman_get_end(res), - ("%s: more interrupts in resource", __func__)); - - res_id = (u_int)rman_get_start(res); - if (intr_map_get_isrc(res_id) != NULL) - panic("Attempt to double activation of resource id: %u\n", - res_id); - intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); - error = intr_resolve_irq(map_dev, map_xref, data, &isrc); - if (error != 0) { - free(data, M_INTRNG); - /* XXX TODO DISCONECTED PICs */ - /* if (error == EINVAL) return(0); */ - return (error); - } - intr_map_set_isrc(res_id, isrc); - rman_set_virtual(res, data); - return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); -} - -int -intr_deactivate_irq(device_t dev, struct resource *res) -{ - struct intr_map_data *data; - struct intr_irqsrc *isrc; - u_int res_id; - int error; - - KASSERT(rman_get_start(res) == rman_get_end(res), - ("%s: more interrupts in resource", __func__)); - - res_id = (u_int)rman_get_start(res); - isrc = intr_map_get_isrc(res_id); - if (isrc == NULL) - panic("Attempt to deactivate non-active resource id: %u\n", - res_id); - - data = rman_get_virtual(res); - error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); - intr_map_set_isrc(res_id, NULL); - rman_set_virtual(res, NULL); - free(data, M_INTRNG); - return (error); -} - -int -intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, - driver_intr_t hand, void *arg, int flags, void **cookiep) -{ - int error; - struct intr_map_data *data; - struct intr_irqsrc *isrc; - const char *name; - u_int res_id; - - KASSERT(rman_get_start(res) == rman_get_end(res), - ("%s: more interrupts in resource", __func__)); - - res_id = (u_int)rman_get_start(res); - isrc = intr_map_get_isrc(res_id); - if (isrc == NULL) { - /* XXX TODO DISCONECTED PICs */ - return (EINVAL); - } - - data = rman_get_virtual(res); - name = device_get_nameunit(dev); - -#ifdef INTR_SOLO - /* - * Standard handling is done through MI interrupt framework. However, - * some interrupts could request solely own special handling. This - * non standard handling can be used for interrupt controllers without - * handler (filter only), so in case that interrupt controllers are - * chained, MI interrupt framework is called only in leaf controller. - * - * Note that root interrupt controller routine is served as well, - * however in intr_irq_handler(), i.e. main system dispatch routine. - */ - if (flags & INTR_SOLO && hand != NULL) { - debugf("irq %u cannot solo on %s\n", irq, name); - return (EINVAL); - } - - if (flags & INTR_SOLO) { - error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, - arg, cookiep); - debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error, - name); - } else -#endif - { - error = intr_add_handler(isrc, name, filt, hand, arg, flags, - cookiep); - debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name); - } - if (error != 0) - return (error); - - mtx_lock(&pic_list_lock); - error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); - if (error == 0) { - isrc->isrc_handlers++; - if (isrc->isrc_handlers == 1) - PIC_ENABLE_INTR(isrc->isrc_dev, isrc); - } - mtx_unlock(&pic_list_lock); - if (error != 0) - intr_event_remove_handler(*cookiep); - return (error); -} - -int -intr_teardown_irq(device_t dev, struct resource *res, void *cookie) -{ - int error; - struct intr_map_data *data; - struct intr_irqsrc *isrc; - u_int res_id; - - KASSERT(rman_get_start(res) == rman_get_end(res), - ("%s: more interrupts in resource", __func__)); - - res_id = (u_int)rman_get_start(res); - isrc = intr_map_get_isrc(res_id); - if (isrc == NULL || isrc->isrc_handlers == 0) - return (EINVAL); - - data = rman_get_virtual(res); - -#ifdef INTR_SOLO - if (isrc->isrc_filter != NULL) { - if (isrc != cookie) - return (EINVAL); - - mtx_lock(&isrc_table_lock); - isrc->isrc_filter = NULL; - isrc->isrc_arg = NULL; - isrc->isrc_handlers = 0; - PIC_DISABLE_INTR(isrc->isrc_dev, isrc); - PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); - isrc_update_name(isrc, NULL); - mtx_unlock(&isrc_table_lock); - return (0); - } -#endif - if (isrc != intr_handler_source(cookie)) - return (EINVAL); - - error = intr_event_remove_handler(cookie); - if (error == 0) { - mtx_lock(&pic_list_lock); - isrc->isrc_handlers--; - if (isrc->isrc_handlers == 0) - PIC_DISABLE_INTR(isrc->isrc_dev, isrc); - PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); - mtx_unlock(&pic_list_lock); - - intr_describe(isrc, NULL, NULL); - } - return (error); -} - /* * Describe an interrupt */ @@ -1230,44 +589,7 @@ intr_describe(struct intr_irqsrc *isrc, void *cookie, const char *descr) return (error); } -int -intr_describe_irq(device_t dev, struct resource *res, void *cookie, - const char *descr) -{ - struct intr_irqsrc *isrc; - u_int res_id; - - KASSERT(rman_get_start(res) == rman_get_end(res), - ("%s: more interrupts in resource", __func__)); - - res_id = (u_int)rman_get_start(res); - isrc = intr_map_get_isrc(res_id); - if (isrc == NULL || isrc->isrc_handlers == 0) - return (EINVAL); - return (intr_describe(isrc, cookie, descr)); -} - #ifdef SMP -int -intr_bind_irq(device_t dev, struct resource *res, int cpu) -{ - struct intr_irqsrc *isrc; - u_int res_id; - - KASSERT(rman_get_start(res) == rman_get_end(res), - ("%s: more interrupts in resource", __func__)); - - res_id = (u_int)rman_get_start(res); - isrc = intr_map_get_isrc(res_id); - if (isrc == NULL || isrc->isrc_handlers == 0) - return (EINVAL); -#ifdef INTR_SOLO - if (isrc->isrc_filter != NULL) - return (intr_isrc_assign_cpu(isrc, cpu)); -#endif - return (intr_event_bind(isrc->isrc_event, cpu)); -} - /* * Return the CPU that the next interrupt source should use. * For now just returns the next CPU according to round-robin. @@ -1347,268 +669,12 @@ intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) } #endif /* SMP */ -/* - * Allocate memory for new intr_map_data structure. - * Initialize common fields. - */ -struct intr_map_data * -intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) -{ - struct intr_map_data *data; - - data = malloc(len, M_INTRNG, flags); - data->type = type; - data->len = len; - return (data); -} - -void intr_free_intr_map_data(struct intr_map_data *data) -{ - - free(data, M_INTRNG); -} - -/* - * Register a MSI/MSI-X interrupt controller - */ -int -intr_msi_register(device_t dev, intptr_t xref) -{ - struct intr_pic *pic; - - if (dev == NULL) - return (EINVAL); - pic = pic_create(dev, xref, FLAG_MSI); - if (pic == NULL) - return (ENOMEM); - - debugf("PIC %p registered for %s \n", pic, - device_get_nameunit(dev), dev, (uintmax_t)xref); - return (0); -} - -int -intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, - int maxcount, int *irqs) -{ - struct iommu_domain *domain; - struct intr_irqsrc **isrc; - struct intr_pic *pic; - device_t pdev; - struct intr_map_data_msi *msi; - int err, i; - - pic = pic_lookup(NULL, xref, FLAG_MSI); - if (pic == NULL) - return (ESRCH); - - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, - ("%s: Found a non-MSI controller: %s", __func__, - device_get_name(pic->pic_dev))); - - /* - * If this is the first time we have used this context ask the - * interrupt controller to map memory the msi source will need. - */ - err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); - if (err != 0) - return (err); - - isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); - err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); - if (err != 0) { - free(isrc, M_INTRNG); - return (err); - } - - for (i = 0; i < count; i++) { - isrc[i]->isrc_iommu = domain; - msi = (struct intr_map_data_msi *)intr_alloc_map_data( - INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); - msi-> isrc = isrc[i]; - - irqs[i] = intr_map_irq(pic->pic_dev, xref, - (struct intr_map_data *)msi); - } - free(isrc, M_INTRNG); - - return (err); -} - -int -intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, - int *irqs) -{ - struct intr_irqsrc **isrc; - struct intr_pic *pic; - struct intr_map_data_msi *msi; - int i, err; - - pic = pic_lookup(NULL, xref, FLAG_MSI); - if (pic == NULL) - return (ESRCH); - - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, - ("%s: Found a non-MSI controller: %s", __func__, - device_get_name(pic->pic_dev))); - - isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); - - for (i = 0; i < count; i++) { - msi = (struct intr_map_data_msi *) - intr_map_get_map_data(irqs[i]); - KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, - ("%s: irq %d map data is not MSI", __func__, - irqs[i])); - isrc[i] = msi->isrc; - } - - MSI_IOMMU_DEINIT(pic->pic_dev, child); - - err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); - - for (i = 0; i < count; i++) { - if (isrc[i] != NULL) - intr_unmap_irq(irqs[i]); - } - - free(isrc, M_INTRNG); - return (err); -} - -int -intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) -{ - struct iommu_domain *domain; - struct intr_irqsrc *isrc; - struct intr_pic *pic; - device_t pdev; - struct intr_map_data_msi *msi; - int err; - - pic = pic_lookup(NULL, xref, FLAG_MSI); - if (pic == NULL) - return (ESRCH); - - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, - ("%s: Found a non-MSI controller: %s", __func__, - device_get_name(pic->pic_dev))); - - /* - * If this is the first time we have used this context ask the - * interrupt controller to map memory the msi source will need. - */ - err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); - if (err != 0) - return (err); - - err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); - if (err != 0) - return (err); - - isrc->isrc_iommu = domain; - msi = (struct intr_map_data_msi *)intr_alloc_map_data( - INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); - msi->isrc = isrc; - *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); - return (0); -} - -int -intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) -{ - struct intr_irqsrc *isrc; - struct intr_pic *pic; - struct intr_map_data_msi *msi; - int err; - - pic = pic_lookup(NULL, xref, FLAG_MSI); - if (pic == NULL) - return (ESRCH); - - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, - ("%s: Found a non-MSI controller: %s", __func__, - device_get_name(pic->pic_dev))); - - msi = (struct intr_map_data_msi *) - intr_map_get_map_data(irq); - KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, - ("%s: irq %d map data is not MSI", __func__, - irq)); - isrc = msi->isrc; - if (isrc == NULL) { - intr_unmap_irq(irq); - return (EINVAL); - } - - MSI_IOMMU_DEINIT(pic->pic_dev, child); - - err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); - intr_unmap_irq(irq); - - return (err); -} - -int -intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, - uint64_t *addr, uint32_t *data) -{ - struct intr_irqsrc *isrc; - struct intr_pic *pic; - int err; - - pic = pic_lookup(NULL, xref, FLAG_MSI); - if (pic == NULL) - return (ESRCH); - - KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, - ("%s: Found a non-MSI controller: %s", __func__, - device_get_name(pic->pic_dev))); - - isrc = intr_map_get_isrc(irq); - if (isrc == NULL) - return (EINVAL); - - err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); - -#ifdef IOMMU - if (isrc->isrc_iommu != NULL) - iommu_translate_msi(isrc->isrc_iommu, addr); -#endif - - return (err); -} - void dosoftints(void); void dosoftints(void) { } -#ifdef SMP -/* - * Init interrupt controller on another CPU. - */ -void -intr_pic_init_secondary(void) -{ - device_t dev; - uint32_t rootnum; - - /* - * QQQ: Only root PICs are aware of other CPUs ??? - */ - //mtx_lock(&pic_list_lock); - for (rootnum = 0; rootnum < INTR_ROOT_COUNT; rootnum++) { - dev = intr_irq_roots[rootnum].dev; - if (dev != NULL) { - PIC_INIT_SECONDARY(dev, rootnum); - } - } - //mtx_unlock(&pic_list_lock); -} -#endif - #ifdef DDB DB_SHOW_COMMAND_FLAGS(irqs, db_show_irqs, DB_CMD_MEMSAFE) { @@ -1631,189 +697,6 @@ DB_SHOW_COMMAND_FLAGS(irqs, db_show_irqs, DB_CMD_MEMSAFE) } #endif -/* - * Interrupt mapping table functions. - * - * Please, keep this part separately, it can be transformed to - * extension of standard resources. - */ -struct intr_map_entry -{ - device_t dev; - intptr_t xref; - struct intr_map_data *map_data; - struct intr_irqsrc *isrc; - /* XXX TODO DISCONECTED PICs */ - /*int flags */ -}; - -/* XXX Convert irq_map[] to dynamicaly expandable one. */ -static struct intr_map_entry **irq_map; -static u_int irq_map_count; -static u_int irq_map_first_free_idx; -static struct mtx irq_map_lock; - -static struct intr_irqsrc * -intr_map_get_isrc(u_int res_id) -{ - struct intr_irqsrc *isrc; - - isrc = NULL; - mtx_lock(&irq_map_lock); - if (res_id < irq_map_count && irq_map[res_id] != NULL) - isrc = irq_map[res_id]->isrc; - mtx_unlock(&irq_map_lock); - - return (isrc); -} - -static void -intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) -{ - - mtx_lock(&irq_map_lock); - if (res_id < irq_map_count && irq_map[res_id] != NULL) - irq_map[res_id]->isrc = isrc; - mtx_unlock(&irq_map_lock); -} - -/* - * Get a copy of intr_map_entry data - */ -static struct intr_map_data * -intr_map_get_map_data(u_int res_id) -{ - struct intr_map_data *data; - - data = NULL; - mtx_lock(&irq_map_lock); - if (res_id >= irq_map_count || irq_map[res_id] == NULL) - panic("Attempt to copy invalid resource id: %u\n", res_id); - data = irq_map[res_id]->map_data; - mtx_unlock(&irq_map_lock); - - return (data); -} - -/* - * Get a copy of intr_map_entry data - */ -static void -intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, - struct intr_map_data **data) -{ - size_t len; - - len = 0; - mtx_lock(&irq_map_lock); - if (res_id >= irq_map_count || irq_map[res_id] == NULL) - panic("Attempt to copy invalid resource id: %u\n", res_id); - if (irq_map[res_id]->map_data != NULL) - len = irq_map[res_id]->map_data->len; - mtx_unlock(&irq_map_lock); - - if (len == 0) - *data = NULL; - else - *data = malloc(len, M_INTRNG, M_WAITOK | M_ZERO); - mtx_lock(&irq_map_lock); - if (irq_map[res_id] == NULL) - panic("Attempt to copy invalid resource id: %u\n", res_id); - if (len != 0) { - if (len != irq_map[res_id]->map_data->len) - panic("Resource id: %u has changed.\n", res_id); - memcpy(*data, irq_map[res_id]->map_data, len); - } - *map_dev = irq_map[res_id]->dev; - *map_xref = irq_map[res_id]->xref; - mtx_unlock(&irq_map_lock); -} - -/* - * Allocate and fill new entry in irq_map table. - */ -u_int -intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) -{ - u_int i; - struct intr_map_entry *entry; - - /* Prepare new entry first. */ - entry = malloc(sizeof(*entry), M_INTRNG, M_WAITOK | M_ZERO); - - entry->dev = dev; - entry->xref = xref; - entry->map_data = data; - entry->isrc = NULL; - - mtx_lock(&irq_map_lock); - for (i = irq_map_first_free_idx; i < irq_map_count; i++) { - if (irq_map[i] == NULL) { - irq_map[i] = entry; - irq_map_first_free_idx = i + 1; - mtx_unlock(&irq_map_lock); - return (i); - } - } - for (i = 0; i < irq_map_first_free_idx; i++) { - if (irq_map[i] == NULL) { - irq_map[i] = entry; - irq_map_first_free_idx = i + 1; - mtx_unlock(&irq_map_lock); - return (i); - } - } - mtx_unlock(&irq_map_lock); - - /* XXX Expand irq_map table */ - panic("IRQ mapping table is full."); -} - -/* - * Remove and free mapping entry. - */ -void -intr_unmap_irq(u_int res_id) -{ - struct intr_map_entry *entry; - - mtx_lock(&irq_map_lock); - if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) - panic("Attempt to unmap invalid resource id: %u\n", res_id); - entry = irq_map[res_id]; - irq_map[res_id] = NULL; - irq_map_first_free_idx = res_id; - mtx_unlock(&irq_map_lock); - intr_free_intr_map_data(entry->map_data); - free(entry, M_INTRNG); -} - -/* - * Clone mapping entry. - */ -u_int -intr_map_clone_irq(u_int old_res_id) -{ - device_t map_dev; - intptr_t map_xref; - struct intr_map_data *data; - - intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); - return (intr_map_irq(map_dev, map_xref, data)); -} - -static void -intr_map_init(void *dummy __unused) -{ - - mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); - - irq_map_count = 2 * intr_nirq; - irq_map = mallocarray(irq_map_count, sizeof(struct intr_map_entry*), - M_INTRNG, M_WAITOK | M_ZERO); -} -SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); - #ifdef SMP /* Virtualization for interrupt source IPI counter increment. */ static inline void diff --git a/sys/kern/subr_intrpic.c b/sys/kern/subr_intrpic.c new file mode 100644 index 00000000000000..a0245bc48584f0 --- /dev/null +++ b/sys/kern/subr_intrpic.c @@ -0,0 +1,1172 @@ +/*- + * Copyright (c) 2015-2016 Svatopluk Kraus + * Copyright (c) 2015-2016 Michal Meloun + * All rights reserved. + * Copyright © 2023-2024 Elliott Mitchell + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +/* + * New-style Interrupt Framework + * + * TODO: - add support for disconnected PICs. + * - to support IPI (PPI) enabling on other CPUs if already started. + * - to complete things for removable PICs. + */ + +#include "opt_hwpmc_hooks.h" +#include "opt_iommu.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HWPMC_HOOKS +#include +#endif + +#ifdef IOMMU +#include +#endif + +#include "pic_if.h" +#include "msi_if.h" + +#ifdef DEBUG +#define debugf(fmt, args...) do { printf("%s(): ", __func__); \ + printf(fmt,##args); } while (0) +#else +#define debugf(fmt, args...) +#endif + +MALLOC_DECLARE(M_INTRPIC); +MALLOC_DEFINE(M_INTRPIC, "intr", "interrupt PIC handling"); + +/* Root interrupt controller stuff. */ +struct intr_irq_root { + device_t dev; + intr_irq_filter_t *filter; + void *arg; +}; + +static struct intr_irq_root intr_irq_roots[INTR_ROOT_COUNT]; + +struct intr_pic_child { + SLIST_ENTRY(intr_pic_child) pc_next; + struct intr_pic *pc_pic; + intr_child_irq_filter_t *pc_filter; + void *pc_filter_arg; + uintptr_t pc_start; + uintptr_t pc_length; +}; + +/* Interrupt controller definition. */ +struct intr_pic { + SLIST_ENTRY(intr_pic) pic_next; + intptr_t pic_xref; /* hardware identification */ + device_t pic_dev; +/* Only one of FLAG_PIC or FLAG_MSI may be set */ +#define FLAG_PIC (1 << 0) +#define FLAG_MSI (1 << 1) +#define FLAG_TYPE_MASK (FLAG_PIC | FLAG_MSI) + u_int pic_flags; + struct mtx pic_child_lock; + SLIST_HEAD(, intr_pic_child) pic_children; +}; + +static struct mtx pic_list_lock; +static SLIST_HEAD(, intr_pic) pic_list; + +static struct intr_pic *pic_lookup(device_t dev, intptr_t xref, u_int flags); + +static struct intr_irqsrc *intr_map_get_isrc(u_int res_id); +static void intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc); +static struct intr_map_data * intr_map_get_map_data(u_int res_id); +static void intr_map_copy_map_data(u_int res_id, device_t *dev, intptr_t *xref, + struct intr_map_data **data); + +/* + * Interrupt framework initialization routine. + */ +static void +intr_irq_init(void *dummy __unused) +{ + + SLIST_INIT(&pic_list); + mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); +} +SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); + +/* + * Main interrupt dispatch handler. It's called straight + * from the assembler, where CPU interrupt is served. + */ +void +intr_irq_handler(struct trapframe *tf, uint32_t rootnum) +{ + struct trapframe * oldframe; + struct thread * td; + struct intr_irq_root *root; + + KASSERT(rootnum < INTR_ROOT_COUNT, + ("%s: invalid interrupt root %d", __func__, rootnum)); + + root = &intr_irq_roots[rootnum]; + KASSERT(root->filter != NULL, ("%s: no filter", __func__)); + + kasan_mark(tf, sizeof(*tf), sizeof(*tf), 0); + kmsan_mark(tf, sizeof(*tf), KMSAN_STATE_INITED); + + VM_CNT_INC(v_intr); + critical_enter(); + td = curthread; + oldframe = td->td_intr_frame; + td->td_intr_frame = tf; + (root->filter)(root->arg); + td->td_intr_frame = oldframe; + critical_exit(); +#ifdef HWPMC_HOOKS + if (pmc_hook && TRAPF_USERMODE(tf) && + (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) + pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); +#endif +} + +int +intr_child_irq_handler(struct intr_pic *parent, uintptr_t irq) +{ + struct intr_pic_child *child; + bool found; + + found = false; + mtx_lock_spin(&parent->pic_child_lock); + SLIST_FOREACH(child, &parent->pic_children, pc_next) { + if (child->pc_start <= irq && + irq < (child->pc_start + child->pc_length)) { + found = true; + break; + } + } + mtx_unlock_spin(&parent->pic_child_lock); + + if (found) + return (child->pc_filter(child->pc_filter_arg, irq)); + + return (FILTER_STRAY); +} + +device_t +intr_irq_root_device(uint32_t rootnum) +{ + KASSERT(rootnum < INTR_ROOT_COUNT, + ("%s: invalid interrupt root %d", __func__, rootnum)); + return (intr_irq_roots[rootnum].dev); +} + +#ifdef SMP +/* + * A support function for a PIC to decide if provided ISRC should be inited + * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of + * struct intr_irqsrc is the following: + * + * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus + * set in isrc_cpu. If not, the ISRC should be inited on every cpu and + * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. + */ +bool +intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) +{ + + if (isrc->isrc_handlers == 0) + return (false); + if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) + return (false); + if (isrc->isrc_flags & INTR_ISRCF_BOUND) + return (CPU_ISSET(cpu, &isrc->isrc_cpu)); + + CPU_SET(cpu, &isrc->isrc_cpu); + return (true); +} +#endif + +int +intr_assign_irq(struct intr_irqsrc *isrc, int cpu, bool do_assignment) +{ +#ifdef SMP + int error; + + mtx_lock(&pic_list_lock); + if (cpu == NOCPU) { + CPU_ZERO(&isrc->isrc_cpu); + isrc->isrc_flags &= ~INTR_ISRCF_BOUND; + } else { + CPU_SETOF(cpu, &isrc->isrc_cpu); + isrc->isrc_flags |= INTR_ISRCF_BOUND; + } + + /* + * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or + * re-balance it to another CPU or enable it on more CPUs. However, + * PIC is expected to change isrc_cpu appropriately to keep us well + * informed if the call is successful. + */ + if (do_assignment) { + error = PIC_BIND_INTR(isrc->isrc_dev, isrc); + if (error) { + CPU_ZERO(&isrc->isrc_cpu); + mtx_unlock(&pic_list_lock); + return (error); + } + } + mtx_unlock(&pic_list_lock); + return (0); +#else + return (EOPNOTSUPP); +#endif +} + +/* + * Lookup interrupt controller locked. + */ +static inline struct intr_pic * +pic_lookup_locked(device_t dev, intptr_t xref, u_int flags) +{ + struct intr_pic *pic; + + mtx_assert(&pic_list_lock, MA_OWNED); + + if (dev == NULL && xref == 0) + return (NULL); + + /* Note that pic->pic_dev is never NULL on registered PIC. */ + SLIST_FOREACH(pic, &pic_list, pic_next) { + if ((pic->pic_flags & FLAG_TYPE_MASK) != + (flags & FLAG_TYPE_MASK)) + continue; + + if (dev == NULL) { + if (xref == pic->pic_xref) + return (pic); + } else if (xref == 0 || pic->pic_xref == 0) { + if (dev == pic->pic_dev) + return (pic); + } else if (xref == pic->pic_xref && dev == pic->pic_dev) + return (pic); + } + return (NULL); +} + +/* + * Lookup interrupt controller. + */ +static struct intr_pic * +pic_lookup(device_t dev, intptr_t xref, u_int flags) +{ + struct intr_pic *pic; + + mtx_lock(&pic_list_lock); + pic = pic_lookup_locked(dev, xref, flags); + mtx_unlock(&pic_list_lock); + return (pic); +} + +/* + * Create interrupt controller. + */ +static struct intr_pic * +pic_create(device_t dev, intptr_t xref, u_int flags) +{ + struct intr_pic *pic; + + mtx_lock(&pic_list_lock); + pic = pic_lookup_locked(dev, xref, flags); + if (pic != NULL) { + mtx_unlock(&pic_list_lock); + return (pic); + } + pic = malloc(sizeof(*pic), M_INTRPIC, M_NOWAIT | M_ZERO); + if (pic == NULL) { + mtx_unlock(&pic_list_lock); + return (NULL); + } + pic->pic_xref = xref; + pic->pic_dev = dev; + pic->pic_flags = flags; + mtx_init(&pic->pic_child_lock, "pic child lock", NULL, MTX_SPIN); + SLIST_INSERT_HEAD(&pic_list, pic, pic_next); + mtx_unlock(&pic_list_lock); + + return (pic); +} +#ifdef notyet +/* + * Destroy interrupt controller. + */ +static void +pic_destroy(device_t dev, intptr_t xref, u_int flags) +{ + struct intr_pic *pic; + + mtx_lock(&pic_list_lock); + pic = pic_lookup_locked(dev, xref, flags); + if (pic == NULL) { + mtx_unlock(&pic_list_lock); + return; + } + SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); + mtx_unlock(&pic_list_lock); + + free(pic, M_INTRPIC); +} +#endif +/* + * Register interrupt controller. + */ +struct intr_pic * +intr_pic_register(device_t dev, intptr_t xref) +{ + struct intr_pic *pic; + + if (dev == NULL) + return (NULL); + pic = pic_create(dev, xref, FLAG_PIC); + if (pic == NULL) + return (NULL); + + debugf("PIC %p registered for %s \n", pic, + device_get_nameunit(dev), dev, (uintmax_t)xref); + return (pic); +} + +/* + * Unregister interrupt controller. + */ +int +intr_pic_deregister(device_t dev, intptr_t xref) +{ + + panic("%s: not implemented", __func__); +} + +/* + * Mark interrupt controller (itself) as a root one. + * + * Note that only an interrupt controller can really know its position + * in interrupt controller's tree. So root PIC must claim itself as a root. + * + * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, + * page 30: + * "The root of the interrupt tree is determined when traversal + * of the interrupt tree reaches an interrupt controller node without + * an interrupts property and thus no explicit interrupt parent." + */ +int +intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, + void *arg, uint32_t rootnum) +{ + struct intr_pic *pic; + struct intr_irq_root *root; + + pic = pic_lookup(dev, xref, FLAG_PIC); + if (pic == NULL) { + device_printf(dev, "not registered\n"); + return (EINVAL); + } + + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, + ("%s: Found a non-PIC controller: %s", __func__, + device_get_name(pic->pic_dev))); + + if (filter == NULL) { + device_printf(dev, "filter missing\n"); + return (EINVAL); + } + + /* + * Only one interrupt controllers could be on the root for now. + * Note that we further suppose that there is not threaded interrupt + * routine (handler) on the root. See intr_irq_handler(). + */ + KASSERT(rootnum < INTR_ROOT_COUNT, + ("%s: invalid interrupt root %d", __func__, rootnum)); + root = &intr_irq_roots[rootnum]; + if (root->dev != NULL) { + device_printf(dev, "another root already set\n"); + return (EBUSY); + } + + root->dev = dev; + root->filter = filter; + root->arg = arg; + + debugf("irq root set to %s\n", device_get_nameunit(dev)); + return (0); +} + +/* + * Add a handler to manage a sub range of a parents interrupts. + */ +int +intr_pic_add_handler(device_t parent, struct intr_pic *pic, + intr_child_irq_filter_t *filter, void *arg, uintptr_t start, + uintptr_t length) +{ + struct intr_pic *parent_pic; + struct intr_pic_child *newchild; +#ifdef INVARIANTS + struct intr_pic_child *child; +#endif + + /* Find the parent PIC */ + parent_pic = pic_lookup(parent, 0, FLAG_PIC); + if (parent_pic == NULL) + return (ENXIO); + + newchild = malloc(sizeof(*newchild), M_INTRPIC, M_WAITOK | M_ZERO); + newchild->pc_pic = pic; + newchild->pc_filter = filter; + newchild->pc_filter_arg = arg; + newchild->pc_start = start; + newchild->pc_length = length; + + mtx_lock_spin(&parent_pic->pic_child_lock); +#ifdef INVARIANTS + SLIST_FOREACH(child, &parent_pic->pic_children, pc_next) { + KASSERT(child->pc_pic != pic, ("%s: Adding a child PIC twice", + __func__)); + } +#endif + SLIST_INSERT_HEAD(&parent_pic->pic_children, newchild, pc_next); + mtx_unlock_spin(&parent_pic->pic_child_lock); + + return (0); +} + +static int +intr_resolve_irq(device_t dev, intptr_t xref, struct intr_map_data *data, + struct intr_irqsrc **isrc) +{ + struct intr_pic *pic; + struct intr_map_data_msi *msi; + + if (data == NULL) + return (EINVAL); + + pic = pic_lookup(dev, xref, + (data->type == INTR_MAP_DATA_MSI) ? FLAG_MSI : FLAG_PIC); + if (pic == NULL) + return (ESRCH); + + switch (data->type) { + case INTR_MAP_DATA_MSI: + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + msi = (struct intr_map_data_msi *)data; + *isrc = msi->isrc; + return (0); + + default: + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_PIC, + ("%s: Found a non-PIC controller: %s", __func__, + device_get_name(pic->pic_dev))); + return (PIC_MAP_INTR(pic->pic_dev, data, isrc)); + } +} + +bool +intr_is_per_cpu(struct resource *res) +{ + u_int res_id; + struct intr_irqsrc *isrc; + + res_id = (u_int)rman_get_start(res); + isrc = intr_map_get_isrc(res_id); + + if (isrc == NULL) + panic("Attempt to get isrc for non-active resource id: %u\n", + res_id); + return ((isrc->isrc_flags & INTR_ISRCF_PPI) != 0); +} + +int +intr_activate_irq(device_t dev, struct resource *res) +{ + device_t map_dev; + intptr_t map_xref; + struct intr_map_data *data; + struct intr_irqsrc *isrc; + u_int res_id; + int error; + + KASSERT(rman_get_start(res) == rman_get_end(res), + ("%s: more interrupts in resource", __func__)); + + res_id = (u_int)rman_get_start(res); + if (intr_map_get_isrc(res_id) != NULL) + panic("Attempt to double activation of resource id: %u\n", + res_id); + intr_map_copy_map_data(res_id, &map_dev, &map_xref, &data); + error = intr_resolve_irq(map_dev, map_xref, data, &isrc); + if (error != 0) { + free(data, M_INTRPIC); + /* XXX TODO DISCONECTED PICs */ + /* if (error == EINVAL) return(0); */ + return (error); + } + intr_map_set_isrc(res_id, isrc); + rman_set_virtual(res, data); + return (PIC_ACTIVATE_INTR(isrc->isrc_dev, isrc, res, data)); +} + +int +intr_deactivate_irq(device_t dev, struct resource *res) +{ + struct intr_map_data *data; + struct intr_irqsrc *isrc; + u_int res_id; + int error; + + KASSERT(rman_get_start(res) == rman_get_end(res), + ("%s: more interrupts in resource", __func__)); + + res_id = (u_int)rman_get_start(res); + isrc = intr_map_get_isrc(res_id); + if (isrc == NULL) + panic("Attempt to deactivate non-active resource id: %u\n", + res_id); + + data = rman_get_virtual(res); + error = PIC_DEACTIVATE_INTR(isrc->isrc_dev, isrc, res, data); + intr_map_set_isrc(res_id, NULL); + rman_set_virtual(res, NULL); + free(data, M_INTRPIC); + return (error); +} + +int +intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, + driver_intr_t hand, void *arg, int flags, void **cookiep) +{ + int error; + struct intr_map_data *data; + struct intr_irqsrc *isrc; + const char *name; + u_int res_id; + + KASSERT(rman_get_start(res) == rman_get_end(res), + ("%s: more interrupts in resource", __func__)); + + res_id = (u_int)rman_get_start(res); + isrc = intr_map_get_isrc(res_id); + if (isrc == NULL) { + /* XXX TODO DISCONECTED PICs */ + return (EINVAL); + } + + data = rman_get_virtual(res); + name = device_get_nameunit(dev); + +#ifdef INTR_SOLO + /* + * Standard handling is done through MI interrupt framework. However, + * some interrupts could request solely own special handling. This + * non standard handling can be used for interrupt controllers without + * handler (filter only), so in case that interrupt controllers are + * chained, MI interrupt framework is called only in leaf controller. + * + * Note that root interrupt controller routine is served as well, + * however in intr_irq_handler(), i.e. main system dispatch routine. + */ + if (flags & INTR_SOLO && hand != NULL) { + debugf("irq %u cannot solo on %s\n", irq, name); + return (EINVAL); + } + + if (flags & INTR_SOLO) { + error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, + arg, cookiep); + debugf("irq %u setup filter error %d on %s\n", isrc->isrc_irq, error, + name); + } else +#endif + { + error = intr_add_handler(isrc, name, filt, hand, arg, flags, + cookiep); + debugf("irq %u add handler error %d on %s\n", isrc->isrc_irq, error, name); + } + if (error != 0) + return (error); + + mtx_lock(&pic_list_lock); + error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); + if (error == 0) { + isrc->isrc_handlers++; + if (isrc->isrc_handlers == 1) + PIC_ENABLE_INTR(isrc->isrc_dev, isrc); + } + mtx_unlock(&pic_list_lock); + if (error != 0) + intr_event_remove_handler(*cookiep); + return (error); +} + +int +intr_teardown_irq(device_t dev, struct resource *res, void *cookie) +{ + int error; + struct intr_map_data *data; + struct intr_irqsrc *isrc; + u_int res_id; + + KASSERT(rman_get_start(res) == rman_get_end(res), + ("%s: more interrupts in resource", __func__)); + + res_id = (u_int)rman_get_start(res); + isrc = intr_map_get_isrc(res_id); + if (isrc == NULL || isrc->isrc_handlers == 0) + return (EINVAL); + + data = rman_get_virtual(res); + +#ifdef INTR_SOLO + if (isrc->isrc_filter != NULL) { + if (isrc != cookie) + return (EINVAL); + + mtx_lock(&isrc_table_lock); + isrc->isrc_filter = NULL; + isrc->isrc_arg = NULL; + isrc->isrc_handlers = 0; + PIC_DISABLE_INTR(isrc->isrc_dev, isrc); + PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); + isrc_update_name(isrc, NULL); + mtx_unlock(&isrc_table_lock); + return (0); + } +#endif + if (isrc != intr_handler_source(cookie)) + return (EINVAL); + + error = intr_event_remove_handler(cookie); + if (error == 0) { + mtx_lock(&pic_list_lock); + isrc->isrc_handlers--; + if (isrc->isrc_handlers == 0) + PIC_DISABLE_INTR(isrc->isrc_dev, isrc); + PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); + mtx_unlock(&pic_list_lock); + + intr_describe(isrc, NULL, NULL); + } + return (error); +} + +int +intr_describe_irq(device_t dev, struct resource *res, void *cookie, + const char *descr) +{ + struct intr_irqsrc *isrc; + u_int res_id; + + KASSERT(rman_get_start(res) == rman_get_end(res), + ("%s: more interrupts in resource", __func__)); + + res_id = (u_int)rman_get_start(res); + isrc = intr_map_get_isrc(res_id); + if (isrc == NULL || isrc->isrc_handlers == 0) + return (EINVAL); + return (intr_describe(isrc, cookie, descr)); +} + +#ifdef SMP +int +intr_bind_irq(device_t dev, struct resource *res, int cpu) +{ + struct intr_irqsrc *isrc; + u_int res_id; + + KASSERT(rman_get_start(res) == rman_get_end(res), + ("%s: more interrupts in resource", __func__)); + + res_id = (u_int)rman_get_start(res); + isrc = intr_map_get_isrc(res_id); + if (isrc == NULL || isrc->isrc_handlers == 0) + return (EINVAL); +#ifdef INTR_SOLO + if (isrc->isrc_filter != NULL) + return (intr_isrc_assign_cpu(isrc, cpu)); +#endif + return (intr_event_bind(isrc->isrc_event, cpu)); +} +#endif /* SMP */ + +/* + * Allocate memory for new intr_map_data structure. + * Initialize common fields. + */ +struct intr_map_data * +intr_alloc_map_data(enum intr_map_data_type type, size_t len, int flags) +{ + struct intr_map_data *data; + + data = malloc(len, M_INTRPIC, flags); + data->type = type; + data->len = len; + return (data); +} + +void intr_free_intr_map_data(struct intr_map_data *data) +{ + + free(data, M_INTRPIC); +} + +/* + * Register a MSI/MSI-X interrupt controller + */ +int +intr_msi_register(device_t dev, intptr_t xref) +{ + struct intr_pic *pic; + + if (dev == NULL) + return (EINVAL); + pic = pic_create(dev, xref, FLAG_MSI); + if (pic == NULL) + return (ENOMEM); + + debugf("PIC %p registered for %s \n", pic, + device_get_nameunit(dev), dev, (uintmax_t)xref); + return (0); +} + +int +intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, + int maxcount, int *irqs) +{ + struct iommu_domain *domain; + struct intr_irqsrc **isrc; + struct intr_pic *pic; + device_t pdev; + struct intr_map_data_msi *msi; + int err, i; + + pic = pic_lookup(NULL, xref, FLAG_MSI); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + /* + * If this is the first time we have used this context ask the + * interrupt controller to map memory the msi source will need. + */ + err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); + if (err != 0) + return (err); + + isrc = malloc(sizeof(*isrc) * count, M_INTRPIC, M_WAITOK); + err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); + if (err != 0) { + free(isrc, M_INTRPIC); + return (err); + } + + for (i = 0; i < count; i++) { + isrc[i]->isrc_iommu = domain; + msi = (struct intr_map_data_msi *)intr_alloc_map_data( + INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); + msi-> isrc = isrc[i]; + + irqs[i] = intr_map_irq(pic->pic_dev, xref, + (struct intr_map_data *)msi); + } + free(isrc, M_INTRPIC); + + return (err); +} + +int +intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, + int *irqs) +{ + struct intr_irqsrc **isrc; + struct intr_pic *pic; + struct intr_map_data_msi *msi; + int i, err; + + pic = pic_lookup(NULL, xref, FLAG_MSI); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + isrc = malloc(sizeof(*isrc) * count, M_INTRPIC, M_WAITOK); + + for (i = 0; i < count; i++) { + msi = (struct intr_map_data_msi *) + intr_map_get_map_data(irqs[i]); + KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, + ("%s: irq %d map data is not MSI", __func__, + irqs[i])); + isrc[i] = msi->isrc; + } + + MSI_IOMMU_DEINIT(pic->pic_dev, child); + + err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); + + for (i = 0; i < count; i++) { + if (isrc[i] != NULL) + intr_unmap_irq(irqs[i]); + } + + free(isrc, M_INTRPIC); + return (err); +} + +int +intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) +{ + struct iommu_domain *domain; + struct intr_irqsrc *isrc; + struct intr_pic *pic; + device_t pdev; + struct intr_map_data_msi *msi; + int err; + + pic = pic_lookup(NULL, xref, FLAG_MSI); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + /* + * If this is the first time we have used this context ask the + * interrupt controller to map memory the msi source will need. + */ + err = MSI_IOMMU_INIT(pic->pic_dev, child, &domain); + if (err != 0) + return (err); + + err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); + if (err != 0) + return (err); + + isrc->isrc_iommu = domain; + msi = (struct intr_map_data_msi *)intr_alloc_map_data( + INTR_MAP_DATA_MSI, sizeof(*msi), M_WAITOK | M_ZERO); + msi->isrc = isrc; + *irq = intr_map_irq(pic->pic_dev, xref, (struct intr_map_data *)msi); + return (0); +} + +int +intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) +{ + struct intr_irqsrc *isrc; + struct intr_pic *pic; + struct intr_map_data_msi *msi; + int err; + + pic = pic_lookup(NULL, xref, FLAG_MSI); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + msi = (struct intr_map_data_msi *) + intr_map_get_map_data(irq); + KASSERT(msi->hdr.type == INTR_MAP_DATA_MSI, + ("%s: irq %d map data is not MSI", __func__, + irq)); + isrc = msi->isrc; + if (isrc == NULL) { + intr_unmap_irq(irq); + return (EINVAL); + } + + MSI_IOMMU_DEINIT(pic->pic_dev, child); + + err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); + intr_unmap_irq(irq); + + return (err); +} + +int +intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, + uint64_t *addr, uint32_t *data) +{ + struct intr_irqsrc *isrc; + struct intr_pic *pic; + int err; + + pic = pic_lookup(NULL, xref, FLAG_MSI); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_TYPE_MASK) == FLAG_MSI, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + isrc = intr_map_get_isrc(irq); + if (isrc == NULL) + return (EINVAL); + + err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); + +#ifdef IOMMU + if (isrc->isrc_iommu != NULL) + iommu_translate_msi(isrc->isrc_iommu, addr); +#endif + + return (err); +} + +#ifdef SMP +/* + * Init interrupt controller on another CPU. + */ +void +intr_pic_init_secondary(void) +{ + device_t dev; + uint32_t rootnum; + + /* + * QQQ: Only root PICs are aware of other CPUs ??? + */ + //mtx_lock(&pic_list_lock); + for (rootnum = 0; rootnum < INTR_ROOT_COUNT; rootnum++) { + dev = intr_irq_roots[rootnum].dev; + if (dev != NULL) { + PIC_INIT_SECONDARY(dev, rootnum); + } + } + //mtx_unlock(&pic_list_lock); +} +#endif + +/* + * Interrupt mapping table functions. + * + * Please, keep this part separately, it can be transformed to + * extension of standard resources. + */ +struct intr_map_entry +{ + device_t dev; + intptr_t xref; + struct intr_map_data *map_data; + struct intr_irqsrc *isrc; + /* XXX TODO DISCONECTED PICs */ + /*int flags */ +}; + +/* XXX Convert irq_map[] to dynamicaly expandable one. */ +static struct intr_map_entry **irq_map; +static u_int irq_map_count; +static u_int irq_map_first_free_idx; +static struct mtx irq_map_lock; + +static struct intr_irqsrc * +intr_map_get_isrc(u_int res_id) +{ + struct intr_irqsrc *isrc; + + isrc = NULL; + mtx_lock(&irq_map_lock); + if (res_id < irq_map_count && irq_map[res_id] != NULL) + isrc = irq_map[res_id]->isrc; + mtx_unlock(&irq_map_lock); + + return (isrc); +} + +static void +intr_map_set_isrc(u_int res_id, struct intr_irqsrc *isrc) +{ + + mtx_lock(&irq_map_lock); + if (res_id < irq_map_count && irq_map[res_id] != NULL) + irq_map[res_id]->isrc = isrc; + mtx_unlock(&irq_map_lock); +} + +/* + * Get a copy of intr_map_entry data + */ +static struct intr_map_data * +intr_map_get_map_data(u_int res_id) +{ + struct intr_map_data *data; + + data = NULL; + mtx_lock(&irq_map_lock); + if (res_id >= irq_map_count || irq_map[res_id] == NULL) + panic("Attempt to copy invalid resource id: %u\n", res_id); + data = irq_map[res_id]->map_data; + mtx_unlock(&irq_map_lock); + + return (data); +} + +/* + * Get a copy of intr_map_entry data + */ +static void +intr_map_copy_map_data(u_int res_id, device_t *map_dev, intptr_t *map_xref, + struct intr_map_data **data) +{ + size_t len; + + len = 0; + mtx_lock(&irq_map_lock); + if (res_id >= irq_map_count || irq_map[res_id] == NULL) + panic("Attempt to copy invalid resource id: %u\n", res_id); + if (irq_map[res_id]->map_data != NULL) + len = irq_map[res_id]->map_data->len; + mtx_unlock(&irq_map_lock); + + if (len == 0) + *data = NULL; + else + *data = malloc(len, M_INTRPIC, M_WAITOK | M_ZERO); + mtx_lock(&irq_map_lock); + if (irq_map[res_id] == NULL) + panic("Attempt to copy invalid resource id: %u\n", res_id); + if (len != 0) { + if (len != irq_map[res_id]->map_data->len) + panic("Resource id: %u has changed.\n", res_id); + memcpy(*data, irq_map[res_id]->map_data, len); + } + *map_dev = irq_map[res_id]->dev; + *map_xref = irq_map[res_id]->xref; + mtx_unlock(&irq_map_lock); +} + +/* + * Allocate and fill new entry in irq_map table. + */ +u_int +intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data) +{ + u_int i; + struct intr_map_entry *entry; + + /* Prepare new entry first. */ + entry = malloc(sizeof(*entry), M_INTRPIC, M_WAITOK | M_ZERO); + + entry->dev = dev; + entry->xref = xref; + entry->map_data = data; + entry->isrc = NULL; + + mtx_lock(&irq_map_lock); + for (i = irq_map_first_free_idx; i < irq_map_count; i++) { + if (irq_map[i] == NULL) { + irq_map[i] = entry; + irq_map_first_free_idx = i + 1; + mtx_unlock(&irq_map_lock); + return (i); + } + } + for (i = 0; i < irq_map_first_free_idx; i++) { + if (irq_map[i] == NULL) { + irq_map[i] = entry; + irq_map_first_free_idx = i + 1; + mtx_unlock(&irq_map_lock); + return (i); + } + } + mtx_unlock(&irq_map_lock); + + /* XXX Expand irq_map table */ + panic("IRQ mapping table is full."); +} + +/* + * Remove and free mapping entry. + */ +void +intr_unmap_irq(u_int res_id) +{ + struct intr_map_entry *entry; + + mtx_lock(&irq_map_lock); + if ((res_id >= irq_map_count) || (irq_map[res_id] == NULL)) + panic("Attempt to unmap invalid resource id: %u\n", res_id); + entry = irq_map[res_id]; + irq_map[res_id] = NULL; + irq_map_first_free_idx = res_id; + mtx_unlock(&irq_map_lock); + intr_free_intr_map_data(entry->map_data); + free(entry, M_INTRPIC); +} + +/* + * Clone mapping entry. + */ +u_int +intr_map_clone_irq(u_int old_res_id) +{ + device_t map_dev; + intptr_t map_xref; + struct intr_map_data *data; + + intr_map_copy_map_data(old_res_id, &map_dev, &map_xref, &data); + return (intr_map_irq(map_dev, map_xref, data)); +} + +static void +intr_map_init(void *dummy __unused) +{ + + mtx_init(&irq_map_lock, "intr map table", NULL, MTX_DEF); + + irq_map_count = 2 * intr_nirq; + irq_map = mallocarray(irq_map_count, sizeof(struct intr_map_entry*), + M_INTRPIC, M_WAITOK | M_ZERO); +} +SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL);