git: fae8755f16ff - main - intrng: Extract arm/arm64 IPI->PIC glue code
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Wed, 24 Jan 2024 23:50:20 UTC
The branch main has been updated by jrtc27:
URL: https://cgit.FreeBSD.org/src/commit/?id=fae8755f16ff5b9bdc32df046e0f16c0cbb48a29
commit fae8755f16ff5b9bdc32df046e0f16c0cbb48a29
Author: Jessica Clarke <jrtc27@FreeBSD.org>
AuthorDate: 2024-01-24 23:49:53 +0000
Commit: Jessica Clarke <jrtc27@FreeBSD.org>
CommitDate: 2024-01-24 23:49:53 +0000
intrng: Extract arm/arm64 IPI->PIC glue code
The arm and arm64 implementations of dispatching IPIs via PIC_IPI_SEND
are almost identical, and entirely MI with the lone exception of a
single store barrier on arm64 (that is likely either redundant or needed
on arm too). Thus, de-duplicate this code by moving it to INTRNG as a
generic IPI glue framework. The ipi_* functions remain declared in MD
smp.h headers and implemented in MD code, but are trivial wrappers
around intr_ipi_send that could be made MI, at least for INTRNG ports,
at a later date.
Note that, whilst both arm and arm64 had an ii_send member in intr_ipi
to abstract over how to send interrupts,, they were always ultimately
using PIC_IPI_SEND, and so this complexity has been removed. A follow-up
commit will re-introduce the same flexibility by instead allowing a
device other than the root PIC to be registered as the IPI sender.
As part of this, strengthen a MAXCPU assertion that was missed in commit
2f0b059eeafc ("intrng: switch from MAXCPU to mp_ncpus") (which itself is
mis-titled).
Reviewed by: mmel, mhorne
MFC after: 1 month
Differential Revision: https://reviews.freebsd.org/D35898
---
sys/arm/arm/machdep_intr.c | 126 +-------------------------
sys/arm/arm/mp_machdep.c | 10 +--
sys/arm/include/intr.h | 13 ---
sys/arm64/arm64/mp_machdep.c | 204 ++-----------------------------------------
sys/arm64/include/intr.h | 4 -
sys/kern/subr_intr.c | 188 ++++++++++++++++++++++++++++++++-------
sys/sys/intr.h | 21 ++---
7 files changed, 177 insertions(+), 389 deletions(-)
diff --git a/sys/arm/arm/machdep_intr.c b/sys/arm/arm/machdep_intr.c
index fc16741a49fa..712336e7c048 100644
--- a/sys/arm/arm/machdep_intr.c
+++ b/sys/arm/arm/machdep_intr.c
@@ -29,39 +29,10 @@
#include <sys/param.h>
#include <sys/systm.h>
-#include <sys/syslog.h>
-#include <sys/kernel.h>
-#include <sys/malloc.h>
-#include <sys/proc.h>
-#include <sys/bus.h>
-#include <sys/interrupt.h>
-#include <sys/conf.h>
-#include <sys/pmc.h>
-#include <sys/pmckern.h>
-#include <sys/smp.h>
-#include <machine/atomic.h>
-#include <machine/bus.h>
-#include <machine/intr.h>
#include <machine/cpu.h>
-#include <machine/smp.h>
-
-#include "pic_if.h"
-
-#ifdef SMP
-#define INTR_IPI_NAMELEN (MAXCOMLEN + 1)
-
-struct intr_ipi {
- intr_ipi_handler_t * ii_handler;
- void * ii_handler_arg;
- intr_ipi_send_t * ii_send;
- void * ii_send_arg;
- char ii_name[INTR_IPI_NAMELEN];
- u_long * ii_count;
-};
-
-static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
-#endif
+#include <machine/cpufunc.h>
+#include <machine/intr.h>
/*
* arm_irq_memory_barrier()
@@ -125,96 +96,3 @@ arm_irq_memory_barrier(uintptr_t irq)
dsb();
cpu_l2cache_drain_writebuf();
}
-
-#ifdef SMP
-static inline struct intr_ipi *
-intr_ipi_lookup(u_int ipi)
-{
-
- if (ipi >= INTR_IPI_COUNT)
- panic("%s: no such IPI %u", __func__, ipi);
-
- return (&ipi_sources[ipi]);
-}
-
-void
-intr_ipi_dispatch(u_int ipi)
-{
- struct intr_ipi *ii;
-
- ii = intr_ipi_lookup(ipi);
- if (ii->ii_count == NULL)
- panic("%s: not setup IPI %u", __func__, ipi);
-
- intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
-
- ii->ii_handler(ii->ii_handler_arg);
-}
-
-void
-intr_ipi_send(cpuset_t cpus, u_int ipi)
-{
- struct intr_ipi *ii;
-
- ii = intr_ipi_lookup(ipi);
- if (ii->ii_count == NULL)
- panic("%s: not setup IPI %u", __func__, ipi);
-
- ii->ii_send(ii->ii_send_arg, cpus, ipi);
-}
-
-void
-intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
- void *h_arg, intr_ipi_send_t *send, void *s_arg)
-{
- struct intr_ipi *ii;
-
- ii = intr_ipi_lookup(ipi);
-
- KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
- KASSERT(send != NULL, ("%s: ipi %u no sender", __func__, ipi));
- KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
-
- ii->ii_handler = hand;
- ii->ii_handler_arg = h_arg;
- ii->ii_send = send;
- ii->ii_send_arg = s_arg;
- strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
- ii->ii_count = intr_ipi_setup_counters(name);
-}
-
-/*
- * Send IPI thru interrupt controller.
- */
-static void
-pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi)
-{
-
- KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
- PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi);
-}
-
-/*
- * Setup IPI handler on interrupt controller.
- *
- * Not SMP coherent.
- */
-int
-intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
- void *arg)
-{
- int error;
- struct intr_irqsrc *isrc;
-
- KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
-
- error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
- if (error != 0)
- return (error);
-
- isrc->isrc_handlers++;
- intr_ipi_setup(ipi, name, hand, arg, pic_ipi_send, isrc);
- PIC_ENABLE_INTR(intr_irq_root_dev, isrc);
- return (0);
-}
-#endif
diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c
index 0d78ef789d6e..01b02d9520ca 100644
--- a/sys/arm/arm/mp_machdep.c
+++ b/sys/arm/arm/mp_machdep.c
@@ -300,11 +300,11 @@ release_aps(void *dummy __unused)
if (mp_ncpus == 1)
return;
- intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
- intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
- intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
- intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
- intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
+ intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
+ intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
+ intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
+ intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
+ intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
atomic_store_rel_int(&aps_ready, 1);
/* Wake the other threads up */
diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h
index 21829c5782bc..d0d0ff9fc32a 100644
--- a/sys/arm/include/intr.h
+++ b/sys/arm/include/intr.h
@@ -49,19 +49,6 @@
#include <sys/intr.h>
-#ifdef SMP
-typedef void intr_ipi_send_t(void *, cpuset_t, u_int);
-typedef void intr_ipi_handler_t(void *);
-
-void intr_ipi_dispatch(u_int);
-void intr_ipi_send(cpuset_t, u_int);
-
-void intr_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *,
- intr_ipi_send_t *, void *);
-
-int intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *);
-#endif
-
void arm_irq_memory_barrier(uintptr_t);
#endif /* _MACHINE_INTR_H */
diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c
index c2b30930fd91..9c6175445572 100644
--- a/sys/arm64/arm64/mp_machdep.c
+++ b/sys/arm64/arm64/mp_machdep.c
@@ -77,8 +77,6 @@
#include <dev/psci/psci.h>
-#include "pic_if.h"
-
#define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE)
#define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */
@@ -98,25 +96,6 @@ static struct {
};
#endif
-typedef void intr_ipi_send_t(void *, cpuset_t, u_int);
-typedef void intr_ipi_handler_t(void *);
-
-#define INTR_IPI_NAMELEN (MAXCOMLEN + 1)
-struct intr_ipi {
- intr_ipi_handler_t * ii_handler;
- void * ii_handler_arg;
- intr_ipi_send_t * ii_send;
- void * ii_send_arg;
- char ii_name[INTR_IPI_NAMELEN];
- u_long * ii_count;
-};
-
-static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
-
-static struct intr_ipi *intr_ipi_lookup(u_int);
-static void intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *,
- void *);
-
static void ipi_ast(void *);
static void ipi_hardclock(void *);
static void ipi_preempt(void *);
@@ -165,12 +144,12 @@ release_aps(void *dummy __unused)
if (mp_ncpus == 1)
return;
- intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
- intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
- intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
- intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
- intr_pic_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL);
- intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
+ intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL);
+ intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL);
+ intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL);
+ intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL);
+ intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL);
+ intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL);
atomic_store_rel_int(&aps_ready, 1);
/* Wake up the other CPUs */
@@ -315,71 +294,6 @@ smp_after_idle_runnable(void *arg __unused)
SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
smp_after_idle_runnable, NULL);
-/*
- * Send IPI thru interrupt controller.
- */
-static void
-pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi)
-{
-
- KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
-
- /*
- * Ensure that this CPU's stores will be visible to IPI
- * recipients before starting to send the interrupts.
- */
- dsb(ishst);
-
- PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi);
-}
-
-/*
- * Setup IPI handler on interrupt controller.
- *
- * Not SMP coherent.
- */
-static void
-intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
- void *arg)
-{
- struct intr_irqsrc *isrc;
- struct intr_ipi *ii;
- int error;
-
- KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
- KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
-
- error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
- if (error != 0)
- return;
-
- isrc->isrc_handlers++;
-
- ii = intr_ipi_lookup(ipi);
- KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
-
- ii->ii_handler = hand;
- ii->ii_handler_arg = arg;
- ii->ii_send = pic_ipi_send;
- ii->ii_send_arg = isrc;
- strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
- ii->ii_count = intr_ipi_setup_counters(name);
-
- PIC_ENABLE_INTR(intr_irq_root_dev, isrc);
-}
-
-static void
-intr_ipi_send(cpuset_t cpus, u_int ipi)
-{
- struct intr_ipi *ii;
-
- ii = intr_ipi_lookup(ipi);
- if (ii->ii_count == NULL)
- panic("%s: not setup IPI %u", __func__, ipi);
-
- ii->ii_send(ii->ii_send_arg, cpus, ipi);
-}
-
static void
ipi_ast(void *dummy __unused)
{
@@ -888,112 +802,6 @@ cpu_mp_setmaxid(void)
}
}
-/*
- * Lookup IPI source.
- */
-static struct intr_ipi *
-intr_ipi_lookup(u_int ipi)
-{
-
- if (ipi >= INTR_IPI_COUNT)
- panic("%s: no such IPI %u", __func__, ipi);
-
- return (&ipi_sources[ipi]);
-}
-
-/*
- * interrupt controller dispatch function for IPIs. It should
- * be called straight from the interrupt controller, when associated
- * interrupt source is learned. Or from anybody who has an interrupt
- * source mapped.
- */
-void
-intr_ipi_dispatch(u_int ipi)
-{
- struct intr_ipi *ii;
-
- ii = intr_ipi_lookup(ipi);
- if (ii->ii_count == NULL)
- panic("%s: not setup IPI %u", __func__, ipi);
-
- intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
-
- ii->ii_handler(ii->ii_handler_arg);
-}
-
-#ifdef notyet
-/*
- * Map IPI into interrupt controller.
- *
- * Not SMP coherent.
- */
-static int
-ipi_map(struct intr_irqsrc *isrc, u_int ipi)
-{
- boolean_t is_percpu;
- int error;
-
- if (ipi >= INTR_IPI_COUNT)
- panic("%s: no such IPI %u", __func__, ipi);
-
- KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
-
- isrc->isrc_type = INTR_ISRCT_NAMESPACE;
- isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI;
- isrc->isrc_nspc_num = ipi_next_num;
-
- error = PIC_REGISTER(intr_irq_root_dev, isrc, &is_percpu);
- if (error == 0) {
- isrc->isrc_dev = intr_irq_root_dev;
- ipi_next_num++;
- }
- return (error);
-}
-
-/*
- * Setup IPI handler to interrupt source.
- *
- * Note that there could be more ways how to send and receive IPIs
- * on a platform like fast interrupts for example. In that case,
- * one can call this function with ASIF_NOALLOC flag set and then
- * call intr_ipi_dispatch() when appropriate.
- *
- * Not SMP coherent.
- */
-int
-intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter,
- void *arg, u_int flags)
-{
- struct intr_irqsrc *isrc;
- int error;
-
- if (filter == NULL)
- return(EINVAL);
-
- isrc = intr_ipi_lookup(ipi);
- if (isrc->isrc_ipifilter != NULL)
- return (EEXIST);
-
- if ((flags & AISHF_NOALLOC) == 0) {
- error = ipi_map(isrc, ipi);
- if (error != 0)
- return (error);
- }
-
- isrc->isrc_ipifilter = filter;
- isrc->isrc_arg = arg;
- isrc->isrc_handlers = 1;
- isrc->isrc_count = intr_ipi_setup_counters(name);
- isrc->isrc_index = 0; /* it should not be used in IPI case */
-
- if (isrc->isrc_dev != NULL) {
- PIC_ENABLE_INTR(isrc->isrc_dev, isrc);
- PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc);
- }
- return (0);
-}
-#endif
-
/* Sending IPI */
void
ipi_all_but_self(u_int ipi)
diff --git a/sys/arm64/include/intr.h b/sys/arm64/include/intr.h
index 8d9c35e81cd7..3cdbc83ff109 100644
--- a/sys/arm64/include/intr.h
+++ b/sys/arm64/include/intr.h
@@ -42,10 +42,6 @@ arm_irq_memory_barrier(uintptr_t irq)
{
}
-#ifdef SMP
-void intr_ipi_dispatch(u_int);
-#endif
-
#ifdef DEV_ACPI
#define ACPI_INTR_XREF 1
#define ACPI_MSI_XREF 2
diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c
index fb75b2e1773b..c89f12a30ec9 100644
--- a/sys/kern/subr_intr.c
+++ b/sys/kern/subr_intr.c
@@ -2,6 +2,11 @@
* Copyright (c) 2015-2016 Svatopluk Kraus
* Copyright (c) 2015-2016 Michal Meloun
* All rights reserved.
+ * Copyright (c) 2015-2016 The FreeBSD Foundation
+ * Copyright (c) 2021 Jessica Clarke <jrtc27@FreeBSD.org>
+ *
+ * Portions of this software were developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -124,6 +129,18 @@ struct intr_pic {
SLIST_HEAD(, intr_pic_child) pic_children;
};
+#ifdef SMP
+#define INTR_IPI_NAMELEN (MAXCOMLEN + 1)
+
+struct intr_ipi {
+ intr_ipi_handler_t *ii_handler;
+ void *ii_handler_arg;
+ struct intr_irqsrc *ii_isrc;
+ char ii_name[INTR_IPI_NAMELEN];
+ u_long *ii_count;
+};
+#endif
+
static struct mtx pic_list_lock;
static SLIST_HEAD(, intr_pic) pic_list;
@@ -140,6 +157,8 @@ static bool irq_assign_cpu = true;
#else
static bool irq_assign_cpu = false;
#endif
+
+static struct intr_ipi ipi_sources[INTR_IPI_COUNT];
#endif
u_int intr_nirq = NIRQ;
@@ -298,39 +317,6 @@ isrc_release_counters(struct intr_irqsrc *isrc)
bit_nclear(intrcnt_bitmap, idx, idx + 1);
}
-#ifdef SMP
-/*
- * Virtualization for interrupt source IPI counters setup.
- */
-u_long *
-intr_ipi_setup_counters(const char *name)
-{
- u_int index, i;
- char str[INTRNAME_LEN];
-
- mtx_lock(&isrc_table_lock);
-
- /*
- * We should never have a problem finding mp_maxid + 1 contiguous
- * counters, in practice. Interrupts will be allocated sequentially
- * during boot, so the array should fill from low to high index. Once
- * reserved, the IPI counters will never be released. Similarly, we
- * will not need to allocate more IPIs once the system is running.
- */
- bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index);
- if (index == -1)
- panic("Failed to allocate %d counters. Array exhausted?",
- mp_maxid + 1);
- bit_nset(intrcnt_bitmap, index, index + mp_maxid);
- for (i = 0; i < mp_maxid + 1; i++) {
- snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name);
- intrcnt_setname(str, index + i);
- }
- mtx_unlock(&isrc_table_lock);
- return (&intrcnt[index]);
-}
-#endif
-
/*
* Main interrupt dispatch handler. It's called straight
* from the assembler, where CPU interrupt is served.
@@ -1774,3 +1760,139 @@ intr_map_init(void *dummy __unused)
M_INTRNG, M_WAITOK | M_ZERO);
}
SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL);
+
+#ifdef SMP
+/* Virtualization for interrupt source IPI counter increment. */
+static inline void
+intr_ipi_increment_count(u_long *counter, u_int cpu)
+{
+
+ KASSERT(cpu < mp_maxid + 1, ("%s: too big cpu %u", __func__, cpu));
+ counter[cpu]++;
+}
+
+/*
+ * Virtualization for interrupt source IPI counters setup.
+ */
+static u_long *
+intr_ipi_setup_counters(const char *name)
+{
+ u_int index, i;
+ char str[INTRNAME_LEN];
+
+ mtx_lock(&isrc_table_lock);
+
+ /*
+ * We should never have a problem finding mp_maxid + 1 contiguous
+ * counters, in practice. Interrupts will be allocated sequentially
+ * during boot, so the array should fill from low to high index. Once
+ * reserved, the IPI counters will never be released. Similarly, we
+ * will not need to allocate more IPIs once the system is running.
+ */
+ bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index);
+ if (index == -1)
+ panic("Failed to allocate %d counters. Array exhausted?",
+ mp_maxid + 1);
+ bit_nset(intrcnt_bitmap, index, index + mp_maxid);
+ for (i = 0; i < mp_maxid + 1; i++) {
+ snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name);
+ intrcnt_setname(str, index + i);
+ }
+ mtx_unlock(&isrc_table_lock);
+ return (&intrcnt[index]);
+}
+
+/*
+ * Lookup IPI source.
+ */
+static struct intr_ipi *
+intr_ipi_lookup(u_int ipi)
+{
+
+ if (ipi >= INTR_IPI_COUNT)
+ panic("%s: no such IPI %u", __func__, ipi);
+
+ return (&ipi_sources[ipi]);
+}
+
+/*
+ * Setup IPI handler on interrupt controller.
+ *
+ * Not SMP coherent.
+ */
+void
+intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
+ void *arg)
+{
+ struct intr_irqsrc *isrc;
+ struct intr_ipi *ii;
+ int error;
+
+ KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
+ KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi));
+
+ error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc);
+ if (error != 0)
+ return;
+
+ isrc->isrc_handlers++;
+
+ ii = intr_ipi_lookup(ipi);
+ KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi));
+
+ ii->ii_handler = hand;
+ ii->ii_handler_arg = arg;
+ ii->ii_isrc = isrc;
+ strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN);
+ ii->ii_count = intr_ipi_setup_counters(name);
+
+ PIC_ENABLE_INTR(intr_irq_root_dev, isrc);
+}
+
+void
+intr_ipi_send(cpuset_t cpus, u_int ipi)
+{
+ struct intr_ipi *ii;
+
+ KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__));
+
+ ii = intr_ipi_lookup(ipi);
+ if (ii->ii_count == NULL)
+ panic("%s: not setup IPI %u", __func__, ipi);
+
+ /*
+ * XXX: Surely needed on other architectures too? Either way should be
+ * some kind of MI hook defined in an MD header, or the responsibility
+ * of the MD caller if not widespread.
+ */
+#ifdef __aarch64__
+ /*
+ * Ensure that this CPU's stores will be visible to IPI
+ * recipients before starting to send the interrupts.
+ */
+ dsb(ishst);
+#endif
+
+ PIC_IPI_SEND(intr_irq_root_dev, ii->ii_isrc, cpus, ipi);
+}
+
+/*
+ * interrupt controller dispatch function for IPIs. It should
+ * be called straight from the interrupt controller, when associated
+ * interrupt source is learned. Or from anybody who has an interrupt
+ * source mapped.
+ */
+void
+intr_ipi_dispatch(u_int ipi)
+{
+ struct intr_ipi *ii;
+
+ ii = intr_ipi_lookup(ipi);
+ if (ii->ii_count == NULL)
+ panic("%s: not setup IPI %u", __func__, ipi);
+
+ intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid));
+
+ ii->ii_handler(ii->ii_handler_arg);
+}
+#endif
diff --git a/sys/sys/intr.h b/sys/sys/intr.h
index bdc693e6cb60..57b0ca393912 100644
--- a/sys/sys/intr.h
+++ b/sys/sys/intr.h
@@ -148,21 +148,18 @@ int intr_release_msix(device_t, device_t, intptr_t, int);
int intr_bind_irq(device_t, struct resource *, int);
void intr_pic_init_secondary(void);
+#endif
-/* Virtualization for interrupt source IPI counter increment. */
-static inline void
-intr_ipi_increment_count(u_long *counter, u_int cpu)
-{
-
- KASSERT(cpu < MAXCPU, ("%s: too big cpu %u", __func__, cpu));
- counter[cpu]++;
-}
+extern u_int intr_nirq; /* number of IRQs on intrng platforms */
-/* Virtualization for interrupt source IPI counters setup. */
-u_long * intr_ipi_setup_counters(const char *name);
+/* Intr interface for IPIs. */
+#ifdef SMP
+typedef void intr_ipi_handler_t(void *);
+void intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand,
+ void *arg);
+void intr_ipi_send(cpuset_t cpus, u_int ipi);
+void intr_ipi_dispatch(u_int ipi);
#endif
-extern u_int intr_nirq; /* number of IRQs on intrng platforms */
-
#endif /* _SYS_INTR_H */