PERFORCE change 31143 for review
Peter Wemm
peter at FreeBSD.org
Tue May 13 23:59:36 PDT 2003
http://perforce.freebsd.org/chv.cgi?CH=31143
Change 31143 by peter at peter_hammer on 2003/05/13 23:59:13
reimplement the lazy critical section masking stuff.
Affected files ...
.. //depot/projects/hammer/sys/amd64/amd64/critical.c#7 edit
.. //depot/projects/hammer/sys/amd64/amd64/exception.S#14 edit
.. //depot/projects/hammer/sys/amd64/amd64/genassym.c#14 edit
.. //depot/projects/hammer/sys/amd64/include/critical.h#6 edit
.. //depot/projects/hammer/sys/amd64/include/pcpu.h#6 edit
.. //depot/projects/hammer/sys/amd64/include/proc.h#6 edit
.. //depot/projects/hammer/sys/amd64/isa/icu_vector.S#4 edit
.. //depot/projects/hammer/sys/amd64/isa/intr_machdep.c#6 edit
Differences ...
==== //depot/projects/hammer/sys/amd64/amd64/critical.c#7 (text+ko) ====
@@ -19,24 +19,148 @@
#include <machine/critical.h>
/*
- * cpu_critical_fork_exit() - cleanup after fork
+ * XXX this mess to get sched_ithd() and call_fast_unpend()
+ */
+#include <sys/bus.h>
+#include <machine/frame.h>
+#include <machine/segments.h>
+#include <amd64/isa/icu.h>
+#include <amd64/isa/intr_machdep.h>
+
+void amd64_unpend(void); /* NOTE: not static, called from assembly */
+
+/*
+ * cpu_unpend() - called from critical_exit() inline after quick
+ * interrupt-pending check.
*/
void
-cpu_critical_fork_exit(void)
+cpu_unpend(void)
{
+ register_t rflags;
struct thread *td;
td = curthread;
- td->td_critnest = 1;
- td->td_md.md_savecrit = read_rflags() | PSL_I;
+ rflags = intr_disable();
+ if (PCPU_GET(int_pending)) {
+ ++td->td_intr_nesting_level;
+ amd64_unpend();
+ --td->td_intr_nesting_level;
+ }
+ intr_restore(rflags);
+}
+
+/*
+ * cpu_critical_fork_exit() - cleanup after fork
+ *
+ * For amd64 we do not have to do anything, td_critnest is
+ * handled by the fork trampoline code.
+ */
+void
+cpu_critical_fork_exit(void)
+{
}
/*
* cpu_thread_link() - thread linkup, initialize machine-dependant fields
+ *
+ * There are currently no machine-dependant fields that require
+ * initialization.
*/
void
cpu_thread_link(struct thread *td)
{
+}
+
+/*
+ * Called from cpu_unpend or called from the assembly vector code
+ * to process any interrupts which may have occured while we were in
+ * a critical section.
+ *
+ * - interrupts must be disabled
+ * - td_critnest must be 0
+ * - td_intr_nesting_level must be incremented by the caller
+ *
+ * NOT STATIC (called from assembly)
+ */
+static __inline u_int
+bsfq(u_long mask)
+{
+ u_long result;
+
+ __asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
+ return (result);
+}
+
+void
+amd64_unpend(void)
+{
+ struct clockframe frame;
+
+ frame.cf_cs = SEL_KPL;
+ frame.cf_rip = (register_t)amd64_unpend;
+ frame.cf_rflags = PSL_KERNEL;
+ KASSERT(curthread->td_critnest == 0, ("unpend critnest != 0"));
+ KASSERT((read_rflags() & PSL_I) == 0, ("unpend interrupts enabled1"));
+ curthread->td_critnest = 1;
+ for (;;) {
+ u_int64_t mask;
+ int irq;
+
+ /*
+ * Fast interrupts have priority
+ */
+ if ((mask = PCPU_GET(fpending)) != 0) {
+ irq = bsfq(mask);
+ PCPU_SET(fpending, mask & ~(1ul << irq));
+ call_fast_unpend(irq);
+ KASSERT((read_rflags() & PSL_I) == 0,
+ ("unpend interrupts enabled2 %d", irq));
+ continue;
+ }
- td->td_md.md_savecrit = 0;
+ /*
+ * Threaded interrupts come next
+ */
+ if ((mask = PCPU_GET(ipending)) != 0) {
+ irq = bsfq(mask);
+ PCPU_SET(ipending, mask & ~(1ul << irq));
+ sched_ithd((void *)(uintptr_t)irq);
+ KASSERT((read_rflags() & PSL_I) == 0,
+ ("unpend interrupts enabled3 %d", irq));
+ continue;
+ }
+
+ /*
+ * Software interrupts and delayed IPIs are last
+ *
+ * XXX give the bits #defined names. see also
+ * isa/xxx_vector.s
+ */
+ if ((mask = PCPU_GET(spending)) != 0) {
+ irq = bsfq(mask);
+ PCPU_SET(spending, mask & ~(1ul << irq));
+ switch(irq) {
+ case 0: /* bit 0 - hardclock */
+ hardclock_process(&frame);
+ break;
+ case 1: /* bit 1 - statclock */
+ if (profprocs != 0)
+ profclock(&frame);
+ if (pscnt == psdiv)
+ statclock(&frame);
+ break;
+ }
+ KASSERT((read_rflags() & PSL_I) == 0,
+ ("unpend interrupts enabled4 %d", irq));
+ continue;
+ }
+ break;
+ }
+ /*
+ * Interrupts are still disabled, we can safely clear int_pending
+ * and td_critnest.
+ */
+ KASSERT((read_rflags() & PSL_I) == 0, ("unpend interrupts enabled5"));
+ PCPU_SET(int_pending, 0);
+ curthread->td_critnest = 0;
}
==== //depot/projects/hammer/sys/amd64/amd64/exception.S#14 (text+ko) ====
@@ -297,6 +297,9 @@
movq %r12, %rdi /* function */
movq %rbx, %rsi /* arg1 */
movq %rsp, %rdx /* trapframe pointer */
+ movq PCPU(CURTHREAD),%rbx /* setup critnest */
+ movl $1,TD_CRITNEST(%rbx)
+ sti
call fork_exit
MEXITCOUNT
jmp doreti /* Handle any ASTs */
==== //depot/projects/hammer/sys/amd64/amd64/genassym.c#14 (text+ko) ====
@@ -181,6 +181,10 @@
ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
ASSYM(PC_SCRATCH_RSP, offsetof(struct pcpu, pc_scratch_rsp));
+ASSYM(PC_INT_PENDING, offsetof(struct pcpu, pc_int_pending));
+ASSYM(PC_IPENDING, offsetof(struct pcpu, pc_ipending));
+ASSYM(PC_FPENDING, offsetof(struct pcpu, pc_fpending));
+ASSYM(PC_SPENDING, offsetof(struct pcpu, pc_spending));
ASSYM(KCSEL, GSEL(GCODE_SEL, SEL_KPL));
ASSYM(KDSEL, GSEL(GDATA_SEL, SEL_KPL));
==== //depot/projects/hammer/sys/amd64/include/critical.h#6 (text+ko) ====
@@ -23,6 +23,7 @@
/*
* Prototypes - see <arch>/<arch>/critical.c
*/
+void cpu_unpend(void);
void cpu_critical_fork_exit(void);
void cpu_thread_link(struct thread *td);
@@ -33,15 +34,12 @@
*
* This routine is called from critical_enter() on the 0->1 transition
* of td_critnest, prior to it being incremented to 1.
+ *
+ * If new-style critical section handling we do not have to do anything.
+ * However, as a side effect any interrupts occuring while td_critnest
+ * is non-zero will be deferred.
*/
-static __inline void
-cpu_critical_enter(void)
-{
- struct thread *td;
-
- td = curthread;
- td->td_md.md_savecrit = intr_disable();
-}
+#define cpu_critical_enter()
/*
* cpu_critical_exit:
@@ -49,14 +47,27 @@
* This routine is called from critical_exit() on a 1->0 transition
* of td_critnest, after it has been decremented to 0. We are
* exiting the last critical section.
+ *
+ * Note that the td->critnest (1->0) transition interrupt race against
+ * our int_pending/unpend() check below is handled by the interrupt
+ * code for us, so we do not have to do anything fancy.
*/
static __inline void
cpu_critical_exit(void)
{
- struct thread *td;
-
- td = curthread;
- intr_restore(td->td_md.md_savecrit);
+ /*
+ * We may have to schedule pending interrupts. Create
+ * conditions similar to an interrupt context and call
+ * unpend().
+ *
+ * note: we do this even if we are in an interrupt
+ * nesting level. Deep nesting is protected by
+ * critical_*() and if we conditionalized it then we
+ * would have to check int_pending again whenever
+ * we decrement td_intr_nesting_level to 0.
+ */
+ if (PCPU_GET(int_pending))
+ cpu_unpend();
}
#else /* !__GNUC__ */
==== //depot/projects/hammer/sys/amd64/include/pcpu.h#6 (text+ko) ====
@@ -40,7 +40,12 @@
*/
#define PCPU_MD_FIELDS \
struct pcpu *pc_prvspace; /* Self-reference */ \
- register_t pc_scratch_rsp; /* User %rsp in syscall */
+ register_t pc_scratch_rsp; /* User %rsp in syscall */ \
+ u_int64_t pc_int_pending; /* master int pending flag */ \
+ u_int64_t pc_ipending; /* pending slow interrupts */ \
+ u_int64_t pc_fpending; /* pending fast interrupts */ \
+ u_int64_t pc_spending /* pending soft interrupts */
+
#if defined(lint)
==== //depot/projects/hammer/sys/amd64/include/proc.h#6 (text+ko) ====
@@ -41,7 +41,7 @@
* Machine-dependent part of the proc structure for AMD64.
*/
struct mdthread {
- register_t md_savecrit;
+ int __dummy__;
};
struct mdproc {
==== //depot/projects/hammer/sys/amd64/isa/icu_vector.S#4 (text+ko) ====
@@ -4,6 +4,7 @@
*/
#define IRQ_BIT(irq_num) (1 << ((irq_num) % 8))
+#define IRQ_LBIT(irq_num) (1 << (irq_num))
#define IRQ_BYTE(irq_num) ((irq_num) >> 3)
#define ENABLE_ICU1 \
@@ -15,12 +16,11 @@
outb %al,$IO_ICU2 ; /* but do second icu first ... */ \
outb %al,$IO_ICU1 /* ... then first icu */
-
/*
* Macros for interrupt interrupt entry, call to handler, and exit.
*/
-#define FAST_INTR(irq_num, vec_name, enable_icus) \
+#define FAST_INTR(irq_num, vec_name, icu, enable_icus) \
.text ; \
SUPERALIGN_TEXT ; \
IDTVEC(vec_name) ; \
@@ -44,8 +44,19 @@
movq %r14,TF_R14(%rsp) ; \
movq %r15,TF_R15(%rsp) ; \
FAKE_MCOUNT((12)*4(%rsp)) ; \
- call critical_enter ; \
movq PCPU(CURTHREAD),%rbx ; \
+ cmpl $0,TD_CRITNEST(%rbx) ; \
+ je 1f ; \
+ movq $1,PCPU(INT_PENDING) ; \
+ orq $IRQ_LBIT(irq_num),PCPU(FPENDING) ; \
+ movb imen + IRQ_BYTE(irq_num),%al ; \
+ orb $IRQ_BIT(irq_num),%al ; \
+ movb %al,imen + IRQ_BYTE(irq_num) ; \
+ outb %al,$icu+ICU_IMR_OFFSET ; \
+ enable_icus ; \
+ jmp 10f ; \
+1: ; \
+ incl TD_CRITNEST(%rbx) ; \
incl TD_INTR_NESTING_LEVEL(%rbx) ; \
movq intr_unit + (irq_num) * 8, %rdi ; \
call *intr_handler + (irq_num) * 8 ; /* do the work ASAP */ \
@@ -53,11 +64,47 @@
incl cnt+V_INTR ; /* book-keeping can wait */ \
movq intr_countp + (irq_num) * 8,%rax ; \
incq (%rax) ; \
+ decl TD_CRITNEST(%rbx) ; \
+ cmpq $0,PCPU(INT_PENDING) ; \
+ je 2f ; \
+ call amd64_unpend ; \
+2: ; \
decl TD_INTR_NESTING_LEVEL(%rbx) ; \
- call critical_exit ; \
+10: ; \
MEXITCOUNT ; \
jmp doreti
+/*
+ * Restart a fast interrupt that was held up by a critical section.
+ * This routine is called from unpend(). unpend() ensures we are
+ * in a critical section and deals with the interrupt nesting level
+ * for us. If we previously masked the irq, we have to unmask it.
+ *
+ * We have a choice. We can regenerate the irq using the 'int'
+ * instruction or we can create a dummy frame and call the interrupt
+ * handler directly. I've chosen to use the dummy-frame method.
+ */
+#define FAST_UNPEND(irq_num, vec_name, icu) \
+ .text ; \
+ SUPERALIGN_TEXT ; \
+IDTVEC(vec_name) ; \
+ pushfq ; /* rflags */ \
+ mov %cs,%ax ; \
+ pushq %rax ; /* cs */ \
+ pushq 24(%rsp) ; /* original caller rip */ \
+ subq $TF_RIP,%rsp ; /* skip rest including tf_err etc */ \
+ movq intr_unit + (irq_num) * 8, %rdi ; \
+ call *intr_handler + (irq_num) * 8 ; /* do the work ASAP */ \
+ incl cnt+V_INTR ; /* book-keeping can wait */ \
+ movq intr_countp + (irq_num) * 8,%rax ; \
+ incq (%rax) ; \
+ movb imen + IRQ_BYTE(irq_num),%al ; \
+ andb $~IRQ_BIT(irq_num),%al ; \
+ movb %al,imen + IRQ_BYTE(irq_num) ; \
+ outb %al,$icu+ICU_IMR_OFFSET ; \
+ addq $TF_RSP,%rsp ; /* dump frame */ \
+ ret
+
/*
* Slow, threaded interrupts.
*
@@ -75,7 +122,8 @@
testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \
jz 1f ; /* Yes, dont swapgs again */ \
swapgs ; \
-1: movq %rdi,TF_RDI(%rsp) ; \
+1: ; \
+ movq %rdi,TF_RDI(%rsp) ; \
movq %rsi,TF_RSI(%rsp) ; \
movq %rdx,TF_RDX(%rsp) ; \
movq %rcx,TF_RCX(%rsp) ; \
@@ -97,33 +145,44 @@
outb %al,$icu+ICU_IMR_OFFSET ; \
enable_icus ; \
movq PCPU(CURTHREAD),%rbx ; \
+ cmpl $0,TD_CRITNEST(%rbx) ; \
+ je 1f ; \
+ movq $1,PCPU(INT_PENDING) ; \
+ orq $IRQ_LBIT(irq_num),PCPU(IPENDING) ; \
+ jmp 10f ; \
+1: ; \
incl TD_INTR_NESTING_LEVEL(%rbx) ; \
FAKE_MCOUNT(13*4(%rsp)) ; /* XXX late to avoid double count */ \
+ cmpq $0,PCPU(INT_PENDING) ; \
+ je 9f ; \
+ call amd64_unpend ; \
+9: ; \
movq $irq_num, %rdi; /* pass the IRQ */ \
call sched_ithd ; \
decl TD_INTR_NESTING_LEVEL(%rbx) ; \
+10: ; \
MEXITCOUNT ; \
/* We could usually avoid the following jmp by inlining some of */ \
/* doreti, but it's probably better to use less cache. */ \
jmp doreti
MCOUNT_LABEL(bintr)
- FAST_INTR(0,fastintr0, ENABLE_ICU1)
- FAST_INTR(1,fastintr1, ENABLE_ICU1)
- FAST_INTR(2,fastintr2, ENABLE_ICU1)
- FAST_INTR(3,fastintr3, ENABLE_ICU1)
- FAST_INTR(4,fastintr4, ENABLE_ICU1)
- FAST_INTR(5,fastintr5, ENABLE_ICU1)
- FAST_INTR(6,fastintr6, ENABLE_ICU1)
- FAST_INTR(7,fastintr7, ENABLE_ICU1)
- FAST_INTR(8,fastintr8, ENABLE_ICU1_AND_2)
- FAST_INTR(9,fastintr9, ENABLE_ICU1_AND_2)
- FAST_INTR(10,fastintr10, ENABLE_ICU1_AND_2)
- FAST_INTR(11,fastintr11, ENABLE_ICU1_AND_2)
- FAST_INTR(12,fastintr12, ENABLE_ICU1_AND_2)
- FAST_INTR(13,fastintr13, ENABLE_ICU1_AND_2)
- FAST_INTR(14,fastintr14, ENABLE_ICU1_AND_2)
- FAST_INTR(15,fastintr15, ENABLE_ICU1_AND_2)
+ FAST_INTR(0,fastintr0, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(1,fastintr1, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(2,fastintr2, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(3,fastintr3, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(4,fastintr4, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(5,fastintr5, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(6,fastintr6, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(7,fastintr7, IO_ICU1, ENABLE_ICU1)
+ FAST_INTR(8,fastintr8, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(9,fastintr9, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(10,fastintr10, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(11,fastintr11, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(12,fastintr12, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(13,fastintr13, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(14,fastintr14, IO_ICU2, ENABLE_ICU1_AND_2)
+ FAST_INTR(15,fastintr15, IO_ICU2, ENABLE_ICU1_AND_2)
#define CLKINTR_PENDING movl $1,CNAME(clkintr_pending)
/* Threaded interrupts */
@@ -144,5 +203,21 @@
INTR(14,intr14, IO_ICU2, ENABLE_ICU1_AND_2,)
INTR(15,intr15, IO_ICU2, ENABLE_ICU1_AND_2,)
+ FAST_UNPEND(0,fastunpend0, IO_ICU1)
+ FAST_UNPEND(1,fastunpend1, IO_ICU1)
+ FAST_UNPEND(2,fastunpend2, IO_ICU1)
+ FAST_UNPEND(3,fastunpend3, IO_ICU1)
+ FAST_UNPEND(4,fastunpend4, IO_ICU1)
+ FAST_UNPEND(5,fastunpend5, IO_ICU1)
+ FAST_UNPEND(6,fastunpend6, IO_ICU1)
+ FAST_UNPEND(7,fastunpend7, IO_ICU1)
+ FAST_UNPEND(8,fastunpend8, IO_ICU2)
+ FAST_UNPEND(9,fastunpend9, IO_ICU2)
+ FAST_UNPEND(10,fastunpend10, IO_ICU2)
+ FAST_UNPEND(11,fastunpend11, IO_ICU2)
+ FAST_UNPEND(12,fastunpend12, IO_ICU2)
+ FAST_UNPEND(13,fastunpend13, IO_ICU2)
+ FAST_UNPEND(14,fastunpend14, IO_ICU2)
+ FAST_UNPEND(15,fastunpend15, IO_ICU2)
+
MCOUNT_LABEL(eintr)
-
==== //depot/projects/hammer/sys/amd64/isa/intr_machdep.c#6 (text+ko) ====
@@ -87,6 +87,17 @@
IDTVEC(fastintr14), IDTVEC(fastintr15),
};
+static unpendhand_t *fastunpend[ICU_LEN] = {
+ IDTVEC(fastunpend0), IDTVEC(fastunpend1),
+ IDTVEC(fastunpend2), IDTVEC(fastunpend3),
+ IDTVEC(fastunpend4), IDTVEC(fastunpend5),
+ IDTVEC(fastunpend6), IDTVEC(fastunpend7),
+ IDTVEC(fastunpend8), IDTVEC(fastunpend9),
+ IDTVEC(fastunpend10), IDTVEC(fastunpend11),
+ IDTVEC(fastunpend12), IDTVEC(fastunpend13),
+ IDTVEC(fastunpend14), IDTVEC(fastunpend15),
+};
+
static inthand_t *slowintr[ICU_LEN] = {
IDTVEC(intr0), IDTVEC(intr1), IDTVEC(intr2), IDTVEC(intr3),
IDTVEC(intr4), IDTVEC(intr5), IDTVEC(intr6), IDTVEC(intr7),
@@ -521,3 +532,10 @@
return (ithread_remove_handler(cookie));
}
+
+void
+call_fast_unpend(int irq)
+{
+
+ fastunpend[irq]();
+}
More information about the p4-projects
mailing list