git: 5bc73c369dba - stable/12 - pf: add hybrid 32- an 64- bit counters
Mateusz Guzik
mjg at FreeBSD.org
Wed Aug 11 13:25:18 UTC 2021
The branch stable/12 has been updated by mjg:
URL: https://cgit.FreeBSD.org/src/commit/?id=5bc73c369dbae8759f5735ca5a1f940127cec2db
commit 5bc73c369dbae8759f5735ca5a1f940127cec2db
Author: Mateusz Guzik <mjg at FreeBSD.org>
AuthorDate: 2021-07-22 20:47:24 +0000
Commit: Mateusz Guzik <mjg at FreeBSD.org>
CommitDate: 2021-08-11 12:20:09 +0000
pf: add hybrid 32- an 64- bit counters
Numerous counters got migrated from straight uint64_t to the counter(9)
API. Unfortunately the implementation comes with a significiant
performance hit on some platforms and cannot be easily fixed.
Work around the problem by implementing a pf-specific variant.
Reviewed by: kp
Sponsored by: Rubicon Communications, LLC ("Netgate")
(cherry picked from commit defdcdd5648dc1ea789bc54bb45108fcab546a6b)
---
sys/net/pfvar.h | 218 ++++++++++++++++++++++++++++++++++++++++++++++++++++
sys/netpfil/pf/pf.c | 28 ++++++-
2 files changed, 245 insertions(+), 1 deletion(-)
diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index eda2e9fb71c0..fd0b36c72a42 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -43,9 +43,11 @@
#include <sys/nv.h>
#include <sys/refcount.h>
#include <sys/sysctl.h>
+#include <sys/smp.h>
#include <sys/lock.h>
#include <sys/rmlock.h>
#include <sys/tree.h>
+#include <sys/seq.h>
#include <vm/uma.h>
#include <net/radix.h>
@@ -64,6 +66,222 @@
#ifdef _KERNEL
+#if defined(__arm__)
+#define PF_WANT_32_TO_64_COUNTER
+#endif
+
+/*
+ * A hybrid of 32-bit and 64-bit counters which can be used on platforms where
+ * counter(9) is very expensive.
+ *
+ * As 32-bit counters are expected to overflow, a periodic job sums them up to
+ * a saved 64-bit state. Fetching the value still walks all CPUs to get the most
+ * current snapshot.
+ */
+#ifdef PF_WANT_32_TO_64_COUNTER
+struct pf_counter_u64_pcpu {
+ u_int32_t current;
+ u_int32_t snapshot;
+};
+
+struct pf_counter_u64 {
+ struct pf_counter_u64_pcpu *pfcu64_pcpu;
+ u_int64_t pfcu64_value;
+ seq_t pfcu64_seqc;
+};
+
+static inline int
+pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags)
+{
+
+ pfcu64->pfcu64_value = 0;
+ pfcu64->pfcu64_seqc = 0;
+ pfcu64->pfcu64_pcpu = uma_zalloc_pcpu(pcpu_zone_64, flags | M_ZERO);
+ if (__predict_false(pfcu64->pfcu64_pcpu == NULL))
+ return (ENOMEM);
+ return (0);
+}
+
+static inline void
+pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64)
+{
+
+ uma_zfree_pcpu(pcpu_zone_64, pfcu64->pfcu64_pcpu);
+}
+
+static inline void
+pf_counter_u64_critical_enter(void)
+{
+
+ critical_enter();
+}
+
+static inline void
+pf_counter_u64_critical_exit(void)
+{
+
+ critical_exit();
+}
+
+static inline void
+pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ u_int32_t val;
+
+ MPASS(curthread->td_critnest > 0);
+ pcpu = zpcpu_get(pfcu64->pfcu64_pcpu);
+ val = atomic_load_int(&pcpu->current);
+ atomic_store_int(&pcpu->current, val + n);
+}
+
+static inline void
+pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+
+ critical_enter();
+ pf_counter_u64_add_protected(pfcu64, n);
+ critical_exit();
+}
+
+static inline u_int64_t
+pf_counter_u64_periodic(struct pf_counter_u64 *pfcu64)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ u_int64_t sum;
+ u_int32_t val;
+ int cpu;
+
+ MPASS(curthread->td_critnest > 0);
+ seq_write_begin(&pfcu64->pfcu64_seqc);
+ sum = pfcu64->pfcu64_value;
+ CPU_FOREACH(cpu) {
+ pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu);
+ val = atomic_load_int(&pcpu->current);
+ sum += (uint32_t)(val - pcpu->snapshot);
+ pcpu->snapshot = val;
+ }
+ pfcu64->pfcu64_value = sum;
+ seq_write_end(&pfcu64->pfcu64_seqc);
+ return (sum);
+}
+
+static inline u_int64_t
+pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ u_int64_t sum;
+ seq_t seqc;
+ int cpu;
+
+ for (;;) {
+ seqc = seq_load(&pfcu64->pfcu64_seqc);
+ sum = 0;
+ CPU_FOREACH(cpu) {
+ pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu);
+ sum += (uint32_t)(atomic_load_int(&pcpu->current) -pcpu->snapshot);
+ }
+ sum += pfcu64->pfcu64_value;
+ if (seq_consistent(&pfcu64->pfcu64_seqc, seqc))
+ break;
+ }
+ return (sum);
+}
+
+static inline void
+pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64)
+{
+ struct pf_counter_u64_pcpu *pcpu;
+ int cpu;
+
+ MPASS(curthread->td_critnest > 0);
+ seq_write_begin(&pfcu64->pfcu64_seqc);
+ CPU_FOREACH(cpu) {
+ pcpu = zpcpu_get_cpu(pfcu64->pfcu64_pcpu, cpu);
+ pcpu->snapshot = atomic_load_int(&pcpu->current);
+ }
+ pfcu64->pfcu64_value = 0;
+ seq_write_end(&pfcu64->pfcu64_seqc);
+}
+
+static inline void
+pf_counter_u64_zero(struct pf_counter_u64 *pfcu64)
+{
+
+ critical_enter();
+ pf_counter_u64_zero_protected(pfcu64);
+ critical_exit();
+}
+#else
+struct pf_counter_u64 {
+ counter_u64_t counter;
+};
+
+static inline int
+pf_counter_u64_init(struct pf_counter_u64 *pfcu64, int flags)
+{
+
+ pfcu64->counter = counter_u64_alloc(flags);
+ if (__predict_false(pfcu64->counter == NULL))
+ return (ENOMEM);
+ return (0);
+}
+
+static inline void
+pf_counter_u64_deinit(struct pf_counter_u64 *pfcu64)
+{
+
+ counter_u64_free(pfcu64->counter);
+}
+
+static inline void
+pf_counter_u64_critical_enter(void)
+{
+
+}
+
+static inline void
+pf_counter_u64_critical_exit(void)
+{
+
+}
+
+static inline void
+pf_counter_u64_add_protected(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+
+ counter_u64_add(pfcu64->counter, n);
+}
+
+static inline void
+pf_counter_u64_add(struct pf_counter_u64 *pfcu64, uint32_t n)
+{
+
+ pf_counter_u64_add_protected(pfcu64, n);
+}
+
+static inline u_int64_t
+pf_counter_u64_fetch(struct pf_counter_u64 *pfcu64)
+{
+
+ return (counter_u64_fetch(pfcu64->counter));
+}
+
+static inline void
+pf_counter_u64_zero_protected(struct pf_counter_u64 *pfcu64)
+{
+
+ counter_u64_zero(pfcu64->counter);
+}
+
+static inline void
+pf_counter_u64_zero(struct pf_counter_u64 *pfcu64)
+{
+
+ pf_counter_u64_zero_protected(pfcu64);
+}
+#endif
+
SYSCTL_DECL(_net_pf);
MALLOC_DECLARE(M_PFHASH);
diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c
index a7b429120c9e..5edafe4b5e7a 100644
--- a/sys/netpfil/pf/pf.c
+++ b/sys/netpfil/pf/pf.c
@@ -155,6 +155,11 @@ VNET_DECLARE(int, pf_vnet_active);
VNET_DEFINE_STATIC(uint32_t, pf_purge_idx);
#define V_pf_purge_idx VNET(pf_purge_idx)
+#ifdef PF_WANT_32_TO_64_COUNTER
+VNET_DEFINE_STATIC(uint32_t, pf_counter_periodic_iter);
+#define V_pf_counter_periodic_iter VNET(pf_counter_periodic_iter)
+#endif
+
/*
* Queue for pf_intr() sends.
*/
@@ -1505,6 +1510,25 @@ pf_intr(void *v)
CURVNET_RESTORE();
}
+#define pf_purge_thread_period (hz / 10)
+
+#ifdef PF_WANT_32_TO_64_COUNTER
+static void
+pf_counter_u64_periodic_main(void)
+{
+ PF_RULES_RLOCK_TRACKER;
+
+ V_pf_counter_periodic_iter++;
+
+ PF_RULES_RLOCK();
+ pf_counter_u64_critical_enter();
+ pf_counter_u64_critical_exit();
+ PF_RULES_RUNLOCK();
+}
+#else
+#define pf_counter_u64_periodic_main() do { } while (0)
+#endif
+
void
pf_purge_thread(void *unused __unused)
{
@@ -1512,7 +1536,7 @@ pf_purge_thread(void *unused __unused)
sx_xlock(&pf_end_lock);
while (pf_end_threads == 0) {
- sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", hz / 10);
+ sx_sleep(pf_purge_thread, &pf_end_lock, 0, "pftm", pf_purge_thread_period);
VNET_LIST_RLOCK();
VNET_FOREACH(vnet_iter) {
@@ -1525,6 +1549,8 @@ pf_purge_thread(void *unused __unused)
continue;
}
+ pf_counter_u64_periodic_main();
+
/*
* Process 1/interval fraction of the state
* table every run.
More information about the dev-commits-src-all
mailing list