git: ab91feabcc6f - main - ovpn: Introduce OpenVPN DCO support
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 28 Jun 2022 11:59:31 UTC
The branch main has been updated by kp:
URL: https://cgit.FreeBSD.org/src/commit/?id=ab91feabcc6f9da21d5c75028153af16d06e679a
commit ab91feabcc6f9da21d5c75028153af16d06e679a
Author: Kristof Provost <kp@FreeBSD.org>
AuthorDate: 2022-02-22 09:21:38 +0000
Commit: Kristof Provost <kp@FreeBSD.org>
CommitDate: 2022-06-28 09:33:10 +0000
ovpn: Introduce OpenVPN DCO support
OpenVPN Data Channel Offload (DCO) moves OpenVPN data plane processing
(i.e. tunneling and cryptography) into the kernel, rather than using tap
devices.
This avoids significant copying and context switching overhead between
kernel and user space and improves OpenVPN throughput.
In my test setup throughput improved from around 660Mbit/s to around
2Gbit/s.
Sponsored by: Rubicon Communications, LLC ("Netgate")
Differential Revision: https://reviews.freebsd.org/D34340
---
share/man/man4/Makefile | 1 +
share/man/man4/ovpn.4 | 54 +
sys/conf/files | 1 +
sys/kern/kern_jail.c | 1 +
sys/modules/Makefile | 1 +
sys/modules/if_ovpn/Makefile | 6 +
sys/net/if_ovpn.c | 2437 ++++++++++++++++++++++++++++++++++++++++++
sys/net/if_ovpn.h | 64 ++
sys/sys/priv.h | 1 +
9 files changed, 2566 insertions(+)
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 613e823d4731..4a7f323e2c5c 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -418,6 +418,7 @@ MAN= aac.4 \
ow.4 \
ow_temp.4 \
owc.4 \
+ ovpn.4 \
${_padlock.4} \
pass.4 \
pca954x.4 \
diff --git a/share/man/man4/ovpn.4 b/share/man/man4/ovpn.4
new file mode 100644
index 000000000000..3481dd0d0bd3
--- /dev/null
+++ b/share/man/man4/ovpn.4
@@ -0,0 +1,54 @@
+.\" Copyright (c) 2022 Rubicon Communications, LLC ("Netgate")
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd April 22, 2022
+.Dt OVPN 4
+.Os
+.Sh NAME
+.Nm ovpn
+.Nd OpenVPN DCO driver
+.Sh SYNOPSIS
+To compile this driver into the kernel,
+place the following lines in your
+kernel configuration file:
+.Bd -ragged -offset indent
+.Cd "device ovpn"
+.Ed
+.Pp
+Alternatively, to load the driver as a
+module at boot time, place the following line in
+.Xr loader.conf 5 :
+.Bd -literal -offset indent
+if_ovpn_load="YES"
+.Ed
+.Sh DESCRIPTION
+The
+.Nm
+device driver provides support for OpenVPN DCO.
+DCO, or Data Channel Offload, moves the OpenVPN data path into the kernel.
+This can improve performance.
+.Pp
+The
+.Nm
+interface is created automatically by the OpenVPN daemon.
+It requires no configuration other than that done by OpenVPN.
diff --git a/sys/conf/files b/sys/conf/files
index e23e23fb4fad..f28a6244da80 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -4141,6 +4141,7 @@ net/if_llatbl.c standard
net/if_me.c optional me inet
net/if_media.c standard
net/if_mib.c standard
+net/if_ovpn.c optional ovpn inet | ovpn inet6
net/if_stf.c optional stf inet inet6
net/if_tuntap.c optional tuntap
net/if_vlan.c optional vlan
diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
index 0521e06ce223..187768b59608 100644
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -3675,6 +3675,7 @@ prison_priv_check(struct ucred *cred, int priv)
case PRIV_NET_GIF:
case PRIV_NET_SETIFVNET:
case PRIV_NET_SETIFFIB:
+ case PRIV_NET_OVPN:
/*
* 802.11-related privileges.
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index 6917074ea5c8..68f1ae38a712 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -158,6 +158,7 @@ SUBDIR= \
${_if_me} \
if_infiniband \
if_lagg \
+ if_ovpn \
${_if_stf} \
if_tuntap \
if_vlan \
diff --git a/sys/modules/if_ovpn/Makefile b/sys/modules/if_ovpn/Makefile
new file mode 100644
index 000000000000..976ff7369c5a
--- /dev/null
+++ b/sys/modules/if_ovpn/Makefile
@@ -0,0 +1,6 @@
+.PATH: ${SRCTOP}/sys/net
+
+KMOD= if_ovpn
+SRCS= if_ovpn.c opt_inet.h opt_inet6.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/net/if_ovpn.c b/sys/net/if_ovpn.c
new file mode 100644
index 000000000000..57e85a414592
--- /dev/null
+++ b/sys/net/if_ovpn.c
@@ -0,0 +1,2437 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2021-2022 Rubicon Communications, LLC (Netgate)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+#include "opt_inet.h"
+#include "opt_inet6.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/buf_ring.h>
+#include <sys/epoch.h>
+#include <sys/file.h>
+#include <sys/filedesc.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mbuf.h>
+#include <sys/module.h>
+#include <sys/nv.h>
+#include <sys/priv.h>
+#include <sys/protosw.h>
+#include <sys/rmlock.h>
+#include <sys/smp.h>
+#include <sys/socket.h>
+#include <sys/socketvar.h>
+#include <sys/sockio.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+
+#include <machine/atomic.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_clone.h>
+#include <net/if_types.h>
+#include <net/if_var.h>
+#include <net/netisr.h>
+#include <net/route/nhop.h>
+
+#include <netinet/in.h>
+#include <netinet/in_fib.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/ip_var.h>
+#include <netinet/udp.h>
+#include <netinet/udp_var.h>
+
+#include <netinet6/ip6_var.h>
+#include <netinet6/in6_fib.h>
+
+#include <machine/in_cksum.h>
+
+#include <opencrypto/cryptodev.h>
+
+#include "if_ovpn.h"
+
+struct ovpn_kkey_dir {
+ int refcount;
+ uint8_t key[32];
+ uint8_t keylen;
+ uint8_t nonce[8];
+ uint8_t noncelen;
+ enum ovpn_key_cipher cipher;
+ crypto_session_t cryptoid;
+
+ struct mtx replay_mtx;
+ /*
+ * Last seen gapless sequence number. New rx seq numbers must be
+ * strictly higher than this.
+ */
+ uint32_t rx_seq;
+ /* Seen packets, relative to rx_seq. bit(0) will always be 0. */
+ uint64_t rx_window;
+};
+
+struct ovpn_kkey {
+ struct ovpn_kkey_dir *encrypt;
+ struct ovpn_kkey_dir *decrypt;
+ uint8_t keyid;
+ uint32_t peerid;
+};
+
+struct ovpn_keepalive {
+ uint32_t interval;
+ uint32_t timeout;
+};
+
+struct ovpn_wire_header {
+ uint32_t opcode; /* opcode, key id, peer id */
+ uint32_t seq;
+ uint8_t auth_tag[16];
+};
+
+struct ovpn_notification {
+ enum ovpn_notif_type type;
+ uint32_t peerid;
+};
+
+struct ovpn_softc;
+
+struct ovpn_kpeer {
+ int refcount;
+ uint32_t peerid;
+
+ struct ovpn_softc *sc;
+ struct sockaddr_storage local;
+ struct sockaddr_storage remote;
+
+ struct in_addr vpn4;
+ struct in6_addr vpn6;
+
+ struct ovpn_kkey keys[2];
+ uint32_t tx_seq;
+
+ struct ovpn_keepalive keepalive;
+ uint32_t *last_active;
+ struct callout ping_send;
+ struct callout ping_rcv;
+};
+
+#define OVPN_MAX_PEERS 128
+
+struct ovpn_counters {
+ uint64_t lost_ctrl_pkts_in;
+ uint64_t lost_ctrl_pkts_out;
+ uint64_t lost_data_pkts_in;
+ uint64_t lost_data_pkts_out;
+ uint64_t nomem_data_pkts_in;
+ uint64_t nomem_data_pkts_out;
+ uint64_t received_ctrl_pkts;
+ uint64_t received_data_pkts;
+ uint64_t sent_ctrl_pkts;
+ uint64_t sent_data_pkts;
+
+ uint64_t transport_bytes_sent;
+ uint64_t transport_bytes_received;
+ uint64_t tunnel_bytes_sent;
+ uint64_t tunnel_bytes_received;
+};
+#define OVPN_COUNTER_SIZE (sizeof(struct ovpn_counters)/sizeof(uint64_t))
+
+struct ovpn_softc {
+ int refcount;
+ struct rmlock lock;
+ struct ifnet *ifp;
+ struct socket *so;
+ int peercount;
+ struct ovpn_kpeer *peers[OVPN_MAX_PEERS]; /* XXX Hard limit for now? */
+
+ /* Pending packets */
+ struct buf_ring *rxring;
+ struct buf_ring *notifring;
+
+ counter_u64_t counters[OVPN_COUNTER_SIZE];
+
+ struct epoch_context epoch_ctx;
+};
+
+static struct ovpn_kpeer *ovpn_find_peer(struct ovpn_softc *, uint32_t);
+static bool ovpn_udp_input(struct mbuf *, int, struct inpcb *,
+ const struct sockaddr *, void *);
+static int ovpn_transmit_to_peer(struct ifnet *, struct mbuf *,
+ struct ovpn_kpeer *, struct rm_priotracker *);
+static int ovpn_encap(struct ovpn_softc *, uint32_t, struct mbuf *);
+static int ovpn_get_af(struct mbuf *);
+static void ovpn_free_kkey_dir(struct ovpn_kkey_dir *);
+static bool ovpn_check_replay(struct ovpn_kkey_dir *, uint32_t);
+
+#define OVPN_MTU_MIN 576
+#define OVPN_MTU_MAX (IP_MAXPACKET - sizeof(struct ip) - \
+ sizeof(struct udphdr) - sizeof(struct ovpn_wire_header))
+
+#define OVPN_OP_DATA_V2 0x09
+#define OVPN_OP_SHIFT 3
+
+VNET_DEFINE_STATIC(struct if_clone *, ovpn_cloner);
+#define V_ovpn_cloner VNET(ovpn_cloner)
+
+#define OVPN_RLOCK_TRACKER struct rm_priotracker _ovpn_lock_tracker; \
+ struct rm_priotracker *_ovpn_lock_trackerp = &_ovpn_lock_tracker
+#define OVPN_RLOCK(sc) rm_rlock(&(sc)->lock, _ovpn_lock_trackerp)
+#define OVPN_RUNLOCK(sc) rm_runlock(&(sc)->lock, _ovpn_lock_trackerp)
+#define OVPN_WLOCK(sc) rm_wlock(&(sc)->lock)
+#define OVPN_WUNLOCK(sc) rm_wunlock(&(sc)->lock)
+#define OVPN_ASSERT(sc) rm_assert(&(sc)->lock, RA_LOCKED)
+#define OVPN_RASSERT(sc) rm_assert(&(sc)->lock, RA_RLOCKED)
+#define OVPN_WASSERT(sc) rm_assert(&(sc)->lock, RA_WLOCKED)
+#define OVPN_UNLOCK_ASSERT(sc) rm_assert(&(sc)->lock, RA_UNLOCKED)
+
+#define OVPN_COUNTER_ADD(sc, name, val) \
+ counter_u64_add(sc->counters[offsetof(struct ovpn_counters, name) / \
+ sizeof(uint64_t)], val)
+
+#define TO_IN(x) ((struct sockaddr_in *)(x))
+#define TO_IN6(x) ((struct sockaddr_in6 *)(x))
+
+static const char ovpnname[] = "ovpn";
+static const char ovpngroupname[] = "openvpn";
+
+static MALLOC_DEFINE(M_OVPN, ovpnname, "OpenVPN DCO Interface");
+
+SYSCTL_DECL(_net_link);
+static SYSCTL_NODE(_net_link, IFT_OTHER, openvpn, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+ "OpenVPN DCO Interface");
+VNET_DEFINE_STATIC(int, replay_protection) = 0;
+#define V_replay_protection VNET(replay_protection)
+SYSCTL_INT(_net_link_openvpn, OID_AUTO, replay_protection, CTLFLAG_VNET | CTLFLAG_RW,
+ &VNET_NAME(replay_protection), 0, "Validate sequence numbers");
+
+static struct ovpn_kpeer *
+ovpn_find_peer(struct ovpn_softc *sc, uint32_t peerid)
+{
+ struct ovpn_kpeer *p = NULL;
+
+ OVPN_ASSERT(sc);
+
+ for (int i = 0; i < OVPN_MAX_PEERS; i++) {
+ p = sc->peers[i];
+ if (p == NULL)
+ continue;
+
+ if (p->peerid == peerid) {
+ MPASS(p->sc == sc);
+ break;
+ }
+ }
+
+ return (p);
+}
+
+static struct ovpn_kpeer *
+ovpn_find_only_peer(struct ovpn_softc *sc)
+{
+ OVPN_ASSERT(sc);
+
+ for (int i = 0; i < OVPN_MAX_PEERS; i++) {
+ if (sc->peers[i] == NULL)
+ continue;
+ return (sc->peers[i]);
+ }
+
+ MPASS(false);
+
+ return (NULL);
+}
+
+static uint16_t
+ovpn_get_port(struct sockaddr_storage *s)
+{
+ switch (s->ss_family) {
+ case AF_INET: {
+ struct sockaddr_in *in = (struct sockaddr_in *)s;
+ return (in->sin_port);
+ }
+ case AF_INET6: {
+ struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)s;
+ return (in6->sin6_port);
+ }
+ default:
+ panic("Unsupported address family %d", s->ss_family);
+ }
+}
+
+static int
+ovpn_nvlist_to_sockaddr(const nvlist_t *nvl, struct sockaddr_storage *sa)
+{
+ int af;
+
+ if (! nvlist_exists_number(nvl, "af"))
+ return (EINVAL);
+ if (! nvlist_exists_binary(nvl, "address"))
+ return (EINVAL);
+ if (! nvlist_exists_number(nvl, "port"))
+ return (EINVAL);
+
+ af = nvlist_get_number(nvl, "af");
+
+ switch (af) {
+#ifdef INET
+ case AF_INET: {
+ struct sockaddr_in *in = (struct sockaddr_in *)sa;
+ size_t len;
+ const void *addr = nvlist_get_binary(nvl, "address", &len);
+ in->sin_family = af;
+ if (len != sizeof(in->sin_addr))
+ return (EINVAL);
+
+ memcpy(&in->sin_addr, addr, sizeof(in->sin_addr));
+ in->sin_port = nvlist_get_number(nvl, "port");
+ break;
+ }
+#endif
+#ifdef INET6
+ case AF_INET6: {
+ struct sockaddr_in6 *in6 = (struct sockaddr_in6 *)sa;
+ size_t len;
+ const void *addr = nvlist_get_binary(nvl, "address", &len);
+ in6->sin6_family = af;
+ if (len != sizeof(in6->sin6_addr))
+ return (EINVAL);
+
+ memcpy(&in6->sin6_addr, addr, sizeof(in6->sin6_addr));
+ in6->sin6_port = nvlist_get_number(nvl, "port");
+ break;
+ }
+#endif
+ default:
+ return (EINVAL);
+ }
+
+ return (0);
+}
+
+static bool
+ovpn_has_peers(struct ovpn_softc *sc)
+{
+ OVPN_ASSERT(sc);
+
+ return (sc->peercount > 0);
+}
+
+static void
+ovpn_rele_so(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
+{
+ bool has_peers;
+
+ OVPN_WASSERT(sc);
+
+ if (sc->so == NULL)
+ return;
+
+ has_peers = ovpn_has_peers(sc);
+
+ /* Only remove the tunnel function if we're releasing the socket for
+ * the last peer. */
+ if (! has_peers)
+ (void)udp_set_kernel_tunneling(sc->so, NULL, NULL, NULL);
+
+ sorele(sc->so);
+
+ if (! has_peers)
+ sc->so = NULL;
+}
+
+static void
+ovpn_notify_del_peer(struct ovpn_softc *sc, struct ovpn_kpeer *peer)
+{
+ struct ovpn_notification *n;
+
+ OVPN_WASSERT(sc);
+
+ n = malloc(sizeof(*n), M_OVPN, M_NOWAIT);
+ if (n == NULL)
+ return;
+
+ n->peerid = peer->peerid;
+ n->type = OVPN_NOTIF_DEL_PEER;
+ if (buf_ring_enqueue(sc->notifring, n) != 0) {
+ free(n, M_OVPN);
+ } else if (sc->so != NULL) {
+ /* Wake up userspace */
+ sc->so->so_error = EAGAIN;
+ sorwakeup(sc->so);
+ sowwakeup(sc->so);
+ }
+}
+
+static void
+ovpn_peer_release_ref(struct ovpn_kpeer *peer, bool locked)
+{
+ struct ovpn_softc *sc;
+
+ atomic_add_int(&peer->refcount, -1);
+
+ if (atomic_load_int(&peer->refcount) > 0)
+ return;
+
+ sc = peer->sc;
+
+ if (! locked) {
+ OVPN_WLOCK(sc);
+
+ /* Might have changed before we acquired the lock. */
+ if (atomic_load_int(&peer->refcount) > 0) {
+ OVPN_WUNLOCK(sc);
+ return;
+ }
+ }
+
+ /* The peer should have been removed from the list already. */
+ MPASS(ovpn_find_peer(sc, peer->peerid) == NULL);
+
+ ovpn_notify_del_peer(sc, peer);
+
+ for (int i = 0; i < 2; i++) {
+ ovpn_free_kkey_dir(peer->keys[i].encrypt);
+ ovpn_free_kkey_dir(peer->keys[i].decrypt);
+ }
+
+ ovpn_rele_so(sc, peer);
+
+ callout_stop(&peer->ping_send);
+ callout_stop(&peer->ping_rcv);
+ uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
+ free(peer, M_OVPN);
+
+ if (! locked)
+ OVPN_WUNLOCK(sc);
+}
+
+static int
+ovpn_new_peer(struct ifnet *ifp, const nvlist_t *nvl)
+{
+#ifdef INET6
+ struct epoch_tracker et;
+#endif
+ struct sockaddr_storage remote;
+ struct ovpn_kpeer *peer = NULL;
+ struct file *fp = NULL;
+ struct sockaddr *name = NULL;
+ struct ovpn_softc *sc = ifp->if_softc;
+ struct thread *td = curthread;
+ struct socket *so = NULL;
+ u_int fflag;
+ int fd;
+ uint32_t peerid;
+ int ret = 0, i;
+
+ if (nvl == NULL)
+ return (EINVAL);
+
+ if (! nvlist_exists_number(nvl, "peerid"))
+ return (EINVAL);
+
+ if (! nvlist_exists_number(nvl, "fd"))
+ return (EINVAL);
+
+ if (! nvlist_exists_nvlist(nvl, "remote"))
+ return (EINVAL);
+
+ peerid = nvlist_get_number(nvl, "peerid");
+
+ ret = ovpn_nvlist_to_sockaddr(nvlist_get_nvlist(nvl, "remote"),
+ &remote);
+ if (ret != 0)
+ return (ret);
+
+ fd = nvlist_get_number(nvl, "fd");
+
+ /* Look up the userspace process and use the fd to find the socket. */
+ ret = getsock_cap(td, fd, &cap_connect_rights, &fp,
+ &fflag, NULL);
+ if (ret != 0)
+ return (ret);
+
+ so = fp->f_data;
+
+ peer = malloc(sizeof(*peer), M_OVPN, M_WAITOK | M_ZERO);
+ peer->peerid = peerid;
+ peer->sc = sc;
+ peer->tx_seq = 1;
+ peer->refcount = 1;
+ peer->last_active = uma_zalloc_pcpu(pcpu_zone_4, M_WAITOK | M_ZERO);
+
+ if (nvlist_exists_binary(nvl, "vpn_ipv4")) {
+ size_t len;
+ const void *addr = nvlist_get_binary(nvl, "vpn_ipv4", &len);
+ if (len != sizeof(peer->vpn4)) {
+ ret = EINVAL;
+ goto error;
+ }
+ memcpy(&peer->vpn4, addr, len);
+ }
+
+ if (nvlist_exists_binary(nvl, "vpn_ipv6")) {
+ size_t len;
+ const void *addr = nvlist_get_binary(nvl, "vpn_ipv6", &len);
+ if (len != sizeof(peer->vpn6)) {
+ ret = EINVAL;
+ goto error;
+ }
+ memcpy(&peer->vpn6, addr, len);
+ }
+
+ callout_init_rm(&peer->ping_send, &sc->lock, CALLOUT_SHAREDLOCK);
+ callout_init_rm(&peer->ping_rcv, &sc->lock, 0);
+
+ ret = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, &name);
+ if (ret)
+ goto error;
+
+ if (ovpn_get_port((struct sockaddr_storage *)name) == 0) {
+ ret = EINVAL;
+ goto error;
+ }
+ if (name->sa_family != remote.ss_family) {
+ ret = EINVAL;
+ goto error;
+ }
+
+ memcpy(&peer->local, name, name->sa_len);
+ memcpy(&peer->remote, &remote, sizeof(remote));
+ free(name, M_SONAME);
+ name = NULL;
+
+#ifdef INET6
+ if (peer->local.ss_family == AF_INET6 &&
+ IN6_IS_ADDR_UNSPECIFIED(&TO_IN6(&peer->local)->sin6_addr)) {
+ NET_EPOCH_ENTER(et);
+ ret = in6_selectsrc_addr(curthread->td_proc->p_fibnum,
+ &TO_IN6(&peer->remote)->sin6_addr,
+ 0, NULL, &TO_IN6(&peer->local)->sin6_addr, NULL);
+ NET_EPOCH_EXIT(et);
+ if (ret != 0) {
+ goto error;
+ }
+ }
+#endif
+ OVPN_WLOCK(sc);
+
+ /* Disallow peer id re-use. */
+ if (ovpn_find_peer(sc, peerid) != NULL) {
+ ret = EEXIST;
+ goto error_locked;
+ }
+
+ /* Must be the same socket as for other peers on this interface. */
+ if (sc->so != NULL && so != sc->so)
+ goto error_locked;
+
+ if (sc->so == NULL)
+ sc->so = so;
+
+ /* Insert the peer into the list. */
+ for (i = 0; i < OVPN_MAX_PEERS; i++) {
+ if (sc->peers[i] != NULL)
+ continue;
+
+ MPASS(sc->peers[i] == NULL);
+ sc->peers[i] = peer;
+ sc->peercount++;
+ soref(sc->so);
+ break;
+ }
+ if (i == OVPN_MAX_PEERS) {
+ ret = ENOSPC;
+ goto error_locked;
+ }
+
+ ret = udp_set_kernel_tunneling(sc->so, ovpn_udp_input, NULL, sc);
+ if (ret == EBUSY) {
+ /* Fine, another peer already set the input function. */
+ ret = 0;
+ }
+ if (ret != 0) {
+ sc->peers[i] = NULL;
+ sc->peercount--;
+ goto error_locked;
+ }
+
+ OVPN_WUNLOCK(sc);
+
+ goto done;
+
+error_locked:
+ OVPN_WUNLOCK(sc);
+error:
+ free(name, M_SONAME);
+ uma_zfree_pcpu(pcpu_zone_4, peer->last_active);
+ free(peer, M_OVPN);
+done:
+ if (fp != NULL)
+ fdrop(fp, td);
+
+ return (ret);
+}
+
+static int
+_ovpn_del_peer(struct ovpn_softc *sc, uint32_t peerid)
+{
+ struct ovpn_kpeer *peer;
+ int i;
+
+ OVPN_WASSERT(sc);
+
+ for (i = 0; i < OVPN_MAX_PEERS; i++) {
+ if (sc->peers[i] == NULL)
+ continue;
+ if (sc->peers[i]->peerid != peerid)
+ continue;
+
+ peer = sc->peers[i];
+ break;
+ }
+ if (i == OVPN_MAX_PEERS)
+ return (ENOENT);
+
+ sc->peers[i] = NULL;
+ sc->peercount--;
+
+ ovpn_peer_release_ref(peer, true);
+
+ return (0);
+}
+
+static int
+ovpn_del_peer(struct ifnet *ifp, nvlist_t *nvl)
+{
+ struct ovpn_softc *sc = ifp->if_softc;
+ uint32_t peerid;
+ int ret;
+
+ OVPN_WASSERT(sc);
+
+ if (nvl == NULL)
+ return (EINVAL);
+
+ if (! nvlist_exists_number(nvl, "peerid"))
+ return (EINVAL);
+
+ peerid = nvlist_get_number(nvl, "peerid");
+
+ ret = _ovpn_del_peer(sc, peerid);
+
+ return (ret);
+}
+
+static int
+ovpn_create_kkey_dir(struct ovpn_kkey_dir **kdirp,
+ const nvlist_t *nvl)
+{
+ struct crypto_session_params csp;
+ struct ovpn_kkey_dir *kdir;
+ const char *ciphername;
+ enum ovpn_key_cipher cipher;
+ const void *key, *iv;
+ size_t keylen = 0, ivlen = 0;
+ int error;
+
+ if (! nvlist_exists_string(nvl, "cipher"))
+ return (EINVAL);
+ ciphername = nvlist_get_string(nvl, "cipher");
+
+ if (strcmp(ciphername, "none") == 0)
+ cipher = OVPN_CIPHER_ALG_NONE;
+ else if (strcmp(ciphername, "AES-256-GCM") == 0)
+ cipher = OVPN_CIPHER_ALG_AES_GCM;
+ else if (strcmp(ciphername, "CHACHA20-POLY1305") == 0)
+ cipher = OVPN_CIPHER_ALG_CHACHA20_POLY1305;
+ else
+ return (EINVAL);
+
+ if (cipher != OVPN_CIPHER_ALG_NONE) {
+ if (! nvlist_exists_binary(nvl, "key"))
+ return (EINVAL);
+ key = nvlist_get_binary(nvl, "key", &keylen);
+ if (keylen > sizeof(kdir->key))
+ return (E2BIG);
+
+ if (! nvlist_exists_binary(nvl, "iv"))
+ return (EINVAL);
+ iv = nvlist_get_binary(nvl, "iv", &ivlen);
+ if (ivlen != 8)
+ return (E2BIG);
+ }
+
+ kdir = malloc(sizeof(struct ovpn_kkey_dir), M_OVPN,
+ M_WAITOK | M_ZERO);
+
+ kdir->cipher = cipher;
+ kdir->keylen = keylen;
+ memcpy(kdir->key, key, keylen);
+ kdir->noncelen = ivlen;
+ memcpy(kdir->nonce, iv, ivlen);
+
+ if (kdir->cipher != OVPN_CIPHER_ALG_NONE) {
+ /* Crypto init */
+ bzero(&csp, sizeof(csp));
+ csp.csp_mode = CSP_MODE_AEAD;
+
+ if (kdir->cipher == OVPN_CIPHER_ALG_CHACHA20_POLY1305)
+ csp.csp_cipher_alg = CRYPTO_CHACHA20_POLY1305;
+ else
+ csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
+
+ csp.csp_flags |= CSP_F_SEPARATE_AAD;
+
+ csp.csp_cipher_klen = kdir->keylen;
+ csp.csp_cipher_key = kdir->key;
+ csp.csp_ivlen = 96 / 8;
+
+ error = crypto_newsession(&kdir->cryptoid, &csp,
+ CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
+ if (error) {
+ free(kdir, M_OVPN);
+ return (error);
+ }
+ }
+
+ mtx_init(&kdir->replay_mtx, "if_ovpn rx replay", NULL, MTX_DEF);
+ *kdirp = kdir;
+
+ return (0);
+}
+
+static void
+ovpn_free_kkey_dir(struct ovpn_kkey_dir *kdir)
+{
+ if (kdir == NULL)
+ return;
+
+ mtx_destroy(&kdir->replay_mtx);
+
+ crypto_freesession(kdir->cryptoid);
+ free(kdir, M_OVPN);
+}
+
+static int
+ovpn_set_key(struct ifnet *ifp, const nvlist_t *nvl)
+{
+ struct ovpn_softc *sc = ifp->if_softc;
+ struct ovpn_kkey_dir *enc, *dec;
+ struct ovpn_kpeer *peer;
+ int slot, keyid, peerid;
+ int error;
+
+ if (nvl == NULL)
+ return (EINVAL);
+
+ if (! nvlist_exists_number(nvl, "slot"))
+ return (EINVAL);
+ slot = nvlist_get_number(nvl, "slot");
+
+ if (! nvlist_exists_number(nvl, "keyid"))
+ return (EINVAL);
+ keyid = nvlist_get_number(nvl, "keyid");
+
+ if (! nvlist_exists_number(nvl, "peerid"))
+ return (EINVAL);
+ peerid = nvlist_get_number(nvl, "peerid");
+
+ if (slot != OVPN_KEY_SLOT_PRIMARY &&
+ slot != OVPN_KEY_SLOT_SECONDARY)
+ return (EINVAL);
+
+ if (! nvlist_exists_nvlist(nvl, "encrypt") ||
+ ! nvlist_exists_nvlist(nvl, "decrypt"))
+ return (EINVAL);
+
+ error = ovpn_create_kkey_dir(&enc, nvlist_get_nvlist(nvl, "encrypt"));
+ if (error)
+ return (error);
+
+ error = ovpn_create_kkey_dir(&dec, nvlist_get_nvlist(nvl, "decrypt"));
+ if (error) {
+ ovpn_free_kkey_dir(enc);
+ return (error);
+ }
+
+ OVPN_WLOCK(sc);
+
+ peer = ovpn_find_peer(sc, peerid);
+ if (peer == NULL) {
+ ovpn_free_kkey_dir(dec);
+ ovpn_free_kkey_dir(enc);
+ OVPN_WUNLOCK(sc);
+ return (ENOENT);
+ }
+
+ ovpn_free_kkey_dir(peer->keys[slot].encrypt);
+ ovpn_free_kkey_dir(peer->keys[slot].decrypt);
+
+ peer->keys[slot].encrypt = enc;
+ peer->keys[slot].decrypt = dec;
+
+ peer->keys[slot].keyid = keyid;
+ peer->keys[slot].peerid = peerid;
+
+ OVPN_WUNLOCK(sc);
+
+ return (0);
+}
+
+static int
+ovpn_check_key(struct ovpn_softc *sc, struct ovpn_kpeer *peer, enum ovpn_key_slot slot)
+{
+ OVPN_ASSERT(sc);
+
+ if (peer->keys[slot].encrypt == NULL)
+ return (ENOLINK);
+
+ if (peer->keys[slot].decrypt == NULL)
+ return (ENOLINK);
+
+ return (0);
+}
+
+static int
+ovpn_start(struct ifnet *ifp)
+{
+ struct ovpn_softc *sc = ifp->if_softc;
+
+ OVPN_WLOCK(sc);
+
*** 1693 LINES SKIPPED ***