git: 0972294ef034 - main - pf: add a dedicated pf pool for route options

From: Kristof Provost <kp_at_FreeBSD.org>
Date: Fri, 24 Jan 2025 10:25:03 UTC
The branch main has been updated by kp:

URL: https://cgit.FreeBSD.org/src/commit/?id=0972294ef034d92f59857b8312dd2e1e3a7adc9c

commit 0972294ef034d92f59857b8312dd2e1e3a7adc9c
Author:     Kristof Provost <kp@FreeBSD.org>
AuthorDate: 2025-01-20 17:25:37 +0000
Commit:     Kristof Provost <kp@FreeBSD.org>
CommitDate: 2025-01-24 10:20:30 +0000

    pf: add a dedicated pf pool for route options
    
    As suggested by henning.
    Which unbreaks ie route-to after the recent pf changes.
    
    With much help debugging and pointing out of missing bits from claudio@
    
    ok claudio@ "looks good" henning@
    
    Obtained from:  OpenBSD, jsg <jsg@openbsd.org>, 7fa5c09028
    Sponsored by:   Rubicon Communications, LLC ("Netgate")
---
 lib/libpfctl/libpfctl.c     |  2 +
 lib/libpfctl/libpfctl.h     |  1 +
 sbin/pfctl/parse.y          | 95 +++++++++++++++++++++++++++------------------
 sbin/pfctl/pfctl.c          | 16 ++++++++
 sbin/pfctl/pfctl_optimize.c |  7 ++++
 sbin/pfctl/pfctl_parser.c   |  1 +
 sys/net/pfvar.h             |  3 +-
 sys/netpfil/pf/if_pfsync.c  |  8 +++-
 sys/netpfil/pf/pf.c         | 11 +++++-
 sys/netpfil/pf/pf.h         |  2 +-
 sys/netpfil/pf/pf_ioctl.c   | 53 +++++++++++++++++++------
 sys/netpfil/pf/pf_nl.c      |  2 +
 sys/netpfil/pf/pf_nl.h      |  1 +
 13 files changed, 147 insertions(+), 55 deletions(-)

diff --git a/lib/libpfctl/libpfctl.c b/lib/libpfctl/libpfctl.c
index f0708c8f0439..2297b24d37a0 100644
--- a/lib/libpfctl/libpfctl.c
+++ b/lib/libpfctl/libpfctl.c
@@ -1227,6 +1227,7 @@ snl_add_msg_attr_pf_rule(struct snl_writer *nw, uint32_t type, const struct pfct
 	snl_add_msg_attr_string(nw, PF_RT_OVERLOAD_TBLNAME, r->overload_tblname);
 	snl_add_msg_attr_rpool(nw, PF_RT_RPOOL_RDR, &r->rdr);
 	snl_add_msg_attr_rpool(nw, PF_RT_RPOOL_NAT, &r->nat);
+	snl_add_msg_attr_rpool(nw, PF_RT_RPOOL_RT, &r->route);
 	snl_add_msg_attr_u32(nw, PF_RT_OS_FINGERPRINT, r->os_fingerprint);
 	snl_add_msg_attr_u32(nw, PF_RT_RTABLEID, r->rtableid);
 	snl_add_msg_attr_timeouts(nw, PF_RT_TIMEOUT, r->timeout);
@@ -1661,6 +1662,7 @@ static struct snl_attr_parser ap_getrule[] = {
 	{ .type = PF_RT_MAX_SRC_CONN, .off = _OUT(r.max_src_conn), .cb = snl_attr_get_uint32 },
 	{ .type = PF_RT_RPOOL_NAT, .off = _OUT(r.nat), .arg = &pool_parser, .cb = snl_attr_get_nested },
 	{ .type = PF_RT_NAF, .off = _OUT(r.naf), .cb = snl_attr_get_uint8 },
+	{ .type = PF_RT_RPOOL_RT, .off = _OUT(r.route), .arg = &pool_parser, .cb = snl_attr_get_nested },
 };
 #undef _OUT
 SNL_DECLARE_PARSER(getrule_parser, struct genlmsghdr, snl_f_p_empty, ap_getrule);
diff --git a/lib/libpfctl/libpfctl.h b/lib/libpfctl/libpfctl.h
index 14ea06fd151a..2532894ffa21 100644
--- a/lib/libpfctl/libpfctl.h
+++ b/lib/libpfctl/libpfctl.h
@@ -180,6 +180,7 @@ struct pfctl_rule {
 		struct pfctl_pool	 rpool;
 		struct pfctl_pool	 rdr;
 	};
+	struct pfctl_pool	 route;
 
 	uint64_t		 evaluations;
 	uint64_t		 packets[2];
diff --git a/sbin/pfctl/parse.y b/sbin/pfctl/parse.y
index ab74d2dd57ab..830581c57f9c 100644
--- a/sbin/pfctl/parse.y
+++ b/sbin/pfctl/parse.y
@@ -304,6 +304,7 @@ static struct filter_opts {
 	}			 divert;
 	struct redirspec	 nat;
 	struct redirspec	 rdr;
+	struct redirspec	 rroute;
 	/* new-style scrub opts */
 	int			 nodf;
 	int			 minttl;
@@ -382,11 +383,12 @@ void		 expand_eth_rule(struct pfctl_eth_rule *,
 		    struct node_host *, struct node_host *, const char *,
 		    const char *);
 void		 expand_rule(struct pfctl_rule *, struct node_if *,
-		    struct redirspec *, struct redirspec *, struct node_host *,
-		    struct node_host *, struct node_proto *, struct node_os *,
-		    struct node_host *, struct node_port *, struct node_host *,
-		    struct node_port *, struct node_uid *, struct node_gid *,
-		    struct node_if *, struct node_icmp *, const char *);
+		    struct redirspec *, struct redirspec *, struct redirspec *,
+		    struct node_host *, struct node_host *, struct node_host *,
+		    struct node_proto *, struct node_os *, struct node_host *,
+		    struct node_port *, struct node_host *, struct node_port *,
+		    struct node_uid *, struct node_gid *, struct node_if *,
+		    struct node_icmp *, const char *);
 int		 expand_altq(struct pf_altq *, struct node_if *,
 		    struct node_queue *, struct node_queue_bw bwspec,
 		    struct node_queue_opt *);
@@ -1080,9 +1082,9 @@ anchorrule	: ANCHOR anchorname dir quick interface af proto fromto
 			decide_address_family($8.src.host, &r.af);
 			decide_address_family($8.dst.host, &r.af);
 
-			expand_rule(&r, $5, NULL, NULL, NULL, NULL, $7, $8.src_os,
-			    $8.src.host, $8.src.port, $8.dst.host, $8.dst.port,
-			    $9.uid, $9.gid, $9.rcv, $9.icmpspec,
+			expand_rule(&r, $5, NULL, NULL, NULL, NULL, NULL, NULL,
+			    $7, $8.src_os, $8.src.host, $8.src.port, $8.dst.host,
+			    $8.dst.port, $9.uid, $9.gid, $9.rcv, $9.icmpspec,
 			    pf->astack[pf->asd + 1] ? pf->alast->name : $2);
 			free($2);
 			pf->astack[pf->asd + 1] = NULL;
@@ -1103,9 +1105,9 @@ anchorrule	: ANCHOR anchorname dir quick interface af proto fromto
 			decide_address_family($6.src.host, &r.af);
 			decide_address_family($6.dst.host, &r.af);
 
-			expand_rule(&r, $3, NULL, NULL, NULL, NULL, $5, $6.src_os,
-			    $6.src.host, $6.src.port, $6.dst.host, $6.dst.port,
-			    0, 0, 0, 0, $2);
+			expand_rule(&r, $3, NULL, NULL, NULL, NULL, NULL, NULL,
+			    $5, $6.src_os, $6.src.host, $6.src.port, $6.dst.host,
+			    $6.dst.port, 0, 0, 0, 0, $2);
 			free($2);
 		}
 		| RDRANCHOR string interface af proto fromto rtable {
@@ -1145,9 +1147,9 @@ anchorrule	: ANCHOR anchorname dir quick interface af proto fromto
 				r.dst.port_op = $6.dst.port->op;
 			}
 
-			expand_rule(&r, $3, NULL, NULL, NULL, NULL, $5, $6.src_os,
-			    $6.src.host, $6.src.port, $6.dst.host, $6.dst.port,
-			    0, 0, 0, 0, $2);
+			expand_rule(&r, $3, NULL, NULL, NULL, NULL, NULL, NULL,
+			    $5, $6.src_os, $6.src.host, $6.src.port, $6.dst.host,
+			    $6.dst.port, 0, 0, 0, 0, $2);
 			free($2);
 		}
 		| BINATANCHOR string interface af proto fromto rtable {
@@ -1468,9 +1470,9 @@ scrubrule	: scrubaction dir logquick interface af proto fromto scrub_opts
 			r.match_tag_not = $8.match_tag_not;
 			r.rtableid = $8.rtableid;
 
-			expand_rule(&r, $4, NULL, NULL, NULL, NULL, $6, $7.src_os,
-			    $7.src.host, $7.src.port, $7.dst.host, $7.dst.port,
-			    NULL, NULL, NULL, NULL, "");
+			expand_rule(&r, $4, NULL, NULL, NULL, NULL, NULL, NULL,
+			    $6, $7.src_os, $7.src.host, $7.src.port, $7.dst.host,
+			    $7.dst.port, NULL, NULL, NULL, NULL, "");
 		}
 		;
 
@@ -1633,8 +1635,8 @@ antispoof	: ANTISPOOF logquick antispoof_ifspc af antispoof_opts {
 				}
 
 				if (h != NULL)
-					expand_rule(&r, j, NULL, NULL, NULL, NULL, NULL, NULL, h,
-					    NULL, NULL, NULL, NULL, NULL,
+					expand_rule(&r, j, NULL, NULL, NULL, NULL, NULL, NULL,
+					    NULL, NULL, h, NULL, NULL, NULL, NULL, NULL,
 					    NULL, NULL, "");
 
 				if ((i->ifa_flags & IFF_LOOPBACK) == 0) {
@@ -1656,7 +1658,7 @@ antispoof	: ANTISPOOF logquick antispoof_ifspc af antispoof_opts {
 						h = ifa_lookup(i->ifname, 0);
 					if (h != NULL)
 						expand_rule(&r, NULL, NULL, NULL, NULL, NULL,
-						    NULL, NULL, h, NULL, NULL,
+						    NULL, NULL, NULL, NULL, h, NULL, NULL,
 						    NULL, NULL, NULL, NULL, NULL, "");
 				} else
 					free(hh);
@@ -2726,9 +2728,9 @@ pfrule		: action dir logquick interface route af proto fromto
 					YYERROR;
 				}
 				r.rt = $5.rt;
-				r.rdr.opts = $5.pool_opts;
+				r.route.opts = $5.pool_opts;
 				if ($5.key != NULL)
-					memcpy(&r.rdr.key, $5.key,
+					memcpy(&r.route.key, $5.key,
 					    sizeof(struct pf_poolhashkey));
 			}
 			if (r.rt) {
@@ -2739,26 +2741,26 @@ pfrule		: action dir logquick interface route af proto fromto
 					    "matching address family found.");
 					YYERROR;
 				}
-				if ((r.rdr.opts & PF_POOL_TYPEMASK) ==
+				if ((r.route.opts & PF_POOL_TYPEMASK) ==
 				    PF_POOL_NONE && ($5.host->next != NULL ||
 				    $5.host->addr.type == PF_ADDR_TABLE ||
 				    DYNIF_MULTIADDR($5.host->addr)))
-					r.rdr.opts |= PF_POOL_ROUNDROBIN;
-				if ((r.rdr.opts & PF_POOL_TYPEMASK) !=
+					r.route.opts |= PF_POOL_ROUNDROBIN;
+				if ((r.route.opts & PF_POOL_TYPEMASK) !=
 				    PF_POOL_ROUNDROBIN &&
 				    disallow_table($5.host, "tables are only "
 				    "supported in round-robin routing pools"))
 					YYERROR;
-				if ((r.rdr.opts & PF_POOL_TYPEMASK) !=
+				if ((r.route.opts & PF_POOL_TYPEMASK) !=
 				    PF_POOL_ROUNDROBIN &&
 				    disallow_alias($5.host, "interface (%s) "
 				    "is only supported in round-robin "
 				    "routing pools"))
 					YYERROR;
 				if ($5.host->next != NULL) {
-					if ((r.rdr.opts & PF_POOL_TYPEMASK) !=
+					if ((r.route.opts & PF_POOL_TYPEMASK) !=
 					    PF_POOL_ROUNDROBIN) {
-						yyerror("r.rdr.opts must "
+						yyerror("r.route.opts must "
 						    "be PF_POOL_ROUNDROBIN");
 						YYERROR;
 					}
@@ -2833,7 +2835,8 @@ pfrule		: action dir logquick interface route af proto fromto
 					YYERROR;
 			}
 
-			expand_rule(&r, $4, &$9.nat, &$9.rdr, $5.host, $9.nat.rdr ? $9.nat.rdr->host : NULL,
+			expand_rule(&r, $4, &$9.nat, &$9.rdr, &$9.rroute,
+			    NULL, $9.nat.rdr ? $9.nat.rdr->host : NULL, $5.host,
 			    $7, $8.src_os, $8.src.host, $8.src.port, $8.dst.host,
 			    $8.dst.port, $9.uid, $9.gid, $9.rcv, $9.icmpspec, "");
 		}
@@ -4989,8 +4992,9 @@ natrule		: nataction interface af proto fromto tag tagged rtable
 				o = o->next;
 			}
 
-			expand_rule(&r, $2, NULL, NULL, $9 == NULL ? NULL : $9->host,
-			    NULL, $4, $5.src_os, $5.src.host, $5.src.port, $5.dst.host,
+			expand_rule(&r, $2, NULL, NULL, NULL,
+			    $9 == NULL ? NULL : $9->host, NULL, NULL, $4,
+			    $5.src_os, $5.src.host, $5.src.port, $5.dst.host,
 			    $5.dst.port, 0, 0, 0, 0, "");
 			free($9);
 		}
@@ -6149,13 +6153,13 @@ expand_eth_rule(struct pfctl_eth_rule *r,
 void
 expand_rule(struct pfctl_rule *r,
     struct node_if *interfaces, struct redirspec *nat,
-    struct redirspec *rdr, struct node_host *rdr_hosts,
-    struct node_host *nat_hosts,
-    struct node_proto *protos, struct node_os *src_oses,
-    struct node_host *src_hosts, struct node_port *src_ports,
-    struct node_host *dst_hosts, struct node_port *dst_ports,
-    struct node_uid *uids, struct node_gid *gids, struct node_if *rcv,
-    struct node_icmp *icmp_types, const char *anchor_call)
+    struct redirspec *rdr, struct redirspec *route,
+    struct node_host *rdr_hosts, struct node_host *nat_hosts,
+    struct node_host *route_hosts, struct node_proto *protos,
+    struct node_os *src_oses, struct node_host *src_hosts,
+    struct node_port *src_ports, struct node_host *dst_hosts,
+    struct node_port *dst_ports, struct node_uid *uids, struct node_gid *gids,
+    struct node_if *rcv, struct node_icmp *icmp_types, const char *anchor_call)
 {
 	sa_family_t		 af = r->af;
 	int			 added = 0, error = 0;
@@ -6331,6 +6335,21 @@ expand_rule(struct pfctl_rule *r,
 				pa->ifname[0] = 0;
 			TAILQ_INSERT_TAIL(&r->nat.list, pa, entries);
 		}
+		TAILQ_INIT(&r->route.list);
+		for (h = route_hosts; h != NULL; h = h->next) {
+			pa = calloc(1, sizeof(struct pf_pooladdr));
+			if (pa == NULL)
+				err(1, "expand_rule: calloc");
+			pa->addr = h->addr;
+			if (h->ifname != NULL) {
+				if (strlcpy(pa->ifname, h->ifname,
+				    sizeof(pa->ifname)) >=
+				    sizeof(pa->ifname))
+					errx(1, "expand_rule: strlcpy");
+			} else
+				pa->ifname[0] = 0;
+			TAILQ_INSERT_TAIL(&r->route.list, pa, entries);
+		}
 
 		r->nat.proxy_port[0] = PF_NAT_PROXY_PORT_LOW;
 		r->nat.proxy_port[1] = PF_NAT_PROXY_PORT_HIGH;
diff --git a/sbin/pfctl/pfctl.c b/sbin/pfctl/pfctl.c
index 7b54bc1c7c7a..ec07a5da999c 100644
--- a/sbin/pfctl/pfctl.c
+++ b/sbin/pfctl/pfctl.c
@@ -1310,6 +1310,10 @@ pfctl_show_rules(int dev, char *path, int opts, enum pfctl_show format,
 		    nr, ri.ticket, PF_SCRUB, path, PF_NAT) != 0)
 			goto error;
 
+		if (pfctl_get_pool(dev, &rule.route,
+		    nr, ri.ticket, PF_SCRUB, path, PF_RT) != 0)
+			goto error;
+
 		switch (format) {
 		case PFCTL_SHOW_LABELS:
 			break;
@@ -1325,6 +1329,7 @@ pfctl_show_rules(int dev, char *path, int opts, enum pfctl_show format,
 		}
 		pfctl_clear_pool(&rule.rdr);
 		pfctl_clear_pool(&rule.nat);
+		pfctl_clear_pool(&rule.route);
 	}
 	ret = pfctl_get_rules_info_h(pfh, &ri, PF_PASS, path);
 	if (ret != 0) {
@@ -1346,6 +1351,10 @@ pfctl_show_rules(int dev, char *path, int opts, enum pfctl_show format,
 		    nr, ri.ticket, PF_PASS, path, PF_NAT) != 0)
 			goto error;
 
+		if (pfctl_get_pool(dev, &rule.route,
+		    nr, ri.ticket, PF_PASS, path, PF_RT) != 0)
+			goto error;
+
 		switch (format) {
 		case PFCTL_SHOW_LABELS: {
 			bool show = false;
@@ -1506,6 +1515,9 @@ pfctl_show_nat(int dev, char *path, int opts, char *anchorname, int depth,
 			if (pfctl_get_pool(dev, &rule.nat, nr,
 			    ri.ticket, nattype[i], path, PF_NAT) != 0)
 				return (-1);
+			if (pfctl_get_pool(dev, &rule.route, nr,
+			    ri.ticket, nattype[i], path, PF_RT) != 0)
+				return (-1);
 
 			if (dotitle) {
 				pfctl_print_title("TRANSLATION RULES:");
@@ -1761,6 +1773,8 @@ pfctl_append_rule(struct pfctl *pf, struct pfctl_rule *r,
 	pfctl_move_pool(&r->rdr, &rule->rdr);
 	TAILQ_INIT(&rule->nat.list);
 	pfctl_move_pool(&r->nat, &rule->nat);
+	TAILQ_INIT(&rule->route.list);
+	pfctl_move_pool(&r->route, &rule->route);
 
 	TAILQ_INSERT_TAIL(rs->rules[rs_num].active.ptr, rule, entries);
 	return (0);
@@ -2065,6 +2079,8 @@ pfctl_load_rule(struct pfctl *pf, char *path, struct pfctl_rule *r, int depth)
 			return (1);
 		if (pfctl_add_pool(pf, &r->nat, r->naf ? r->naf : r->af, PF_NAT))
 			return (1);
+		if (pfctl_add_pool(pf, &r->route, r->af, PF_RT))
+			return (1);
 		error = pfctl_add_rule_h(pf->h, r, anchor, name, ticket,
 		    pf->paddr.ticket);
 		switch (error) {
diff --git a/sbin/pfctl/pfctl_optimize.c b/sbin/pfctl/pfctl_optimize.c
index a97664e0c929..7817bcfd284a 100644
--- a/sbin/pfctl/pfctl_optimize.c
+++ b/sbin/pfctl/pfctl_optimize.c
@@ -137,6 +137,7 @@ static struct pf_rule_field {
     PF_RULE_FIELD(flush,		BREAK),
     PF_RULE_FIELD(rdr,			BREAK),
     PF_RULE_FIELD(nat,			BREAK),
+    PF_RULE_FIELD(route,		BREAK),
     PF_RULE_FIELD(logif,		BREAK),
 
     /*
@@ -303,6 +304,12 @@ pfctl_optimize_ruleset(struct pfctl *pf, struct pfctl_ruleset *rs)
 		} else
 			bzero(&por->por_rule.nat,
 			    sizeof(por->por_rule.nat));
+		if (TAILQ_FIRST(&r->route.list) != NULL) {
+			TAILQ_INIT(&por->por_rule.route.list);
+			pfctl_move_pool(&r->route, &por->por_rule.route);
+		} else
+			bzero(&por->por_rule.route,
+			    sizeof(por->por_rule.route));
 
 		TAILQ_INSERT_TAIL(&opt_queue, por, por_entry);
 	}
diff --git a/sbin/pfctl/pfctl_parser.c b/sbin/pfctl/pfctl_parser.c
index 85f1797e58e1..df76f8312cf3 100644
--- a/sbin/pfctl/pfctl_parser.c
+++ b/sbin/pfctl/pfctl_parser.c
@@ -943,6 +943,7 @@ print_rule(struct pfctl_rule *r, const char *anchor_call, int verbose, int numer
 			printf(" dup-to");
 		printf(" ");
 		print_pool(&r->rdr, 0, 0, r->af, PF_PASS);
+		print_pool(&r->route, 0, 0, r->af, PF_PASS);
 	}
 	if (r->af) {
 		if (r->af == AF_INET)
diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index 6e9418f59aef..e50fbc96a8ba 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -787,6 +787,7 @@ struct pf_krule {
 	TAILQ_ENTRY(pf_krule)	 entries;
 	struct pf_kpool		 nat;
 	struct pf_kpool		 rdr;
+	struct pf_kpool		 route;
 
 	struct pf_counter_u64	 evaluations;
 	struct pf_counter_u64	 packets[2];
@@ -2217,7 +2218,7 @@ VNET_DECLARE(struct unrhdr64, pf_stateid);
 TAILQ_HEAD(pf_altqqueue, pf_altq);
 VNET_DECLARE(struct pf_altqqueue,	 pf_altqs[4]);
 #define	V_pf_altqs			 VNET(pf_altqs)
-VNET_DECLARE(struct pf_kpalist,		 pf_pabuf[2]);
+VNET_DECLARE(struct pf_kpalist,		 pf_pabuf[3]);
 #define	V_pf_pabuf			 VNET(pf_pabuf)
 
 VNET_DECLARE(u_int32_t,			 ticket_altqs_active);
diff --git a/sys/netpfil/pf/if_pfsync.c b/sys/netpfil/pf/if_pfsync.c
index 60bfb05d1570..98a2367b79b0 100644
--- a/sys/netpfil/pf/if_pfsync.c
+++ b/sys/netpfil/pf/if_pfsync.c
@@ -574,6 +574,12 @@ pfsync_state_import(union pfsync_state_union *sp, int flags, int msg_version)
 		 * from the local ruleset.
 		 */
 		if (r != &V_pf_default_rule) {
+			struct pf_kpool		*pool = &r->route;
+
+			/* Backwards compatibility. */
+			if (TAILQ_EMPTY(&pool->list))
+				pool = &r->rdr;
+
 			/*
 			 * The ruleset is identical, try to recover. If the rule
 			 * has a redirection pool with a single interface, there
@@ -582,7 +588,7 @@ pfsync_state_import(union pfsync_state_union *sp, int flags, int msg_version)
 			 * give up, as we can't be sure that we will pick the
 			 * same one as the pfsync peer did.
 			 */
-			rpool_first = TAILQ_FIRST(&(r->rdr.list));
+			rpool_first = TAILQ_FIRST(&(pool->list));
 			if ((rpool_first == NULL) ||
 			    (TAILQ_NEXT(rpool_first, entries) != NULL)) {
 				DPFPRINTF(PF_DEBUG_MISC,
diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c
index 83eca735d2bb..00d6583234c7 100644
--- a/sys/netpfil/pf/pf.c
+++ b/sys/netpfil/pf/pf.c
@@ -164,7 +164,7 @@ SDT_PROBE_DEFINE2(pf, purge, state, rowcount, "int", "size_t");
 
 /* state tables */
 VNET_DEFINE(struct pf_altqqueue,	 pf_altqs[4]);
-VNET_DEFINE(struct pf_kpalist,		 pf_pabuf[2]);
+VNET_DEFINE(struct pf_kpalist,		 pf_pabuf[3]);
 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_active);
 VNET_DEFINE(struct pf_altqqueue *,	 pf_altq_ifs_active);
 VNET_DEFINE(struct pf_altqqueue *,	 pf_altqs_inactive);
@@ -1267,6 +1267,7 @@ pf_initialize(void)
 	TAILQ_INIT(&V_pf_altqs[3]);
 	TAILQ_INIT(&V_pf_pabuf[0]);
 	TAILQ_INIT(&V_pf_pabuf[1]);
+	TAILQ_INIT(&V_pf_pabuf[2]);
 	V_pf_altqs_active = &V_pf_altqs[0];
 	V_pf_altq_ifs_active = &V_pf_altqs[1];
 	V_pf_altqs_inactive = &V_pf_altqs[2];
@@ -5900,6 +5901,12 @@ nextrule:
 	if (r->rt) {
 		struct pf_ksrc_node	*sn = NULL;
 		struct pf_srchash	*snh = NULL;
+		struct pf_kpool		*pool = &r->route;
+
+		/* Backwards compatibility. */
+		if (TAILQ_EMPTY(&pool->list))
+			pool = &r->rdr;
+
 		/*
 		 * Set act.rt here instead of in pf_rule_to_actions() because
 		 * it is applied only from the last pass rule.
@@ -5907,7 +5914,7 @@ nextrule:
 		pd->act.rt = r->rt;
 		/* Don't use REASON_SET, pf_map_addr increases the reason counters */
 		reason = pf_map_addr_sn(pd->af, r, pd->src, &pd->act.rt_addr,
-		    &pd->act.rt_kif, NULL, &sn, &snh, &r->rdr);
+		    &pd->act.rt_kif, NULL, &sn, &snh, pool);
 		if (reason != 0)
 			goto cleanup;
 	}
diff --git a/sys/netpfil/pf/pf.h b/sys/netpfil/pf/pf.h
index 5de85c1fe7ef..24249ead6ba2 100644
--- a/sys/netpfil/pf/pf.h
+++ b/sys/netpfil/pf/pf.h
@@ -49,7 +49,7 @@
 enum	{ PF_INOUT, PF_IN, PF_OUT };
 enum	{ PF_PASS, PF_DROP, PF_SCRUB, PF_NOSCRUB, PF_NAT, PF_NONAT,
 	  PF_BINAT, PF_NOBINAT, PF_RDR, PF_NORDR, PF_SYNPROXY_DROP, PF_DEFER,
-	  PF_MATCH, PF_AFRT };
+	  PF_MATCH, PF_AFRT, PF_RT };
 enum	{ PF_RULESET_SCRUB, PF_RULESET_FILTER, PF_RULESET_NAT,
 	  PF_RULESET_BINAT, PF_RULESET_RDR, PF_RULESET_MAX };
 enum	{ PF_OP_NONE, PF_OP_IRG, PF_OP_EQ, PF_OP_NE, PF_OP_LT,
diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c
index d206a9f8da43..340e7c25a501 100644
--- a/sys/netpfil/pf/pf_ioctl.c
+++ b/sys/netpfil/pf/pf_ioctl.c
@@ -436,7 +436,7 @@ pf_get_kpool(const char *anchor, u_int32_t ticket, u_int8_t rule_action,
 	struct pf_krule		*rule;
 	int			 rs_num;
 
-	MPASS(which == PF_RDR || which == PF_NAT);
+	MPASS(which == PF_RDR || which == PF_NAT || which == PF_RT);
 
 	ruleset = pf_find_kruleset(anchor);
 	if (ruleset == NULL)
@@ -470,10 +470,16 @@ pf_get_kpool(const char *anchor, u_int32_t ticket, u_int8_t rule_action,
 	if (rule == NULL)
 		return (NULL);
 
-	if (which == PF_NAT)
-		return (&rule->nat);
-	else
+	switch (which) {
+	case PF_RDR:
 		return (&rule->rdr);
+	case PF_NAT:
+		return (&rule->nat);
+	case PF_RT:
+		return (&rule->route);
+	default:
+		panic("Unknow pool type %d", which);
+	}
 }
 
 static void
@@ -612,6 +618,7 @@ pf_free_rule(struct pf_krule *rule)
 	pf_kanchor_remove(rule);
 	pf_empty_kpool(&rule->rdr.list);
 	pf_empty_kpool(&rule->nat.list);
+	pf_empty_kpool(&rule->route.list);
 
 	pf_krule_free(rule);
 }
@@ -1832,6 +1839,7 @@ pf_krule_alloc(void)
 	rule = malloc(sizeof(struct pf_krule), M_PFRULE, M_WAITOK | M_ZERO);
 	mtx_init(&rule->nat.mtx, "pf_krule_nat_pool", NULL, MTX_DEF);
 	mtx_init(&rule->rdr.mtx, "pf_krule_rdr_pool", NULL, MTX_DEF);
+	mtx_init(&rule->route.mtx, "pf_krule_route_pool", NULL, MTX_DEF);
 	rule->timestamp = uma_zalloc_pcpu(pf_timestamp_pcpu_zone,
 	    M_WAITOK | M_ZERO);
 	return (rule);
@@ -1871,6 +1879,7 @@ pf_krule_free(struct pf_krule *rule)
 
 	mtx_destroy(&rule->nat.mtx);
 	mtx_destroy(&rule->rdr.mtx);
+	mtx_destroy(&rule->route.mtx);
 	free(rule, M_PFRULE);
 }
 
@@ -2106,6 +2115,7 @@ pf_ioctl_addrule(struct pf_krule *rule, uint32_t ticket,
 	rule->cpid = pid;
 	TAILQ_INIT(&rule->rdr.list);
 	TAILQ_INIT(&rule->nat.list);
+	TAILQ_INIT(&rule->route.list);
 
 	PF_CONFIG_LOCK();
 	PF_RULES_WLOCK();
@@ -2203,7 +2213,7 @@ pf_ioctl_addrule(struct pf_krule *rule, uint32_t ticket,
 	    (rule->set_prio[0] > PF_PRIO_MAX ||
 	    rule->set_prio[1] > PF_PRIO_MAX))
 		error = EINVAL;
-	for (int i = 0; i < 2; i++) {
+	for (int i = 0; i < 3; i++) {
 		TAILQ_FOREACH(pa, &V_pf_pabuf[i], entries)
 			if (pa->addr.type == PF_ADDR_TABLE) {
 				pa->addr.p.tbl = pfr_attach_table(ruleset,
@@ -2225,10 +2235,12 @@ pf_ioctl_addrule(struct pf_krule *rule, uint32_t ticket,
 
 	pf_mv_kpool(&V_pf_pabuf[0], &rule->nat.list);
 	pf_mv_kpool(&V_pf_pabuf[1], &rule->rdr.list);
+	pf_mv_kpool(&V_pf_pabuf[2], &rule->route.list);
 	if (((((rule->action == PF_NAT) || (rule->action == PF_RDR) ||
 	    (rule->action == PF_BINAT)) && rule->anchor == NULL) ||
 	    (rule->rt > PF_NOPFROUTE)) &&
-	    (TAILQ_FIRST(&rule->rdr.list) == NULL))
+	    (TAILQ_FIRST(&rule->rdr.list) == NULL &&
+	     TAILQ_FIRST(&rule->route.list) == NULL))
 		error = EINVAL;
 
 	if (rule->action == PF_PASS && rule->rdr.opts & PF_POOL_STICKYADDR &&
@@ -2244,6 +2256,7 @@ pf_ioctl_addrule(struct pf_krule *rule, uint32_t ticket,
 
 	rule->nat.cur = TAILQ_FIRST(&rule->nat.list);
 	rule->rdr.cur = TAILQ_FIRST(&rule->rdr.list);
+	rule->route.cur = TAILQ_FIRST(&rule->route.list);
 	TAILQ_INSERT_TAIL(ruleset->rules[rs_num].inactive.ptr,
 	    rule, entries);
 	ruleset->rules[rs_num].inactive.rcount++;
@@ -2553,6 +2566,7 @@ pf_ioctl_begin_addrs(uint32_t *ticket)
 	PF_RULES_WLOCK();
 	pf_empty_kpool(&V_pf_pabuf[0]);
 	pf_empty_kpool(&V_pf_pabuf[1]);
+	pf_empty_kpool(&V_pf_pabuf[2]);
 	*ticket = ++V_ticket_pabuf;
 	PF_RULES_WUNLOCK();
 
@@ -2566,7 +2580,8 @@ pf_ioctl_add_addr(struct pf_nl_pooladdr *pp)
 	struct pfi_kkif		*kif = NULL;
 	int error;
 
-	if (pp->which != PF_RDR && pp->which != PF_NAT)
+	if (pp->which != PF_RDR && pp->which != PF_NAT &&
+	    pp->which != PF_RT)
 		return (EINVAL);
 
 #ifndef INET
@@ -2613,8 +2628,17 @@ pf_ioctl_add_addr(struct pf_nl_pooladdr *pp)
 		PF_RULES_WUNLOCK();
 		goto out;
 	}
-	TAILQ_INSERT_TAIL(&V_pf_pabuf[pp->which == PF_RDR ? 1 : 0],
-	    pa, entries);
+	switch (pp->which) {
+	case PF_NAT:
+		TAILQ_INSERT_TAIL(&V_pf_pabuf[0], pa, entries);
+		break;
+	case PF_RDR:
+		TAILQ_INSERT_TAIL(&V_pf_pabuf[1], pa, entries);
+		break;
+	case PF_RT:
+		TAILQ_INSERT_TAIL(&V_pf_pabuf[2], pa, entries);
+		break;
+	}
 	PF_RULES_WUNLOCK();
 
 	return (0);
@@ -2632,7 +2656,8 @@ pf_ioctl_get_addrs(struct pf_nl_pooladdr *pp)
 
 	PF_RULES_RLOCK_TRACKER;
 
-	if (pp->which != PF_RDR && pp->which != PF_NAT)
+	if (pp->which != PF_RDR && pp->which != PF_NAT &&
+	    pp->which != PF_RT)
 		return (EINVAL);
 
 	pp->anchor[sizeof(pp->anchor) - 1] = 0;
@@ -2659,7 +2684,8 @@ pf_ioctl_get_addr(struct pf_nl_pooladdr *pp)
 	struct pf_kpooladdr	*pa;
 	u_int32_t		 nr = 0;
 
-	if (pp->which != PF_RDR && pp->which != PF_NAT)
+	if (pp->which != PF_RDR && pp->which != PF_NAT &&
+	    pp->which != PF_RT)
 		return (EINVAL);
 
 	PF_RULES_RLOCK_TRACKER;
@@ -3652,6 +3678,7 @@ DIOCGETRULENV_error:
 			newrule->cpid = td->td_proc ? td->td_proc->p_pid : 0;
 			TAILQ_INIT(&newrule->nat.list);
 			TAILQ_INIT(&newrule->rdr.list);
+			TAILQ_INIT(&newrule->route.list);
 		}
 #define	ERROUT(x)	ERROUT_IOCTL(DIOCCHANGERULE_error, x)
 
@@ -3748,7 +3775,7 @@ DIOCGETRULENV_error:
 				error = ENOMEM;
 			if (pf_kanchor_setup(newrule, ruleset, pcr->anchor_call))
 				error = EINVAL;
-			for (int i = 0; i < 2; i++) {
+			for (int i = 0; i < 3; i++) {
 				TAILQ_FOREACH(pa, &V_pf_pabuf[i], entries)
 					if (pa->addr.type == PF_ADDR_TABLE) {
 						pa->addr.p.tbl =
@@ -3772,6 +3799,7 @@ DIOCGETRULENV_error:
 
 			pf_mv_kpool(&V_pf_pabuf[0], &newrule->nat.list);
 			pf_mv_kpool(&V_pf_pabuf[1], &newrule->rdr.list);
+			pf_mv_kpool(&V_pf_pabuf[2], &newrule->route.list);
 			if (((((newrule->action == PF_NAT) ||
 			    (newrule->action == PF_RDR) ||
 			    (newrule->action == PF_BINAT) ||
@@ -3792,6 +3820,7 @@ DIOCGETRULENV_error:
 		}
 		pf_empty_kpool(&V_pf_pabuf[0]);
 		pf_empty_kpool(&V_pf_pabuf[1]);
+		pf_empty_kpool(&V_pf_pabuf[2]);
 
 		if (pcr->action == PF_CHANGE_ADD_HEAD)
 			oldrule = TAILQ_FIRST(
diff --git a/sys/netpfil/pf/pf_nl.c b/sys/netpfil/pf/pf_nl.c
index 3e7a6965d387..c0f722b1fd18 100644
--- a/sys/netpfil/pf/pf_nl.c
+++ b/sys/netpfil/pf/pf_nl.c
@@ -736,6 +736,7 @@ static const struct nlattr_parser nla_p_rule[] = {
 	{ .type = PF_RT_MAX_SRC_CONN, .off = _OUT(max_src_conn), .cb = nlattr_get_uint32 },
 	{ .type = PF_RT_RPOOL_NAT, .off = _OUT(nat), .arg = &pool_parser, .cb = nlattr_get_nested },
 	{ .type = PF_RT_NAF, .off = _OUT(naf), .cb = nlattr_get_uint8 },
+	{ .type = PF_RT_RPOOL_RT, .off = _OUT(route), .arg = &pool_parser, .cb = nlattr_get_nested },
 };
 NL_DECLARE_ATTR_PARSER(rule_parser, nla_p_rule);
 #undef _OUT
@@ -909,6 +910,7 @@ pf_handle_getrule(struct nlmsghdr *hdr, struct nl_pstate *npt)
 	nlattr_add_string(nw, PF_RT_OVERLOAD_TBLNAME, rule->overload_tblname);
 	nlattr_add_pool(nw, PF_RT_RPOOL_RDR, &rule->rdr);
 	nlattr_add_pool(nw, PF_RT_RPOOL_NAT, &rule->nat);
+	nlattr_add_pool(nw, PF_RT_RPOOL_RT, &rule->route);
 	nlattr_add_u32(nw, PF_RT_OS_FINGERPRINT, rule->os_fingerprint);
 	nlattr_add_u32(nw, PF_RT_RTABLEID, rule->rtableid);
 	nlattr_add_timeout(nw, PF_RT_TIMEOUT, rule->timeout);
diff --git a/sys/netpfil/pf/pf_nl.h b/sys/netpfil/pf/pf_nl.h
index 8c0149891773..d749ef3ab99e 100644
--- a/sys/netpfil/pf/pf_nl.h
+++ b/sys/netpfil/pf/pf_nl.h
@@ -267,6 +267,7 @@ enum pf_rule_type_t {
 	PF_RT_MAX_SRC_CONN	= 74, /* u32 */
 	PF_RT_RPOOL_NAT		= 75, /* nested, pf_rpool_type_t */
 	PF_RT_NAF		= 76, /* u8 */
+	PF_RT_RPOOL_RT		= 77, /* nested, pf_rpool_type_t */
 };
 
 enum pf_addrule_type_t {