rule not responding to incoming packets
Mike M
mail at miketm.com
Wed Jun 8 08:23:07 UTC 2011
Hi,
I have an issue with pf where incoming packets matching a particular
rule, are not being responded to, resulting in public users being
unable to access a web server. I'm receiving a SYN flood on 80/TCP
(currently ~50mbit @ 100kpps) so am trying to implement some
protection on the box. I don't believe the current DDoS is actually
causing this issue though, as packets matching the <whitelist> and
<staff> tables, can establish connections without a problem.
Other packets matching this other rule however, seem to be unable to
establish a connection. I see the SYN packets in via tcpdump, but
they are not returned. Eventually, the connection closes.
Another thing I've noticed is that the src limits seem to have an
effect (state table is typically 4k-7k entries), as without this in
place, the state table fills rapidly, rendering the box near unusable.
Using 'synproxy state' also seems to have a similar effect. I have
never observed any IP addresses within the <attacksource> table (via
'pfctl -T show -t attacksource')
System is FreeBSD 8.2-RELEASE, 1GB RAM, Intel P4 3GHz (2 x CPU w/SMP)
Have provided some sanitized information below -- any assistance would
be much appreciated.... I'm pulling my hair out. Any other DDoS
hardening advice based on below is also very welcome :>
Please advise if more information is required.
Cheers,
- Mike
[root at sb ~]# more /etc/pf.conf.conf
# --- firewall
#
# ---- interfaces
if_pub = "em0"
if_priv = "em1"
# -- loopback
if_loop = "lo0"
# ---- hosts
# -- public interface
h_pub = "10.0.1.1"
# -- external hosts
# TBA
# ---- tables
table <attacksource> persist
table <blacklist> persist file "/etc/pf/blacklist.pf"
table <whitelist> persist file "/etc/pf/whitelist.pf"
table <staff> persist file "/etc/pf/staff.pf"
# ---- set policies
# -- rule optimization
set optimization aggressive
#set optimization normal
# -- adaptive timeouts
set timeout { tcp.first 20, adaptive.start 30000, adaptive.end 1800000 }
# -- set max states
set limit states 1800000
# -- statistics logging
set loginterface $if_pub
# -- don't filter on interface lo
set skip on lo
# -- normalization
scrub in
# ---- filter rules
# -- block to start
block in
# -- disallow basic spoof
antispoof quick for { lo }
# -- whitelist
pass quick from <whitelist>
# -- blacklists
block quick from <blacklist>
block quick from <attacksource>
# -- block juno flood traffic
block in quick proto tcp from any port { 1024, 3072 } to any
# -- block UDP floods
block in quick proto udp from any to $h_pub
# -- HTTP public
pass in proto tcp from any to $h_pub port 80 flags S/SA keep state
(max-src-conn 100, max-src-conn-rate 20/5, overload <attacksource>
flush global)
# -- HTTP staff
pass in proto tcp from <staff> to any port 80
# ---- allow all outbound
pass out keep state
- EOF -
PACKET CAPTURE ON WEB SERVER (10.0.1.1), WATCHING INCOMING PACKETS
FROM A REMOTE TEST HOST (10.0.2.2)
tcpdump: listening on em0, link-type EN10MB (Ethernet), capture size 96 bytes
00:00:00.000000 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 74: (tos 0x0, ttl 54, id 7809, offset 0, flags [DF],
proto TCP (6), length 60)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x3e61 (correct),
seq 2488345924, win 65535, options [mss 1460,nop,wscale 3,sackOK,TS
val 181704348 ecr 0], length 0
00:00:02.995155 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 74: (tos 0x0, ttl 54, id 7823, offset 0, flags [DF],
proto TCP (6), length 60)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x3d35 (correct),
seq 2488345924, win 65535, options [mss 1460,nop,wscale 3,sackOK,TS
val 181704648 ecr 0], length 0
00:00:03.198923 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 74: (tos 0x0, ttl 54, id 7826, offset 0, flags [DF],
proto TCP (6), length 60)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x3bf5 (correct),
seq 2488345924, win 65535, options [mss 1460,nop,wscale 3,sackOK,TS
val 181704968 ecr 0], length 0
00:00:03.199068 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 54, id 7828, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
00:00:03.198994 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 54, id 7830, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
00:00:03.198967 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 54, id 7833, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
00:00:06.198124 b0:c6:9a:df:0b:80 > 00:30:48:73:16:60, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 54, id 7835, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
PACKET CAPTURE ON REMOTE TEST HOST (10.0.2.2), WATCHING OUTGOING
PACKETS TO WEB SERVER (10.0.1.1)
tcpdump: listening on em0, link-type EN10MB (Ethernet), capture size 96 bytes
00:00:00.000000 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 74: (tos 0x0, ttl 64, id 7809, offset 0, flags [DF],
proto TCP (6), length 60)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x3e61 (correct),
seq 2488345924, win 65535, options [mss 1460,nop,wscale 3,sackOK,TS
val 181704348 ecr 0], length 0
00:00:02.995160 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 74: (tos 0x0, ttl 64, id 7823, offset 0, flags [DF],
proto TCP (6), length 60)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x3d35 (correct),
seq 2488345924, win 65535, options [mss 1460,nop,wscale 3,sackOK,TS
val 181704648 ecr 0], length 0
00:00:03.198889 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 74: (tos 0x0, ttl 64, id 7826, offset 0, flags [DF],
proto TCP (6), length 60)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x3bf5 (correct),
seq 2488345924, win 65535, options [mss 1460,nop,wscale 3,sackOK,TS
val 181704968 ecr 0], length 0
00:00:03.198865 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 64, id 7828, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
00:00:03.198888 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 64, id 7830, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
00:00:03.198860 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 64, id 7833, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
00:00:06.197917 00:50:56:b8:3c:dd > 00:0c:db:e8:8d:00, ethertype IPv4
(0x0800), length 62: (tos 0x0, ttl 64, id 7835, offset 0, flags [DF],
proto TCP (6), length 48)
10.0.2.2.21254 > 10.0.1.1.80: Flags [S], cksum 0x1bee (correct),
seq 2488345924, win 65535, options [mss 1460,sackOK,eol], length 0
-
[root at sb ~]# pfctl -s info
Status: Enabled for 0 days 00:06:59 Debug: Urgent
Interface Stats for em0 IPv4 IPv6
Bytes In 1975306344 0
Bytes Out 39548 0
Packets In
Passed 121174 0
Blocked 41031088 0
Packets Out
Passed 346 0
Blocked 0 0
State Table Total Rate
current entries 6821
searches 41152607 98216.2/s
inserts 120838 288.4/s
removals 114017 272.1/s
Counters
match 41151925 98214.6/s
bad-offset 0 0.0/s
fragment 0 0.0/s
short 0 0.0/s
normalize 0 0.0/s
memory 0 0.0/s
bad-timestamp 0 0.0/s
congestion 0 0.0/s
ip-option 0 0.0/s
proto-cksum 0 0.0/s
state-mismatch 0 0.0/s
state-insert 0 0.0/s
state-limit 0 0.0/s
src-limit 17662863 42154.8/s
synproxy 0 0.0/s
[root at sb ~]## pfctl -s memory
states hard limit 1800000
src-nodes hard limit 10000
frags hard limit 5000
tables hard limit 1000
table-entries hard limit 200000
[root at sb ~]# vmstat -z
ITEM SIZE LIMIT USED FREE REQUESTS FAILURES
UMA Kegs: 128, 0, 104, 16, 104, 0
UMA Zones: 888, 0, 104, 0, 104, 0
UMA Slabs: 284, 0, 521, 11, 1044, 0
UMA RCntSlabs: 544, 0, 269, 4, 269, 0
UMA Hash: 128, 0, 4, 26, 4, 0
16 Bucket: 76, 0, 53, 47, 72, 0
32 Bucket: 140, 0, 49, 7, 71, 0
64 Bucket: 268, 0, 49, 7, 96, 13
128 Bucket: 524, 0, 107, 5, 19206423, 112
VM OBJECT: 136, 0, 1235, 128, 21092, 0
MAP: 140, 0, 7, 21, 7, 0
KMAP ENTRY: 72, 109657, 36, 229, 4128, 0
MAP ENTRY: 72, 0, 656, 245, 39175, 0
DP fakepg: 72, 0, 0, 0, 0, 0
SG fakepg: 72, 0, 0, 0, 0, 0
mt_zone: 2056, 0, 261, 0, 261, 0
16: 16, 0, 2777, 471, 33486, 0
32: 32, 0, 2202, 284, 34835, 0
64: 64, 0, 4397, 323, 44761313, 0
128: 128, 0, 2253, 117, 7923, 0
256: 256, 0, 579, 51, 3860, 0
512: 512, 0, 58, 30, 1022, 0
1024: 1024, 0, 39, 141, 4871, 0
2048: 2048, 0, 357, 29, 596, 0
4096: 4096, 0, 132, 32, 5847, 0
Files: 56, 0, 88, 314, 8532, 0
TURNSTILE: 72, 0, 141, 39, 141, 0
umtx pi: 52, 0, 0, 0, 0, 0
MAC labels: 20, 0, 0, 0, 0, 0
PROC: 680, 0, 52, 32, 1603, 0
THREAD: 720, 0, 126, 14, 126, 0
SLEEPQUEUE: 44, 0, 141, 95, 141, 0
VMSPACE: 232, 0, 29, 39, 1581, 0
cpuset: 40, 0, 2, 182, 2, 0
audit_record: 816, 0, 0, 0, 0, 0
mbuf_packet: 256, 0, 257, 255, 44749548, 0
mbuf: 256, 0, 3, 265, 1175, 0
mbuf_cluster: 2048, 128000, 512, 16, 512, 0
mbuf_jumbo_page: 4096, 12800, 0, 5, 3, 0
mbuf_jumbo_9k: 9216, 6400, 0, 0, 0, 0
mbuf_jumbo_16k: 16384, 3200, 0, 0, 0, 0
mbuf_ext_refcnt: 4, 0, 0, 0, 0, 0
g_bio: 140, 0, 0, 336, 8357, 0
ttyinq: 152, 0, 150, 84, 330, 0
ttyoutq: 256, 0, 80, 10, 176, 0
ata_request: 204, 0, 0, 114, 2111, 0
ata_composite: 180, 0, 0, 0, 0, 0
VNODE: 268, 0, 759, 25, 785, 0
VNODEPOLL: 60, 0, 0, 0, 0, 0
S VFS Cache: 72, 0, 761, 87, 7689, 0
L VFS Cache: 292, 0, 0, 0, 0, 0
NAMEI: 1024, 0, 0, 48, 24697, 0
NFSMOUNT: 528, 0, 0, 0, 0, 0
NFSNODE: 484, 0, 0, 0, 0, 0
DIRHASH: 1024, 0, 39, 9, 39, 0
pipe: 392, 0, 4, 36, 1138, 0
ksiginfo: 80, 0, 69, 987, 109, 0
itimer: 220, 0, 0, 0, 0, 0
KNOTE: 72, 0, 0, 159, 14, 0
socket: 412, 204804, 24, 30, 351, 0
ipq: 32, 4068, 0, 0, 0, 0
udp_inpcb: 220, 204804, 3, 51, 295, 0
udpcb: 8, 204827, 3, 403, 295, 0
tcp_inpcb: 220, 204804, 7, 47, 15, 0
tcpcb: 632, 204804, 7, 11, 15, 0
tcptw: 52, 31824, 0, 0, 0, 0
syncache: 112, 15365, 0, 105, 8, 0
hostcache: 76, 15400, 1, 99, 1, 0
tcpreass: 20, 8112, 0, 0, 0, 0
sackhole: 20, 0, 0, 0, 0, 0
sctp_ep: 864, 65536, 0, 0, 0, 0
sctp_asoc: 1488, 40000, 0, 0, 0, 0
sctp_laddr: 24, 80040, 0, 145, 2, 0
sctp_raddr: 420, 80001, 0, 0, 0, 0
sctp_chunk: 92, 400008, 0, 0, 0, 0
sctp_readq: 76, 400000, 0, 0, 0, 0
sctp_stream_msg_out: 64, 400020, 0, 0, 0, 0
sctp_asconf: 24, 400055, 0, 0, 0, 0
sctp_asconf_ack: 24, 400055, 0, 0, 0, 0
ripcb: 220, 204804, 0, 0, 0, 0
unpcb: 172, 204815, 14, 55, 40, 0
rtentry: 108, 0, 6, 66, 6, 0
pfsrctrpl: 124, 10013, 10013, 0, 130907, 19205233
pfrulepl: 828, 0, 15, 5, 15, 0
pfstatepl: 284, 1800008, 6603, 3519, 131757, 0
pfaltqpl: 224, 0, 0, 0, 0, 0
pfpooladdrpl: 68, 0, 0, 0, 0, 0
pfrktable: 1240, 1002, 5, 10, 10, 0
pfrkentry: 156, 200000, 15, 35, 15, 0
pfrkentry2: 156, 0, 0, 0, 0, 0
pffrent: 16, 5075, 0, 203, 1, 0
pffrag: 48, 0, 0, 156, 1, 0
pffrcache: 48, 10062, 0, 0, 0, 0
pffrcent: 12, 50141, 0, 0, 0, 0
pfstatescrub: 28, 0, 0, 0, 0, 0
pfiaddrpl: 100, 0, 0, 0, 0, 0
pfospfen: 108, 0, 696, 24, 696, 0
pfosfp: 28, 0, 407, 228, 407, 0
selfd: 28, 0, 45, 336, 4495, 0
ip4flow: 40, 50232, 2, 274, 13, 0
ip6flow: 64, 50228, 0, 0, 0, 0
SWAPMETA: 276, 121576, 0, 0, 0, 0
Mountpoints: 644, 0, 3, 9, 3, 0
FFS inode: 116, 0, 729, 63, 754, 0
FFS1 dinode: 128, 0, 0, 0, 0, 0
FFS2 dinode: 256, 0, 729, 36, 754, 0
[root at sb ~]# netstat -m
262/518/780 mbufs in use (current/cache/total)
258/270/528/128000 mbuf clusters in use (current/cache/total/max)
258/254 mbuf+clusters out of packet secondary zone in use (current/cache)
0/5/5/12800 4k (page size) jumbo clusters in use (current/cache/total/max)
0/0/0/6400 9k jumbo clusters in use (current/cache/total/max)
0/0/0/3200 16k jumbo clusters in use (current/cache/total/max)
581K/689K/1271K bytes allocated to network (current/cache/total)
0/0/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters)
0/0/0 requests for jumbo clusters denied (4k/9k/16k)
0/4/6656 sfbufs in use (current/peak/max)
0 requests for sfbufs denied
0 requests for sfbufs delayed
0 requests for I/O initiated by sendfile
0 calls to protocol drain routines
[root at sb ~]# vmstat -i
interrupt total rate
irq4: uart0 530 0
irq6: fdc0 6 0
irq14: ata0 2174 3
irq15: ata1 35 0
cpu0: timer 1150587 1994
cpu1: timer 1151248 1995
Total 2304580 3994
[root at sb ~]#
[root at sb ~]# more /etc/sysctl.conf
# -- forward for routing
net.inet.ip.forwarding=1
# -- security & attack assistance
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=1
net.inet.tcp.drop_synfin=1
# -- allow more memory allocation
kern.ipc.nmbclusters=128000
# -- increase max connections, for DDoS
kern.ipc.somaxconn=65000
# -- icmp may not RST
# -- useful for spoofed icmp/udp floods
net.inet.tcp.icmp_may_rst=0
# -- max files allowed for in kernel
kern.maxfiles=65536
kern.maxfilesperproc=32768
# -- decrease receive buffer to decrease liklihood of buffer overflow
during DDoS
#sysctl net.inet.tcp.recvspace=4096
# -- less stringent:
#sysctl net.inet.tcp.recvspace=8192
# -- increase range of outgoing ports
net.inet.ip.portrange.first=2000
# -- use ports in natural order
net.inet.ip.portrange.randomized=0
# -- don't create TIME_WAIT for localhost connections
net.inet.tcp.nolocaltimewait=1
# -- open sockets
kern.ipc.maxsockets=204800
kern.ipc.maxsockbuf=16777216
# -- manipulate TCP keepalive
# 10000 + (5000 x 8) = 50000 msec (50 sec)
#net.inet.tcp.keepidle=10000
#net.inet.tcp.keepintvl=5000
# -- maximum segment life
# -- how long to ait for SYN-ACK response (ACK) before closing
# 5 secs
net.inet.tcp.msl=5000
# -- limit ICMP replies to 50 p/sec
net.inet.icmp.icmplim=50
# -- polling tuning
kern.polling.idle_poll=1
kern.polling.reg_frac=20
kern.polling.user_frac=40
kern.polling.each_burst=20
kern.polling.burst_max=500
# -- use syncookies to reduce memory allocation during handshake
net.inet.tcp.syncookies_only=1
# -- needed for pgsql
kern.ipc.shm_use_phys=1
kern.ipc.shmall=32768
kern.ipc.shmmax=134217728
kern.ipc.semmap=256
[root at sb ~]#
[root at sb ~]# sysctl -a | grep tcp
net.inet.tcp.rfc1323: 1
net.inet.tcp.mssdflt: 512
net.inet.tcp.keepidle: 7200000
net.inet.tcp.keepintvl: 75000
net.inet.tcp.sendspace: 32768
net.inet.tcp.recvspace: 65536
net.inet.tcp.keepinit: 75000
net.inet.tcp.delacktime: 100
net.inet.tcp.hostcache.purge: 0
net.inet.tcp.hostcache.prune: 300
net.inet.tcp.hostcache.expire: 3600
net.inet.tcp.hostcache.count: 1
net.inet.tcp.hostcache.bucketlimit: 30
net.inet.tcp.hostcache.hashsize: 512
net.inet.tcp.hostcache.cachelimit: 15360
net.inet.tcp.read_locking: 1
net.inet.tcp.recvbuf_max: 262144
net.inet.tcp.recvbuf_inc: 16384
net.inet.tcp.recvbuf_auto: 1
net.inet.tcp.insecure_rst: 0
net.inet.tcp.ecn.maxretries: 1
net.inet.tcp.ecn.enable: 0
net.inet.tcp.abc_l_var: 2
net.inet.tcp.rfc3465: 1
net.inet.tcp.rfc3390: 1
net.inet.tcp.rfc3042: 1
net.inet.tcp.drop_synfin: 1
net.inet.tcp.delayed_ack: 1
net.inet.tcp.blackhole: 2
net.inet.tcp.log_in_vain: 0
net.inet.tcp.sendbuf_max: 262144
net.inet.tcp.sendbuf_inc: 8192
net.inet.tcp.sendbuf_auto: 1
net.inet.tcp.tso: 1
net.inet.tcp.newreno: 1
net.inet.tcp.local_slowstart_flightsize: 4
net.inet.tcp.slowstart_flightsize: 1
net.inet.tcp.path_mtu_discovery: 1
net.inet.tcp.reass.overflows: 0
net.inet.tcp.reass.cursegments: 0
net.inet.tcp.reass.maxsegments: 8112
net.inet.tcp.sack.globalholes: 0
net.inet.tcp.sack.globalmaxholes: 65536
net.inet.tcp.sack.maxholes: 128
net.inet.tcp.sack.enable: 1
net.inet.tcp.inflight.stab: 20
net.inet.tcp.inflight.max: 1073725440
net.inet.tcp.inflight.min: 6144
net.inet.tcp.inflight.rttthresh: 10
net.inet.tcp.inflight.debug: 0
net.inet.tcp.inflight.enable: 0
net.inet.tcp.isn_reseed_interval: 0
net.inet.tcp.icmp_may_rst: 0
net.inet.tcp.pcbcount: 7
net.inet.tcp.do_tcpdrain: 1
net.inet.tcp.tcbhashsize: 512
net.inet.tcp.log_debug: 0
net.inet.tcp.minmss: 216
net.inet.tcp.syncache.rst_on_sock_fail: 1
net.inet.tcp.syncache.rexmtlimit: 3
net.inet.tcp.syncache.hashsize: 512
net.inet.tcp.syncache.count: 0
net.inet.tcp.syncache.cachelimit: 15360
net.inet.tcp.syncache.bucketlimit: 30
net.inet.tcp.syncookies_only: 1
net.inet.tcp.syncookies: 1
net.inet.tcp.timer_race: 0
net.inet.tcp.finwait2_timeout: 60000
net.inet.tcp.fast_finwait2_recycle: 0
net.inet.tcp.always_keepalive: 1
net.inet.tcp.rexmit_slop: 200
net.inet.tcp.rexmit_min: 30
net.inet.tcp.msl: 5000
net.inet.tcp.nolocaltimewait: 1
net.inet.tcp.maxtcptw: 31767
net.inet.flowtable.tcp_expire: 86400
[root at sb ~]#
More information about the freebsd-pf
mailing list