svn commit: r189374 - user/kmacy/HEAD_fast_net_merge/sys/netinet

[Kip Macy]

Author: kmacy
Date: Wed Mar  4 21:22:39 2009
New Revision: 189374
URL: http://svn.freebsd.org/changeset/base/189374

Log:
  Use per-cpu callouts for tcp_timer
  186694, 187660
  
  186694:
  - convert tcp_timer_activate over to using
    per-cpu callouts
  - don't acquire the tcbinfo lock exclusively
    in tcp_timer_rexmt unless needed for tcp_drop
  
  187660:
  - mp_maxid may not be valid ensure that we
    re-schedule on cpuid less than or equal to
    the current one for tcp callouts

Modified:
  user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c

Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c
==============================================================================
--- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c	Wed Mar  4 21:04:52 2009	(r189373)
+++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c	Wed Mar  4 21:22:39 2009	(r189374)
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/protosw.h>
+#include <sys/smp.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
@@ -118,6 +119,8 @@ int	tcp_maxpersistidle;
 	/* max idle time in persist */
 int	tcp_maxidle;
 
+#define	INP_CPU(inp)	min(curcpu, ((inp)->inp_flowid % mp_maxid))
+
 /*
  * Tcp protocol timeout routine called every 500 ms.
  * Updates timestamps used for TCP
@@ -255,8 +258,8 @@ tcp_timer_2msl(void *xtp)
 	} else {
 		if (tp->t_state != TCPS_TIME_WAIT &&
 		   (ticks - tp->t_rcvtime) <= tcp_maxidle)
-		       callout_reset(&tp->t_timers->tt_2msl, tcp_keepintvl,
-				     tcp_timer_2msl, tp);
+		       callout_reset_on(&tp->t_timers->tt_2msl, tcp_keepintvl,
+			   tcp_timer_2msl, tp, INP_CPU(inp));
 	       else
 		       tp = tcp_close(tp);
        }
@@ -340,9 +343,9 @@ tcp_timer_keep(void *xtp)
 				    tp->rcv_nxt, tp->snd_una - 1, 0);
 			free(t_template, M_TEMP);
 		}
-		callout_reset(&tp->t_timers->tt_keep, tcp_keepintvl, tcp_timer_keep, tp);
+		callout_reset_on(&tp->t_timers->tt_keep, tcp_keepintvl, tcp_timer_keep, tp, INP_CPU(inp));
 	} else
-		callout_reset(&tp->t_timers->tt_keep, tcp_keepidle, tcp_timer_keep, tp);
+		callout_reset_on(&tp->t_timers->tt_keep, tcp_keepidle, tcp_timer_keep, tp, INP_CPU(inp));
 
 #ifdef TCPDEBUG
 	if (inp->inp_socket->so_options & SO_DEBUG)
@@ -447,15 +450,13 @@ tcp_timer_rexmt(void * xtp)
 	CURVNET_SET(tp->t_vnet);
 	INIT_VNET_INET(tp->t_vnet);
 	int rexmt;
-	int headlocked;
 	struct inpcb *inp;
 #ifdef TCPDEBUG
 	int ostate;
 
 	ostate = tp->t_state;
 #endif
-	INP_INFO_WLOCK(&V_tcbinfo);
-	headlocked = 1;
+	INP_INFO_RLOCK(&V_tcbinfo);
 	inp = tp->t_inpcb;
 	/*
 	 * XXXRW: While this assert is in fact correct, bugs in the tcpcb
@@ -466,15 +467,16 @@ tcp_timer_rexmt(void * xtp)
 	 */
 	if (inp == NULL) {
 		tcp_timer_race++;
-		INP_INFO_WUNLOCK(&V_tcbinfo);
+		INP_INFO_RUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 		return;
 	}
 	INP_WLOCK(inp);
-	if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_rexmt)
+	if ((inp->inp_vflag & INP_DROPPED)
+	    || callout_pending(&tp->t_timers->tt_rexmt)
 	    || !callout_active(&tp->t_timers->tt_rexmt)) {
+		INP_INFO_RUNLOCK(&V_tcbinfo);
 		INP_WUNLOCK(inp);
-		INP_INFO_WUNLOCK(&V_tcbinfo);
 		CURVNET_RESTORE();
 		return;
 	}
@@ -488,12 +490,22 @@ tcp_timer_rexmt(void * xtp)
 	if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) {
 		tp->t_rxtshift = TCP_MAXRXTSHIFT;
 		V_tcpstat.tcps_timeoutdrop++;
+		in_pcbref(inp);
+		INP_INFO_RUNLOCK(&V_tcbinfo);
+		INP_WUNLOCK(inp);
+		INP_INFO_WLOCK(&V_tcbinfo);
+		INP_WLOCK(inp);
+		if (in_pcbrele(inp)) {
+			INP_INFO_WUNLOCK(&V_tcbinfo);
+			CURVNET_RESTORE();
+			return;
+		}
 		tp = tcp_drop(tp, tp->t_softerror ?
 			      tp->t_softerror : ETIMEDOUT);
+		INP_INFO_WUNLOCK(&V_tcbinfo);
 		goto out;
 	}
-	INP_INFO_WUNLOCK(&V_tcbinfo);
-	headlocked = 0;
+	INP_INFO_RUNLOCK(&V_tcbinfo);
 	if (tp->t_rxtshift == 1) {
 		/*
 		 * first retransmit; record ssthresh and cwnd so they can
@@ -598,8 +610,6 @@ out:
 #endif
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
-	if (headlocked)
-		INP_INFO_WUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
 }
 
@@ -608,6 +618,8 @@ tcp_timer_activate(struct tcpcb *tp, int
 {
 	struct callout *t_callout;
 	void *f_callout;
+	struct inpcb *inp = tp->t_inpcb;
+	int cpu = INP_CPU(inp);
 
 	switch (timer_type) {
 		case TT_DELACK:
@@ -636,7 +648,7 @@ tcp_timer_activate(struct tcpcb *tp, int
 	if (delta == 0) {
 		callout_stop(t_callout);
 	} else {
-		callout_reset(t_callout, delta, f_callout, tp);
+		callout_reset_on(t_callout, delta, f_callout, tp, cpu);
 	}
 }