svn commit: r258645 - projects/sendfile/sys/kern

[Gleb Smirnoff]

Author: glebius
Date: Tue Nov 26 12:00:56 2013
New Revision: 258645
URL: http://svnweb.freebsd.org/changeset/base/258645

Log:
  Remove two mbuf chain traversals from sendfile(). The first one is
  a short traversal of of headers chain done in m_last(), and the second
  one is m_length() done right before the socket write.
  
  To achieve this, store header mbuf in *mh and its tail in *mhtail.
  Stack the *mh and *m out of the inner loop, before the socket write.
  
  While here, simplify the headers handling block of code.
  
  Sponsored by:	Netflix
  Sponsored by:	Nginx, Inc.

Modified:
  projects/sendfile/sys/kern/uipc_syscalls.c

Modified: projects/sendfile/sys/kern/uipc_syscalls.c
==============================================================================
--- projects/sendfile/sys/kern/uipc_syscalls.c	Tue Nov 26 11:13:24 2013	(r258644)
+++ projects/sendfile/sys/kern/uipc_syscalls.c	Tue Nov 26 12:00:56 2013	(r258645)
@@ -2206,7 +2206,7 @@ vn_sendfile(struct file *fp, int sockfd,
 	struct vnode *vp;
 	struct vm_object *obj;
 	struct socket *so;
-	struct mbuf *m;
+	struct mbuf *m, *mh, *mhtail;
 	struct sf_buf *sf;
 	struct vm_page *pg;
 	struct shmfd *shmfd;
@@ -2222,7 +2222,7 @@ vn_sendfile(struct file *fp, int sockfd,
 	m = NULL;
 	sfs = NULL;
 	fsbytes = sbytes = 0;
-	hdrlen = mnw = 0;
+	mnw = 0;
 	rem = nbytes;
 	obj_size = 0;
 	inflight_called = false;
@@ -2258,29 +2258,29 @@ vn_sendfile(struct file *fp, int sockfd,
 #endif
 
 	/* If headers are specified copy them into mbufs. */
-	if (hdr_uio != NULL) {
+	if (hdr_uio != NULL && hdr_uio->uio_resid > 0) {
 		hdr_uio->uio_td = td;
 		hdr_uio->uio_rw = UIO_WRITE;
-		if (hdr_uio->uio_resid > 0) {
-			/*
-			 * In FBSD < 5.0 the nbytes to send also included
-			 * the header.  If compat is specified subtract the
-			 * header size from nbytes.
-			 */
-			if (kflags & SFK_COMPAT) {
-				if (nbytes > hdr_uio->uio_resid)
-					nbytes -= hdr_uio->uio_resid;
-				else
-					nbytes = 0;
-			}
-			m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK),
-			    0, 0, 0);
-			if (m == NULL) {
-				error = mnw ? EAGAIN : ENOBUFS;
-				goto out;
-			}
-			hdrlen = m_length(m, NULL);
+		/*
+		 * In FBSD < 5.0 the nbytes to send also included
+		 * the header.  If compat is specified subtract the
+		 * header size from nbytes.
+		 */
+		if (kflags & SFK_COMPAT) {
+			if (nbytes > hdr_uio->uio_resid)
+				nbytes -= hdr_uio->uio_resid;
+			else
+				nbytes = 0;
+		}
+		mh = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK), 0, 0, 0);
+		if (mh == NULL) {
+			error = mnw ? EAGAIN : ENOBUFS;
+			goto out;
 		}
+		hdrlen = m_length(mh, &mhtail);
+	} else {
+		mh = NULL;
+		hdrlen = 0;
 	}
 
 	/*
@@ -2485,8 +2485,6 @@ retry_space:
 			/* Append to mbuf chain. */
 			if (mtail != NULL)
 				mtail->m_next = m0;
-			else if (m != NULL)
-				m_last(m)->m_next = m0;
 			else
 				m = m0;
 			mtail = m0;
@@ -2505,11 +2503,20 @@ retry_space:
 		if (vp != NULL)
 			VOP_UNLOCK(vp, 0);
 
+		/* Prepend header, if any. */
+		if (hdrlen) {
+			mhtail->m_next = m;
+			m = mh;
+		}
+
 		/* Add the buffer chain to the socket buffer. */
-		if (m != NULL) {
-			int mlen, err;
+		if (loopbytes + hdrlen > 0) {
+			int err;
+
+			KASSERT(m_length(m, NULL) == loopbytes + hdrlen,
+			    ("%s: mlen %u loop %d hdr %d", __func__,
+			    m_length(m, NULL), loopbytes, hdrlen));
 
-			mlen = m_length(m, NULL);
 			SOCKBUF_LOCK(&so->so_snd);
 			if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
 				error = EPIPE;
@@ -2532,12 +2539,10 @@ retry_space:
 				 * - fsbytes contains the total amount
 				 *   of bytes sent from the file.
 				 */
-				sbytes += mlen;
-				fsbytes += mlen;
-				if (hdrlen) {
-					fsbytes -= hdrlen;
+				sbytes += loopbytes + hdrlen;
+				fsbytes += loopbytes;
+				if (hdrlen)
 					hdrlen = 0;
-				}
 			} else if (error == 0)
 				error = err;
 			m = NULL;	/* pru_send always consumes */