recv() with MSG_WAITALL appears to be broken (sometimes)

[Lewis Donzis]

I posted this on bugzilla (https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=212716) but thought I’d see if anyone here can offer some advice.

We have a simple test program to illustrate the problem.

Run "server" on a machine, and then run "client" on the same machine.

Server creates a listening socket and loops accepting connections, sending a little data, closing the socket, and then back to the top of the loop.

Client creates a socket, connects to the server, does a recv() with MSG_WAITALL, closes the socket, and loops.

This runs anywhere from one to a few dozen times and then hangs.  The server socket is in FIN_WAIT_2, and the client socket is in CLOSE_WAIT.  So the client side is waiting for the application to close the socket, but it's still stuck in the recv(), never being awakened by the EOF from the server closing the socket.

The same code runs on Linux and QNX without any problem.

This seems ridiculously simple and seems like it would affect many programs, so perhaps we’re just overlooking something.  

Note that it makes no difference whether it's run in the same machine (to localhost or to the machine's own IP address) or across the network, or whether it's IPv4 or IPv6.  It’s also the same with a UNIX domain STREAM socket (but not a SEQPACKET socket), so it appears to be unrelated to TCP and perhaps more of a general problem in the socket layer.  In all cases, the recv() never unblocks when the peer closes the socket.

Thanks,
lew


client.c:

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <err.h>

int main (int argc, char *argv[])
{
   int c;
   bool verbose = false;
   char *addr = "127.0.0.1";

   signal(SIGPIPE, SIG_IGN);
   while ((c = getopt(argc, argv, "v")) != -1) {
      switch (c) {
         case 'v': ++verbose;    break;
         default:  return 1;     break;
         }
      }
   argc -= optind - 1;
   argv += optind - 1;

   if (argc > 1) addr = argv[1];

   for (int try = 1;; try++) {
      printf("Try %d\n", try);
      int s;
      if ((s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
         err(1, "socket");
   
      struct sockaddr_in sa = { .sin_family = AF_INET, .sin_port = htons(79), .sin_addr.s_addr = inet_addr(addr) };
      if (connect(s, (struct sockaddr *)&sa, sizeof sa) < 0)
         err(1, "connect");

      // get the response
      char rbuf [4096];
      int rcvLen;
      if ((rcvLen = recv(s, rbuf, sizeof rbuf, MSG_WAITALL)) < 0)
         err(1, "recv");
      if (verbose) printf("Received: '%.*s'\n", rcvLen, rbuf);
      close(s);
      }

   return 0;
   }


server.c:

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <ctype.h>
#include <signal.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <err.h>

int main (int argc, char *argv[])
{
   int c;
   bool verbose = false;

   signal(SIGPIPE, SIG_IGN);
   while ((c = getopt(argc, argv, "v")) != -1) {
      switch (c) {
         case 'v': ++verbose;    break;
         default:  return 1;     break;
         }
      }
   argc -= optind - 1;
   argv += optind - 1;

   int s0;
   if ((s0 = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)
      err(1, "socket");
   int on = true;
   if (setsockopt(s0, SOL_SOCKET, SO_REUSEPORT, &on, sizeof on) < 0)
      err(1, "setsockopt");
   struct sockaddr_in sa = { .sin_family = AF_INET, .sin_port = htons(79) };

   if (bind(s0, (struct sockaddr *)&sa, sizeof sa) < 0)
      err(1, "bind");
   if (listen(s0, 1000) < 0)
      err(1, "listen");

   printf("Listening\n");

   for (int try = 1;; try++) {
      socklen_t sl;
      int s = accept(s0, (struct sockaddr *)&sa, &sl);
      if (s < 0) err(1, "accept");
      printf("Try %d\n", try);

      // send the response
      static char sbuf [] = "Hello from server\n";
      if (send(s, sbuf, sizeof sbuf - 1, 0) < 0)
         err(1, "send");
      if (verbose) printf("Sent:     %s", sbuf);
      close(s);
      }

   return 0;
   }