What could be the reason for reading sockets very slowly?

I use TCP socket lock for my client and server. Whenever I read, I first check to see if data in the stream is available using select. I always read and write 40 bytes at a time. While most views take a few milliseconds or less, some just take more than half a second. This is after I know that there is data available on the socket.

I also use TCP_NODELAY

What could be the reason for this?

EDIT 2

I analyzed the timestamp for each packet sent and received, and saw that this delay only occurred when the client tried to read the object before the next object was written by the server. For example, the server wrote object number x, after which the client tried to read object x before the server could start writing object number x + 1. This makes me suspect that some kind of coalescing is happening on the server side.

EDIT

The server listens for 3 different ports. The client connects one at a time to each of these ports.

There are three connections: one that often sends some data from the server to the client. The second, which sends data only from the client to the server. And the third, which is very rarely used to send one byte of data. Before the first connection, I ran into a problem. I verify select()that the data is available on this connection, and then when I mark a 40-byte read, I find that about one and a half seconds have been taken for this read.

Any pointers on how to profile this will be very helpful.

using gcc on linux.

rdrr_server_start (void) {
int rr_sd; int input_sd; int ack_sd; int fp_sd;

startTcpServer (& rr_sd, remote_rr_port); startTcpServer (& input_sd, remote_input_port); startTcpServer (& ack_sd, remote_ack_port); startTcpServer (& fp_sd, remote_fp_port);

connFD_rr = getTcpConnection(rr_sd); connFD_input = getTcpConnection(input_sd); connFD_ack= getTcpConnection(ack_sd); connFD_fp=getTcpConnection(fp_sd); }

static int getTcpConnection(int sd) { socklen_t l en;
struct sockaddr_in clientAddress; len = sizeof(clientAddress); int connFD = accept(sd, (struct sockaddr*) &clientAddress, &len); nodelay(connFD); fflush(stdout); return connFD; }

static void startTcpServer(int *sd, const int port) { *sd= socket(AF_INET, SOCK_STREAM, 0); ASSERT(*sd > 0);

// Set socket option so that port can be reused int enable = 1; setsockopt(*sd, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int));

struct sockaddr_in a; memset(&a,0,sizeof(a)); a.sin_family = AF_INET; a.sin_port = port; a.sin_addr.s_addr = INADDR_ANY; int bindResult = bind(*sd, (struct sockaddr *) &a, sizeof(a)); ASSERT(bindResult ==0); listen(*sd,2); } static void nodelay(int fd) { int flag=1; ASSERT(setsockopt(fd, SOL_TCP, TCP_NODELAY, &flag, sizeof flag)==0); }

startTcpClient() { connFD_rr = socket(AF_INET, SOCK_STREAM, 0); connFD_input = socket(AF_INET, SOCK_STREAM, 0); connFD_ack = socket(AF_INET, SOCK_STREAM, 0); connFD_fp= socket(AF_INET, SOCK_STREAM, 0);

struct sockaddr_in a; memset(&a,0,sizeof(a)); a.sin_family = AF_INET; a.sin_port = remote_rr_port; a.sin_addr.s_addr = inet_addr(remote_server_ip);

int CONNECT_TO_SERVER= connect(connFD_rr, &a, sizeof(a)); ASSERT(CONNECT_TO_SERVER==0) ;

a.sin_port = remote_input_port; CONNECT_TO_SERVER= connect(connFD_input, &a, sizeof(a)); ASSERT(CONNECT_TO_SERVER==0) ;

a.sin_port = remote_ack_port; CONNECT_TO_SERVER= connect(connFD_ack, &a, sizeof(a)); ASSERT(CONNECT_TO_SERVER==0) ;

a.sin_port = remote_fp_port; CONNECT_TO_SERVER= connect(connFD_fp, &a, sizeof(a)); ASSERT(CONNECT_TO_SERVER==0) ;

nodelay(connFD_rr); nodelay(connFD_input); nodelay(connFD_ack); nodelay(connFD_fp); }