|
|
@ -119,10 +119,12 @@ int64 iob_send(int64 s,io_batch* b) {
|
|
|
|
#include <unistd.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include "havealloca.h"
|
|
|
|
#include "havealloca.h"
|
|
|
|
|
|
|
|
#include "io_internal.h"
|
|
|
|
#include "iob_internal.h"
|
|
|
|
#include "iob_internal.h"
|
|
|
|
|
|
|
|
|
|
|
|
int64 iob_send(int64 s,io_batch* b) {
|
|
|
|
int64 iob_send(int64 s,io_batch* b) {
|
|
|
|
iob_entry* e,* last;
|
|
|
|
iob_entry* e,* last;
|
|
|
|
|
|
|
|
io_entry* E;
|
|
|
|
struct iovec* v;
|
|
|
|
struct iovec* v;
|
|
|
|
uint64 total;
|
|
|
|
uint64 total;
|
|
|
|
int64 sent;
|
|
|
|
int64 sent;
|
|
|
@ -137,8 +139,13 @@ int64 iob_send(int64 s,io_batch* b) {
|
|
|
|
#ifdef TCP_CORK
|
|
|
|
#ifdef TCP_CORK
|
|
|
|
int corked=0;
|
|
|
|
int corked=0;
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#ifdef MSG_ZEROCOPY
|
|
|
|
|
|
|
|
size_t sum=0;
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
if (b->bytesleft==0) return 0;
|
|
|
|
if (b->bytesleft==0) return 0;
|
|
|
|
|
|
|
|
E=iarray_get(&io_fds,s);
|
|
|
|
|
|
|
|
if (!E) { errno=EBADF; return -3; }
|
|
|
|
last=(iob_entry*)(((char*)array_start(&b->b))+array_bytes(&b->b));
|
|
|
|
last=(iob_entry*)(((char*)array_start(&b->b))+array_bytes(&b->b));
|
|
|
|
v=alloca(b->bufs*sizeof(struct iovec));
|
|
|
|
v=alloca(b->bufs*sizeof(struct iovec));
|
|
|
|
total=0;
|
|
|
|
total=0;
|
|
|
@ -154,6 +161,9 @@ int64 iob_send(int64 s,io_batch* b) {
|
|
|
|
if (e[i].type==FROMFILE) break;
|
|
|
|
if (e[i].type==FROMFILE) break;
|
|
|
|
v[i].iov_base=(char*)(e[i].buf+e[i].offset);
|
|
|
|
v[i].iov_base=(char*)(e[i].buf+e[i].offset);
|
|
|
|
v[i].iov_len=e[i].n;
|
|
|
|
v[i].iov_len=e[i].n;
|
|
|
|
|
|
|
|
#ifdef MSG_ZEROCOPY
|
|
|
|
|
|
|
|
if (sum + v[i].iov_len > sum) sum += v[i].iov_len; else sum=-1;
|
|
|
|
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
}
|
|
|
|
headers=i;
|
|
|
|
headers=i;
|
|
|
|
#ifdef HAVE_BSDSENDFILE
|
|
|
|
#ifdef HAVE_BSDSENDFILE
|
|
|
@ -210,20 +220,39 @@ eagain:
|
|
|
|
corked=1;
|
|
|
|
corked=1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (headers) {
|
|
|
|
if (headers) {
|
|
|
|
if (docork<0) { /* write+writev */
|
|
|
|
int ZEROCOPY=0;
|
|
|
|
|
|
|
|
#ifdef MSG_ZEROCOPY
|
|
|
|
|
|
|
|
static int nozerocopy;
|
|
|
|
|
|
|
|
int dozerocopy=1;
|
|
|
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
if (nozerocopy && dozerocopy==0 && docork<0) { /* write+writev */
|
|
|
|
if (headers==1) /* cosmetics for strace */
|
|
|
|
if (headers==1) /* cosmetics for strace */
|
|
|
|
sent=write(s,v[0].iov_base,v[0].iov_len);
|
|
|
|
sent=write(s,v[0].iov_base,v[0].iov_len);
|
|
|
|
else
|
|
|
|
else
|
|
|
|
sent=writev(s,v,headers);
|
|
|
|
sent=writev(s,v,headers);
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
|
|
|
|
#ifdef MSG_ZEROCOPY
|
|
|
|
|
|
|
|
if (!nozerocopy && sum>=8*1024) {
|
|
|
|
|
|
|
|
/* MSG_ZEROCOPY has page table management overhead,
|
|
|
|
|
|
|
|
* it only pays off after 8k or so */
|
|
|
|
|
|
|
|
if (E->zerocopy==0) {
|
|
|
|
|
|
|
|
if (setsockopt(s, SOL_SOCKET, SO_ZEROCOPY, (int[]){ 1 },sizeof(int)) == 0) {
|
|
|
|
|
|
|
|
E->zerocopy=1;
|
|
|
|
|
|
|
|
ZEROCOPY=MSG_ZEROCOPY;
|
|
|
|
|
|
|
|
} else
|
|
|
|
|
|
|
|
nozerocopy=1;
|
|
|
|
|
|
|
|
} else
|
|
|
|
|
|
|
|
ZEROCOPY=MSG_ZEROCOPY;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|
|
|
|
if (headers==1) /* cosmetics for strace */
|
|
|
|
if (headers==1) /* cosmetics for strace */
|
|
|
|
sent=sendto(s,v[0].iov_base,v[0].iov_len,MSG_MORE, NULL, 0);
|
|
|
|
sent=sendto(s, v[0].iov_base, v[0].iov_len, MSG_MORE|ZEROCOPY, NULL, 0);
|
|
|
|
else {
|
|
|
|
else {
|
|
|
|
struct msghdr msg;
|
|
|
|
struct msghdr msg;
|
|
|
|
memset(&msg,0,sizeof(msg));
|
|
|
|
memset(&msg,0,sizeof(msg));
|
|
|
|
msg.msg_iov=v;
|
|
|
|
msg.msg_iov=v;
|
|
|
|
msg.msg_iovlen=headers;
|
|
|
|
msg.msg_iovlen=headers;
|
|
|
|
sent=sendmsg(s,&msg,MSG_MORE);
|
|
|
|
sent=sendmsg(s,&msg,MSG_MORE|ZEROCOPY);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (sent==-1) {
|
|
|
|
if (sent==-1) {
|
|
|
|