lib: Curl_read/Curl_write clarifications

- replace `Curl_read()`, `Curl_write()` and `Curl_nwrite()` to
  clarify when and at what level they operate
- send/recv of transfer related data is now done via
  `Curl_xfer_send()/Curl_xfer_recv()` which no longer has
  socket/socketindex as parameter. It decides on the transfer
  setup of `conn->sockfd` and `conn->writesockfd` on which
  connection filter chain to operate.
- send/recv on a specific connection filter chain is done via
  `Curl_conn_send()/Curl_conn_recv()` which get the socket index
  as parameter.
- rename `Curl_setup_transfer()` to `Curl_xfer_setup()` for
  naming consistency
- clarify that the special CURLE_AGAIN hangling to return
  `CURLE_OK` with length 0 only applies to `Curl_xfer_send()`
  and CURLE_AGAIN is returned by all other send() variants.
- fix a bug in websocket `curl_ws_recv()` that mixed up data
  when it arrived in more than a single chunk

The method for sending not just raw bytes, but bytes that are either
"headers" or "body". The send abstraction stack, to to bottom, now is:

* `Curl_req_send()`: has parameter to indicate amount of header bytes,
  buffers all data.
* `Curl_xfer_send()`: knows on which socket index to send, returns
  amount of bytes sent.
* `Curl_conn_send()`: called with socket index, returns amount of bytes
  sent.

In addition there is `Curl_req_flush()` for writing out all buffered
bytes.

`Curl_req_send()` is active for requests without body,
`Curl_buffer_send()` still being used for others. This is because the
special quirks need to be addressed in future parts:

* `expect-100` handling
* `Curl_fillreadbuffer()` needs to add directly to the new
  `data->req.sendbuf`
* special body handlings, like `chunked` encodings and line end
  conversions will be moved into something like a Client Reader.

In functions of the pattern `CURLcode xxx_send(..., ssize_t *written)`,
replace the `ssize_t` with a `size_t`. It makes no sense to allow for negative
values as the returned `CURLcode` already specifies error conditions. This
allows easier handling of lengths without casting.

Closes #12964

lib/c-hyper.c

@@ -112,11 +112,13 @@ size_t Curl_hyper_send(void *userp, hyper_context *ctx,
   struct hyp_io_ctx *io_ctx = userp;
   struct Curl_easy *data = io_ctx->data;
   CURLcode result;
-  ssize_t nwrote;
+  size_t nwrote;
 
   DEBUGF(infof(data, "Curl_hyper_send(%zu)", buflen));
   result = Curl_conn_send(data, io_ctx->sockindex,
                           (void *)buf, buflen, &nwrote);
+  if(!result && !nwrote)
+    result = CURLE_AGAIN;
   if(result == CURLE_AGAIN) {
     DEBUGF(infof(data, "Curl_hyper_send(%zu) -> EAGAIN", buflen));
     /* would block, register interest */
@@ -759,7 +761,6 @@ static int uploadstreamed(void *userdata, hyper_context *ctx,
  */
 
 static CURLcode bodysend(struct Curl_easy *data,
-                         struct connectdata *conn,
                          hyper_headers *headers,
                          hyper_request *hyperreq,
                          Curl_HttpReq httpreq)
@@ -772,7 +773,7 @@ static CURLcode bodysend(struct Curl_easy *data,
   else {
     hyper_body *body;
     Curl_dyn_init(&req, DYN_HTTP_REQUEST);
-    result = Curl_http_bodysend(data, conn, &req, httpreq);
+    result = Curl_http_req_send(data, &req, httpreq);
 
     if(!result)
       result = Curl_hyper_header(data, headers, Curl_dyn_ptr(&req));
@@ -1171,7 +1172,7 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
   if(result)
     goto error;
 
-  result = bodysend(data, conn, headers, req, httpreq);
+  result = bodysend(data, headers, req, httpreq);
   if(result)
     goto error;
 

lib/cf-h1-proxy.c

@@ -212,6 +212,11 @@ static void tunnel_free(struct Curl_cfilter *cf,
   }
 }
 
+static bool tunnel_want_send(struct h1_tunnel_state *ts)
+{
+  return (ts->tunnel_state == H1_TUNNEL_CONNECT);
+}
+
 #ifndef USE_HYPER
 static CURLcode start_CONNECT(struct Curl_cfilter *cf,
                               struct Curl_easy *data,
@@ -1032,7 +1037,7 @@ static void cf_h1_proxy_adjust_pollset(struct Curl_cfilter *cf,
          wait for the socket to become readable to be able to get the
          response headers or if we're still sending the request, wait
          for write. */
-      if(ts->CONNECT.sending == HTTPSEND_REQUEST)
+      if(tunnel_want_send(ts))
         Curl_pollset_set_out_only(data, ps, sock);
       else
         Curl_pollset_set_in_only(data, ps, sock);

lib/cf-haproxy.c

@@ -129,7 +129,7 @@ static CURLcode cf_haproxy_connect(struct Curl_cfilter *cf,
   case HAPROXY_SEND:
     len = Curl_dyn_len(&ctx->data_out);
     if(len > 0) {
-      ssize_t written;
+      size_t written;
       result = Curl_conn_send(data, cf->sockindex,
                               Curl_dyn_ptr(&ctx->data_out),
                               len, &written);
@@ -139,8 +139,7 @@ static CURLcode cf_haproxy_connect(struct Curl_cfilter *cf,
       }
       else if(result)
         goto out;
-      DEBUGASSERT(written >= 0);
-      Curl_dyn_tail(&ctx->data_out, len - (size_t)written);
+      Curl_dyn_tail(&ctx->data_out, len - written);
       if(Curl_dyn_len(&ctx->data_out) > 0) {
         result = CURLE_OK;
         goto out;

lib/cfilters.c

@@ -694,7 +694,7 @@ CURLcode Curl_conn_recv(struct Curl_easy *data, int sockindex,
 
 CURLcode Curl_conn_send(struct Curl_easy *data, int sockindex,
                         const void *buf, size_t blen,
-                        ssize_t *pnwritten)
+                        size_t *pnwritten)
 {
   ssize_t nwritten;
   CURLcode result = CURLE_OK;
@@ -719,7 +719,7 @@ CURLcode Curl_conn_send(struct Curl_easy *data, int sockindex,
 #endif
   nwritten = conn->send[sockindex](data, sockindex, buf, blen, &result);
   DEBUGASSERT((nwritten >= 0) || result);
-  *pnwritten = nwritten;
+  *pnwritten = (nwritten < 0)? 0 : (size_t)nwritten;
   return result;
 }
 

lib/cfilters.h

@@ -518,7 +518,7 @@ CURLcode Curl_conn_recv(struct Curl_easy *data, int sockindex,
  */
 CURLcode Curl_conn_send(struct Curl_easy *data, int sockindex,
                         const void *buf, size_t blen,
-                        ssize_t *pnwritten);
+                        size_t *pnwritten);
 
 
 void Curl_pollset_reset(struct Curl_easy *data,

lib/dict.c

@@ -127,7 +127,7 @@ static CURLcode sendf(struct Curl_easy *data,
 
 static CURLcode sendf(struct Curl_easy *data, const char *fmt, ...)
 {
-  ssize_t bytes_written;
+  size_t bytes_written;
   size_t write_len;
   CURLcode result = CURLE_OK;
   char *s;

lib/easy.c

@@ -1251,7 +1251,7 @@ CURLcode Curl_connect_only_attach(struct Curl_easy *data)
  * This is the private internal version of curl_easy_send()
  */
 CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer,
-                       size_t buflen, ssize_t *n)
+                       size_t buflen, size_t *n)
 {
   CURLcode result;
   struct connectdata *c = NULL;
@@ -1283,13 +1283,13 @@ CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer,
 CURLcode curl_easy_send(struct Curl_easy *data, const void *buffer,
                         size_t buflen, size_t *n)
 {
-  ssize_t written = 0;
+  size_t written = 0;
   CURLcode result;
   if(Curl_is_in_callback(data))
     return CURLE_RECURSIVE_API_CALL;
 
   result = Curl_senddata(data, buffer, buflen, &written);
-  *n = (size_t)written;
+  *n = written;
   return result;
 }
 

lib/easyif.h

@@ -28,7 +28,7 @@
  * Prototypes for library-wide functions provided by easy.c
  */
 CURLcode Curl_senddata(struct Curl_easy *data, const void *buffer,
-                       size_t buflen, ssize_t *n);
+                       size_t buflen, size_t *n);
 
 #ifdef USE_WEBSOCKETS
 CURLcode Curl_connect_only_attach(struct Curl_easy *data);

lib/gopher.c

@@ -139,8 +139,8 @@ static CURLcode gopher_do(struct Curl_easy *data, bool *done)
   char *sel = NULL;
   char *sel_org = NULL;
   timediff_t timeout_ms;
-  ssize_t amount, k;
-  size_t len;
+  ssize_t k;
+  size_t amount, len;
   int what;
 
   *done = TRUE; /* unconditionally */

lib/http.c

@@ -1174,6 +1174,7 @@ static bool http_should_fail(struct Curl_easy *data)
   return data->state.authproblem;
 }
 
+#ifndef USE_HYPER
 /*
  * readmoredata() is a "fread() emulation" to provide POST and/or request
  * data. It is used when a huge POST is to be made and the entire chunk wasn't
@@ -1238,17 +1239,16 @@ static size_t readmoredata(char *buffer,
  *
  * Returns CURLcode
  */
-CURLcode Curl_buffer_send(struct dynbuf *in,
-                          struct Curl_easy *data,
-                          struct HTTP *http,
-                          /* add the number of sent bytes to this
-                             counter */
-                          curl_off_t *bytes_written,
-                          /* how much of the buffer contains body data */
-                          curl_off_t included_body_bytes,
-                          int sockindex)
+static CURLcode buffer_send(struct dynbuf *in,
+                            struct Curl_easy *data,
+                            struct HTTP *http,
+                            /* add the number of sent bytes to this
+                               counter */
+                            curl_off_t *bytes_written,
+                            /* how much of the buffer contains body data */
+                            curl_off_t included_body_bytes)
 {
-  ssize_t amount;
+  size_t amount;
   CURLcode result;
   char *ptr;
   size_t size;
@@ -1256,8 +1256,6 @@ CURLcode Curl_buffer_send(struct dynbuf *in,
   size_t sendsize;
   size_t headersize;
 
-  DEBUGASSERT(sockindex <= SECONDARYSOCKET && sockindex >= 0);
-
   /* The looping below is required since we use non-blocking sockets, but due
      to the circumstances we will just loop and try again and again etc */
 
@@ -1356,11 +1354,7 @@ CURLcode Curl_buffer_send(struct dynbuf *in,
       sendsize = (size_t)data->set.upload_buffer_size;
   }
 
-  result = Curl_conn_send(data, sockindex, ptr, sendsize, &amount);
-  if(result == CURLE_AGAIN) {
-    result = CURLE_OK;
-    amount = 0;
-  }
+  result = Curl_xfer_send(data, ptr, sendsize, &amount);
 
   if(!result) {
     /*
@@ -1443,6 +1437,11 @@ CURLcode Curl_buffer_send(struct dynbuf *in,
 
 /* end of the add_buffer functions */
 /* ------------------------------------------------------------------------- */
+#else /* !USE_HYPER */
+  /* In hyper, this is an ugly NOP */
+#define buffer_send(a,b,c,d,e) CURLE_OK
+
+#endif /* !USE_HYPER(else) */
 
 
 
@@ -1619,13 +1618,12 @@ static const char *get_http_string(const struct Curl_easy *data,
 #endif
 
 /* check and possibly add an Expect: header */
-static CURLcode expect100(struct Curl_easy *data,
-                          struct connectdata *conn,
-                          struct dynbuf *req)
+static CURLcode expect100(struct Curl_easy *data, struct dynbuf *req)
 {
   CURLcode result = CURLE_OK;
-  if(!data->state.disableexpect && Curl_use_http_1_1plus(data, conn) &&
-     (conn->httpversion < 20)) {
+  if(!data->state.disableexpect &&
+     Curl_use_http_1_1plus(data, data->conn) &&
+     (data->conn->httpversion < 20)) {
     /* if not doing HTTP 1.0 or version 2, or disabled explicitly, we add an
        Expect: 100-continue to the headers which actually speeds up post
        operations (as there is one packet coming back from the web server) */
@@ -2441,8 +2439,7 @@ CURLcode Curl_http_body(struct Curl_easy *data, struct connectdata *conn,
   return result;
 }
 
-static CURLcode addexpect(struct Curl_easy *data, struct connectdata *conn,
-                          struct dynbuf *r)
+static CURLcode addexpect(struct Curl_easy *data, struct dynbuf *r)
 {
   data->state.expect100header = FALSE;
   /* Avoid Expect: 100-continue if Upgrade: is used */
@@ -2459,24 +2456,22 @@ static CURLcode addexpect(struct Curl_easy *data, struct connectdata *conn,
                            STRCONST("100-continue"));
     }
     else if(http->postsize > EXPECT_100_THRESHOLD || http->postsize < 0)
-      return expect100(data, conn, r);
+      return expect100(data, r);
   }
   return CURLE_OK;
 }
 
-CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
+CURLcode Curl_http_req_send(struct Curl_easy *data,
                             struct dynbuf *r, Curl_HttpReq httpreq)
 {
 #ifndef USE_HYPER
   /* Hyper always handles the body separately */
   curl_off_t included_body = 0;
-#else
-  /* from this point down, this function should not be used */
-#define Curl_buffer_send(a,b,c,d,e,f) CURLE_OK
 #endif
   CURLcode result = CURLE_OK;
   struct HTTP *http = data->req.p.http;
 
+  DEBUGASSERT(data->conn);
   switch(httpreq) {
   case HTTPREQ_PUT: /* Let's PUT the data to the server! */
 
@@ -2495,7 +2490,7 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
         return result;
     }
 
-    result = addexpect(data, conn, r);
+    result = addexpect(data, r);
     if(result)
       return result;
 
@@ -2508,9 +2503,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
     Curl_pgrsSetUploadSize(data, http->postsize);
 
     /* this sends the buffer and frees all the buffer resources */
-    result = Curl_buffer_send(r, data, data->req.p.http,
-                              &data->info.request_size, 0,
-                              FIRSTSOCKET);
+    result = buffer_send(r, data, data->req.p.http,
+                         &data->info.request_size, 0);
     if(result)
       failf(data, "Failed sending PUT request");
     else
@@ -2531,9 +2525,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
       if(result)
         return result;
 
-      result = Curl_buffer_send(r, data, data->req.p.http,
-                                &data->info.request_size, 0,
-                                FIRSTSOCKET);
+      result = buffer_send(r, data, data->req.p.http,
+                           &data->info.request_size, 0);
       if(result)
         failf(data, "Failed sending POST request");
       else
@@ -2571,7 +2564,7 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
     }
 #endif
 
-    result = addexpect(data, conn, r);
+    result = addexpect(data, r);
     if(result)
       return result;
 
@@ -2589,9 +2582,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
     http->sending = HTTPSEND_BODY;
 
     /* this sends the buffer and frees all the buffer resources */
-    result = Curl_buffer_send(r, data, data->req.p.http,
-                              &data->info.request_size, 0,
-                              FIRSTSOCKET);
+    result = buffer_send(r, data, data->req.p.http,
+                         &data->info.request_size, 0);
     if(result)
       failf(data, "Failed sending POST request");
     else
@@ -2633,7 +2625,7 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
         return result;
     }
 
-    result = addexpect(data, conn, r);
+    result = addexpect(data, r);
     if(result)
       return result;
 
@@ -2732,9 +2724,8 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
       }
     }
     /* issue the request */
-    result = Curl_buffer_send(r, data, data->req.p.http,
-                              &data->info.request_size, included_body,
-                              FIRSTSOCKET);
+    result = buffer_send(r, data, data->req.p.http,
+                         &data->info.request_size, included_body);
 
     if(result)
       failf(data, "Failed sending HTTP POST request");
@@ -2749,13 +2740,12 @@ CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
       return result;
 
     /* issue the request */
-    result = Curl_buffer_send(r, data, data->req.p.http,
-                              &data->info.request_size, 0,
-                              FIRSTSOCKET);
+    result = Curl_req_send_hds(data, Curl_dyn_ptr(r), Curl_dyn_len(r));
+    Curl_dyn_free(r);
     if(result)
       failf(data, "Failed sending HTTP request");
 #ifdef USE_WEBSOCKETS
-    else if((conn->handler->protocol & (CURLPROTO_WS|CURLPROTO_WSS)) &&
+    else if((data->conn->handler->protocol & (CURLPROTO_WS|CURLPROTO_WSS)) &&
             !(data->set.connect_only))
       /* Set up the transfer for two-way since without CONNECT_ONLY set, this
          request probably wants to send data too post upgrade */
@@ -3329,8 +3319,8 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done)
        (httpreq == HTTPREQ_HEAD))
       Curl_pgrsSetUploadSize(data, 0); /* nothing */
 
-    /* bodysend takes ownership of the 'req' memory on success */
-    result = Curl_http_bodysend(data, conn, &req, httpreq);
+    /* req_send takes ownership of the 'req' memory on success */
+    result = Curl_http_req_send(data, &req, httpreq);
   }
   if(result) {
     Curl_dyn_free(&req);

lib/http.h

@@ -74,12 +74,6 @@ char *Curl_checkProxyheaders(struct Curl_easy *data,
                              const char *thisheader,
                              const size_t thislen);
 struct HTTP; /* see below */
-CURLcode Curl_buffer_send(struct dynbuf *in,
-                          struct Curl_easy *data,
-                          struct HTTP *http,
-                          curl_off_t *bytes_written,
-                          curl_off_t included_body_bytes,
-                          int socketindex);
 
 CURLcode Curl_add_timecondition(struct Curl_easy *data,
 #ifndef USE_HYPER
@@ -118,7 +112,7 @@ CURLcode Curl_transferencode(struct Curl_easy *data);
 CURLcode Curl_http_body(struct Curl_easy *data, struct connectdata *conn,
                         Curl_HttpReq httpreq,
                         const char **teep);
-CURLcode Curl_http_bodysend(struct Curl_easy *data, struct connectdata *conn,
+CURLcode Curl_http_req_send(struct Curl_easy *data,
                             struct dynbuf *r, Curl_HttpReq httpreq);
 bool Curl_use_http_1_1plus(const struct Curl_easy *data,
                            const struct connectdata *conn);

lib/krb5.c

@@ -66,7 +66,7 @@
 static CURLcode ftpsend(struct Curl_easy *data, struct connectdata *conn,
                         const char *cmd)
 {
-  ssize_t bytes_written;
+  size_t bytes_written;
 #define SBUF_SIZE 1024
   char s[SBUF_SIZE];
   size_t write_len;
@@ -100,9 +100,9 @@ static CURLcode ftpsend(struct Curl_easy *data, struct connectdata *conn,
     if(result)
       break;
 
-    Curl_debug(data, CURLINFO_HEADER_OUT, sptr, (size_t)bytes_written);
+    Curl_debug(data, CURLINFO_HEADER_OUT, sptr, bytes_written);
 
-    if(bytes_written != (ssize_t)write_len) {
+    if(bytes_written != write_len) {
       write_len -= bytes_written;
       sptr += bytes_written;
     }
@@ -494,7 +494,7 @@ socket_write(struct Curl_easy *data, int sockindex, const void *to,
 {
   const char *to_p = to;
   CURLcode result;
-  ssize_t written;
+  size_t written;
 
   while(len > 0) {
     result = Curl_conn_send(data, sockindex, to_p, len, &written);

lib/mqtt.c

@@ -119,12 +119,12 @@ static CURLcode mqtt_send(struct Curl_easy *data,
 {
   CURLcode result = CURLE_OK;
   struct MQTT *mq = data->req.p.mqtt;
-  ssize_t n;
+  size_t n;
   result = Curl_xfer_send(data, buf, len, &n);
   if(result)
     return result;
   Curl_debug(data, CURLINFO_HEADER_OUT, buf, (size_t)n);
-  if(len != (size_t)n) {
+  if(len != n) {
     size_t nsend = len - n;
     char *sendleftovers = Curl_memdup(&buf[n], nsend);
     if(!sendleftovers)

lib/pingpong.c

@@ -164,7 +164,7 @@ CURLcode Curl_pp_vsendf(struct Curl_easy *data,
                         const char *fmt,
                         va_list args)
 {
-  ssize_t bytes_written = 0;
+  size_t bytes_written = 0;
   size_t write_len;
   char *s;
   CURLcode result;
@@ -211,9 +211,9 @@ CURLcode Curl_pp_vsendf(struct Curl_easy *data,
   conn->data_prot = (unsigned char)data_sec;
 #endif
 
-  Curl_debug(data, CURLINFO_HEADER_OUT, s, (size_t)bytes_written);
+  Curl_debug(data, CURLINFO_HEADER_OUT, s, bytes_written);
 
-  if(bytes_written != (ssize_t)write_len) {
+  if(bytes_written != write_len) {
     /* the whole chunk was not sent, keep it around and adjust sizes */
     pp->sendthis = s;
     pp->sendsize = write_len;
@@ -398,7 +398,7 @@ CURLcode Curl_pp_flushsend(struct Curl_easy *data,
                            struct pingpong *pp)
 {
   /* we have a piece of a command still left to send */
-  ssize_t written;
+  size_t written;
   CURLcode result;
 
   result = Curl_conn_send(data, FIRSTSOCKET,
@@ -411,7 +411,7 @@ CURLcode Curl_pp_flushsend(struct Curl_easy *data,
   if(result)
     return result;
 
-  if(written != (ssize_t)pp->sendleft) {
+  if(written != pp->sendleft) {
     /* only a fraction was sent */
     pp->sendleft -= written;
   }

lib/request.c

@@ -27,8 +27,10 @@
 #include "urldata.h"
 #include "dynbuf.h"
 #include "doh.h"
+#include "progress.h"
 #include "request.h"
 #include "sendf.h"
+#include "transfer.h"
 #include "url.h"
 
 /* The last 3 #include files should be in this order */
@@ -39,8 +41,6 @@
 CURLcode Curl_req_init(struct SingleRequest *req)
 {
   memset(req, 0, sizeof(*req));
-  Curl_bufq_init2(&req->sendbuf, UPLOADBUFFER_DEFAULT, 1,
-                  BUFQ_OPT_SOFT_LIMIT);
   return CURLE_OK;
 }
 
@@ -49,6 +49,20 @@ CURLcode Curl_req_start(struct SingleRequest *req,
 {
   req->start = Curl_now();
   Curl_cw_reset(data);
+  if(!req->sendbuf_init) {
+    Curl_bufq_init2(&req->sendbuf, data->set.upload_buffer_size, 1,
+                    BUFQ_OPT_SOFT_LIMIT);
+    req->sendbuf_init = TRUE;
+  }
+  else {
+    Curl_bufq_reset(&req->sendbuf);
+    if(data->set.upload_buffer_size != req->sendbuf.chunk_size) {
+      Curl_bufq_free(&req->sendbuf);
+      Curl_bufq_init2(&req->sendbuf, data->set.upload_buffer_size, 1,
+                      BUFQ_OPT_SOFT_LIMIT);
+    }
+  }
+
   return CURLE_OK;
 }
 
@@ -56,33 +70,40 @@ CURLcode Curl_req_done(struct SingleRequest *req,
                        struct Curl_easy *data, bool aborted)
 {
   (void)req;
-  /* TODO: add flush handling for client output */
-  (void)aborted;
+  if(!aborted)
+    (void)Curl_req_flush(data);
   Curl_cw_reset(data);
   return CURLE_OK;
 }
 
 void Curl_req_reset(struct SingleRequest *req, struct Curl_easy *data)
 {
+  struct bufq savebuf;
+  bool save_init;
+
   /* This is a bit ugly. `req->p` is a union and we assume we can
    * free this safely without leaks. */
   Curl_safefree(req->p.http);
   Curl_safefree(req->newurl);
   Curl_cw_reset(data);
 
-  Curl_bufq_reset(&req->sendbuf);
-  if(data->set.upload_buffer_size != req->sendbuf.chunk_size) {
-    Curl_bufq_free(&req->sendbuf);
-    Curl_bufq_init2(&req->sendbuf, data->set.upload_buffer_size, 1,
-                    BUFQ_OPT_SOFT_LIMIT);
-  }
-
 #ifndef CURL_DISABLE_DOH
   if(req->doh) {
     Curl_close(&req->doh->probe[0].easy);
     Curl_close(&req->doh->probe[1].easy);
   }
 #endif
+
+  savebuf = req->sendbuf;
+  save_init = req->sendbuf_init;
+
+  memset(req, 0, sizeof(*req));
+  data->req.size = data->req.maxdownload = -1;
+  data->req.no_body = data->set.opt_no_body;
+  if(save_init) {
+    req->sendbuf = savebuf;
+    req->sendbuf_init = save_init;
+  }
 }
 
 void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data)
@@ -91,7 +112,8 @@ void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data)
    * free this safely without leaks. */
   Curl_safefree(req->p.http);
   Curl_safefree(req->newurl);
-  Curl_bufq_free(&req->sendbuf);
+  if(req->sendbuf_init)
+    Curl_bufq_free(&req->sendbuf);
   Curl_cw_reset(data);
 
 #ifndef CURL_DISABLE_DOH
@@ -106,3 +128,130 @@ void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data)
 #endif
 }
 
+static CURLcode req_send(struct Curl_easy *data,
+                         const char *buf, size_t blen,
+                         size_t hds_len, size_t *pnwritten)
+{
+  CURLcode result = CURLE_OK;
+
+  *pnwritten = 0;
+#ifdef CURLDEBUG
+  {
+    /* Allow debug builds to override this logic to force short initial
+       sends
+     */
+    char *p = getenv("CURL_SMALLREQSEND");
+    if(p) {
+      size_t altsize = (size_t)strtoul(p, NULL, 10);
+      if(altsize && altsize < blen)
+        blen = altsize;
+    }
+  }
+#endif
+  /* Make sure this doesn't send more body bytes than what the max send
+     speed says. The headers do not count to the max speed. */
+  if(data->set.max_send_speed) {
+    size_t body_bytes = blen - hds_len;
+    if((curl_off_t)body_bytes > data->set.max_send_speed)
+      blen = hds_len + (size_t)data->set.max_send_speed;
+  }
+
+  result = Curl_xfer_send(data, buf, blen, pnwritten);
+  if(!result && *pnwritten) {
+    if(hds_len)
+      Curl_debug(data, CURLINFO_HEADER_OUT, (char *)buf,
+                 CURLMIN(hds_len, *pnwritten));
+    if(*pnwritten > hds_len) {
+      size_t body_len = *pnwritten - hds_len;
+      Curl_debug(data, CURLINFO_DATA_OUT, (char *)buf + hds_len, body_len);
+      data->req.writebytecount += body_len;
+      Curl_pgrsSetUploadCounter(data, data->req.writebytecount);
+    }
+  }
+  return result;
+}
+
+static CURLcode req_send_buffer_add(struct Curl_easy *data,
+                                    const char *buf, size_t blen,
+                                    size_t hds_len)
+{
+  CURLcode result = CURLE_OK;
+  ssize_t n;
+  n = Curl_bufq_write(&data->req.sendbuf,
+                      (const unsigned char *)buf, blen, &result);
+  if(n < 0)
+    return result;
+  /* We rely on a SOFTLIMIT on sendbuf, so it can take all data in */
+  DEBUGASSERT((size_t)n == blen);
+  data->req.sendbuf_hds_len += hds_len;
+  return CURLE_OK;
+}
+
+static CURLcode req_send_buffer_flush(struct Curl_easy *data)
+{
+  CURLcode result = CURLE_OK;
+  const unsigned char *buf;
+  size_t blen;
+
+  while(Curl_bufq_peek(&data->req.sendbuf, &buf, &blen)) {
+    size_t nwritten, hds_len = CURLMIN(data->req.sendbuf_hds_len, blen);
+    result = req_send(data, (const char *)buf, blen, hds_len, &nwritten);
+    if(result)
+      break;
+
+    Curl_bufq_skip(&data->req.sendbuf, nwritten);
+    if(hds_len)
+      data->req.sendbuf_hds_len -= CURLMIN(hds_len, nwritten);
+    /* leave if we could not send all. Maybe network blocking or
+     * speed limits on transfer */
+    if(nwritten < blen)
+      break;
+  }
+  return result;
+}
+
+CURLcode Curl_req_flush(struct Curl_easy *data)
+{
+  CURLcode result;
+
+  if(!data || !data->conn)
+    return CURLE_FAILED_INIT;
+
+  if(!Curl_bufq_is_empty(&data->req.sendbuf)) {
+    result = req_send_buffer_flush(data);
+    if(result)
+      return result;
+    if(!Curl_bufq_is_empty(&data->req.sendbuf)) {
+      return CURLE_AGAIN;
+    }
+  }
+  return CURLE_OK;
+}
+
+CURLcode Curl_req_send(struct Curl_easy *data,
+                       const char *buf, size_t blen,
+                       size_t hds_len)
+{
+  CURLcode result;
+
+  if(!data || !data->conn)
+    return CURLE_FAILED_INIT;
+
+  /* We always buffer and send from there. The reason is that on
+   * blocking, we can retry using the same memory address. This is
+   * important for TLS libraries that expect this.
+   * We *could* optimized for non-TLS transfers, but that would mean
+   * separate code paths and seems not worth it. */
+  result = req_send_buffer_add(data, buf, blen, hds_len);
+  if(result)
+    return result;
+  result = req_send_buffer_flush(data);
+  if(result == CURLE_AGAIN)
+    result = CURLE_OK;
+  return result;
+}
+
+bool Curl_req_want_send(struct Curl_easy *data)
+{
+  return data->req.sendbuf_init && !Curl_bufq_is_empty(&data->req.sendbuf);
+}

lib/request.h

@@ -64,7 +64,7 @@ struct SingleRequest {
   curl_off_t bytecount;         /* total number of bytes read */
   curl_off_t writebytecount;    /* number of bytes written */
 
-  curl_off_t pendingheader;      /* this many bytes left to send is actually
+  size_t pendingheader;         /* this many bytes left to send is actually
                                     header and not body */
   struct curltime start;         /* transfer started at this time */
   unsigned int headerbytecount;  /* received server headers (not CONNECT
@@ -91,6 +91,7 @@ struct SingleRequest {
    * checks, pausing by client callbacks. */
   struct Curl_cwriter *writer_stack;
   struct bufq sendbuf; /* data which needs to be send to the server */
+  size_t sendbuf_hds_len; /* amount of header bytes in sendbuf */
   time_t timeofdoc;
   long bodywrites;
   char *location;   /* This points to an allocated version of the Location:
@@ -100,7 +101,7 @@ struct SingleRequest {
 
   /* 'upload_present' is used to keep a byte counter of how much data there is
      still left in the buffer, aimed for upload. */
-  ssize_t upload_present;
+  size_t upload_present;
 
   /* 'upload_fromhere' is used as a read-pointer when we uploaded parts of a
      buffer, so the next read should read from where this pointer points to,
@@ -154,6 +155,7 @@ struct SingleRequest {
                         that we are creating a request with an auth header,
                         but it is not the final request in the auth
                         negotiation. */
+  BIT(sendbuf_init); /* sendbuf is initialized */
 };
 
 /**
@@ -189,4 +191,34 @@ void Curl_req_free(struct SingleRequest *req, struct Curl_easy *data);
 void Curl_req_reset(struct SingleRequest *req, struct Curl_easy *data);
 
 
+/**
+ * Send request bytes for transfer. If not all could be sent
+ * they will be buffered. Use `Curl_req_flush()` to make sure
+ * bytes are really send.
+ * @param data      the transfer making the request
+ * @param buf       the bytes to send
+ * @param blen      the number of bytes to send
+ * @param hds_len   the number of bytes from the start that are headers
+ * @return CURLE_OK (on blocking with *pnwritten == 0) or error.
+ */
+CURLcode Curl_req_send(struct Curl_easy *data,
+                        const char *buf, size_t blen,
+                        size_t hds_len);
+
+/* Convenience for sending only header bytes */
+#define Curl_req_send_hds(data, buf, blen) \
+          Curl_req_send((data), (buf), (blen), (blen))
+
+/**
+ * Flush all buffered request bytes.
+ * @return CURLE_OK on success, CURLE_AGAIN if sending was blocked,
+ *         or the error on the sending.
+ */
+CURLcode Curl_req_flush(struct Curl_easy *data);
+
+/**
+ * TRUE iff the request wants to send, e.g. has buffered bytes.
+ */
+bool Curl_req_want_send(struct Curl_easy *data);
+
 #endif /* HEADER_CURL_REQUEST_H */

lib/rtsp.c

@@ -566,8 +566,9 @@ static CURLcode rtsp_do(struct Curl_easy *data, bool *done)
   }
 
   /* issue the request */
-  result = Curl_buffer_send(&req_buffer, data, data->req.p.http,
-                            &data->info.request_size, 0, FIRSTSOCKET);
+  result = Curl_req_send_hds(data, Curl_dyn_ptr(&req_buffer),
+                             Curl_dyn_len(&req_buffer));
+  Curl_dyn_free(&req_buffer);
   if(result) {
     failf(data, "Failed sending RTSP request");
     return result;

lib/smb.c

@@ -559,12 +559,12 @@ static void smb_format_message(struct Curl_easy *data, struct smb_header *h,
   h->pid = smb_swap16((unsigned short) pid);
 }
 
-static CURLcode smb_send(struct Curl_easy *data, ssize_t len,
+static CURLcode smb_send(struct Curl_easy *data, size_t len,
                          size_t upload_size)
 {
   struct connectdata *conn = data->conn;
   struct smb_conn *smbc = &conn->proto.smbc;
-  ssize_t bytes_written;
+  size_t bytes_written;
   CURLcode result;
 
   result = Curl_xfer_send(data, data->state.ulbuf, len, &bytes_written);
@@ -585,8 +585,8 @@ static CURLcode smb_flush(struct Curl_easy *data)
 {
   struct connectdata *conn = data->conn;
   struct smb_conn *smbc = &conn->proto.smbc;
-  ssize_t bytes_written;
-  ssize_t len = smbc->send_size - smbc->sent;
+  size_t bytes_written;
+  size_t len = smbc->send_size - smbc->sent;
   CURLcode result;
 
   if(!smbc->send_size)

lib/smtp.c

@@ -1395,8 +1395,7 @@ static CURLcode smtp_done(struct Curl_easy *data, CURLcode status,
   struct SMTP *smtp = data->req.p.smtp;
   struct pingpong *pp = &conn->proto.smtpc.pp;
   char *eob;
-  ssize_t len;
-  ssize_t bytes_written;
+  size_t len, bytes_written;
 
   (void)premature;
 

lib/telnet.c

@@ -1231,20 +1231,24 @@ process_iac:
 static CURLcode send_telnet_data(struct Curl_easy *data,
                                  char *buffer, ssize_t nread)
 {
-  ssize_t i, outlen;
+  size_t i, outlen;
   unsigned char *outbuf;
   CURLcode result = CURLE_OK;
-  ssize_t bytes_written, total_written = 0;
+  size_t bytes_written;
+  size_t total_written = 0;
   struct connectdata *conn = data->conn;
   struct TELNET *tn = data->req.p.telnet;
 
   DEBUGASSERT(tn);
+  DEBUGASSERT(nread > 0);
+  if(nread < 0)
+    return CURLE_TOO_LARGE;
 
   if(memchr(buffer, CURL_IAC, nread)) {
     /* only use the escape buffer when necessary */
     Curl_dyn_reset(&tn->out);
 
-    for(i = 0; i < nread && !result; i++) {
+    for(i = 0; i < (size_t)nread && !result; i++) {
       result = Curl_dyn_addn(&tn->out, &buffer[i], 1);
       if(!result && ((unsigned char)buffer[i] == CURL_IAC))
         /* IAC is FF in hex */
@@ -1255,7 +1259,7 @@ static CURLcode send_telnet_data(struct Curl_easy *data,
     outbuf = Curl_dyn_uptr(&tn->out);
   }
   else {
-    outlen = nread;
+    outlen = (size_t)nread;
     outbuf = (unsigned char *)buffer;
   }
   while(!result && total_written < outlen) {

lib/transfer.c

@@ -607,7 +607,7 @@ static void win_update_buffer_size(curl_socket_t sockfd)
 #endif
 
 #define curl_upload_refill_watermark(data) \
-        ((ssize_t)((data)->set.upload_buffer_size >> 5))
+        ((size_t)((data)->set.upload_buffer_size >> 5))
 
 /*
  * Send data to upload to the server, when the socket is writable.
@@ -617,7 +617,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
                                  int *didwhat)
 {
   ssize_t i, si;
-  ssize_t bytes_written;
+  size_t bytes_written;
   CURLcode result;
   ssize_t nread; /* number of bytes read */
   bool sending_http_headers = FALSE;
@@ -625,6 +625,14 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
 
   *didwhat |= KEEP_SEND;
 
+  if(!(k->keepon & KEEP_SEND_PAUSE)) {
+    result = Curl_req_flush(data);
+    if(result == CURLE_AGAIN) /* unable to send all we have */
+      return CURLE_OK;
+    else if(result)
+      return result;
+  }
+
   do {
     curl_off_t nbody;
     ssize_t offset = 0;
@@ -633,8 +641,8 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
        k->upload_present < curl_upload_refill_watermark(data) &&
        !k->upload_chunky &&/*(variable sized chunked header; append not safe)*/
        !k->upload_done &&  /*!(k->upload_done once k->upload_present sent)*/
-       !(k->writebytecount + k->upload_present - k->pendingheader ==
-         data->state.infilesize)) {
+       !(k->writebytecount + (curl_off_t)k->upload_present -
+         (curl_off_t)k->pendingheader == data->state.infilesize)) {
       offset = k->upload_present;
     }
 
@@ -770,9 +778,6 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
          that instead of reading more data */
     }
 
-    if(!Curl_bufq_is_empty(&k->sendbuf)) {
-      DEBUGASSERT(0);
-    }
     /* write to socket (send away data) */
     result = Curl_xfer_send(data,
                             k->upload_fromhere, /* buffer pointer */
@@ -793,9 +798,9 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
 
     if(k->pendingheader) {
       /* parts of what was sent was header */
-      curl_off_t n = CURLMIN(k->pendingheader, bytes_written);
+      size_t n = CURLMIN(k->pendingheader, bytes_written);
       /* show the data before we change the pointer upload_fromhere */
-      Curl_debug(data, CURLINFO_HEADER_OUT, k->upload_fromhere, (size_t)n);
+      Curl_debug(data, CURLINFO_HEADER_OUT, k->upload_fromhere, n);
       k->pendingheader -= n;
       nbody = bytes_written - n; /* size of the written body part */
     }
@@ -1607,22 +1612,19 @@ void Curl_xfer_setup(
   struct SingleRequest *k = &data->req;
   struct connectdata *conn = data->conn;
   struct HTTP *http = data->req.p.http;
-  bool httpsending;
+  bool want_send = Curl_req_want_send(data);
 
   DEBUGASSERT(conn != NULL);
   DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
   DEBUGASSERT((writesockindex <= 1) && (writesockindex >= -1));
 
-  httpsending = ((conn->handler->protocol&PROTO_FAMILY_HTTP) &&
-                 (http->sending == HTTPSEND_REQUEST));
-
-  if(conn->bits.multiplex || conn->httpversion >= 20 || httpsending) {
+  if(conn->bits.multiplex || conn->httpversion >= 20 || want_send) {
     /* when multiplexing, the read/write sockets need to be the same! */
     conn->sockfd = sockindex == -1 ?
       ((writesockindex == -1 ? CURL_SOCKET_BAD : conn->sock[writesockindex])) :
       conn->sock[sockindex];
     conn->writesockfd = conn->sockfd;
-    if(httpsending)
+    if(want_send)
       /* special and very HTTP-specific */
       writesockindex = FIRSTSOCKET;
   }
@@ -1732,7 +1734,7 @@ CURLcode Curl_xfer_write_done(struct Curl_easy *data, bool premature)
 
 CURLcode Curl_xfer_send(struct Curl_easy *data,
                         const void *buf, size_t blen,
-                        ssize_t *pnwritten)
+                        size_t *pnwritten)
 {
   CURLcode result;
   int sockindex;

lib/transfer.h

@@ -97,7 +97,7 @@ CURLcode Curl_xfer_write_done(struct Curl_easy *data, bool premature);
  */
 CURLcode Curl_xfer_send(struct Curl_easy *data,
                         const void *buf, size_t blen,
-                        ssize_t *pnwritten);
+                        size_t *pnwritten);
 
 /**
  * Receive data on the socket/connection filter designated

lib/url.c

@@ -3845,9 +3845,6 @@ CURLcode Curl_connect(struct Curl_easy *data,
 
   /* init the single-transfer specific data */
   Curl_req_reset(&data->req, data);
-  memset(&data->req, 0, sizeof(struct SingleRequest));
-  data->req.size = data->req.maxdownload = -1;
-  data->req.no_body = data->set.opt_no_body;
 
   /* call the stuff that needs to be called */
   result = create_conn(data, &conn, asyncp);

lib/vssh/libssh2.c

@@ -3213,7 +3213,7 @@ static ssize_t ssh_tls_send(libssh2_socket_t sock, const void *buffer,
                             size_t length, int flags, void **abstract)
 {
   struct Curl_easy *data = (struct Curl_easy *)*abstract;
-  ssize_t nwrite;
+  size_t nwrite;
   CURLcode result;
   struct connectdata *conn = data->conn;
   Curl_send *backup = conn->send[0];
@@ -3230,8 +3230,8 @@ static ssize_t ssh_tls_send(libssh2_socket_t sock, const void *buffer,
     return -EAGAIN; /* magic return code for libssh2 */
   else if(result)
     return -1; /* error */
-  Curl_debug(data, CURLINFO_DATA_OUT, (char *)buffer, (size_t)nwrite);
-  return nwrite;
+  Curl_debug(data, CURLINFO_DATA_OUT, (char *)buffer, nwrite);
+  return (ssize_t)nwrite;
 }
 #endif
 

lib/ws.c

@@ -1014,8 +1014,7 @@ static CURLcode ws_flush(struct Curl_easy *data, struct websocket *ws,
   if(!Curl_bufq_is_empty(&ws->sendbuf)) {
     CURLcode result;
     const unsigned char *out;
-    size_t outlen;
-    ssize_t n;
+    size_t outlen, n;
 
     while(Curl_bufq_peek(&ws->sendbuf, &out, &outlen)) {
       if(data->set.connect_only)
@@ -1044,8 +1043,8 @@ static CURLcode ws_flush(struct Curl_easy *data, struct websocket *ws,
         }
       }
       else {
-        infof(data, "WS: flushed %zu bytes", (size_t)n);
-        Curl_bufq_skip(&ws->sendbuf, (size_t)n);
+        infof(data, "WS: flushed %zu bytes", n);
+        Curl_bufq_skip(&ws->sendbuf, n);
       }
     }
   }
@@ -1058,8 +1057,8 @@ CURL_EXTERN CURLcode curl_ws_send(CURL *data, const void *buffer,
                                   unsigned int flags)
 {
   struct websocket *ws;
-  ssize_t nwritten, n;
-  size_t space;
+  ssize_t n;
+  size_t nwritten, space;
   CURLcode result;
 
   *sent = 0;
@@ -1097,7 +1096,7 @@ CURL_EXTERN CURLcode curl_ws_send(CURL *data, const void *buffer,
 
     infof(data, "WS: wanted to send %zu bytes, sent %zu bytes",
           buflen, nwritten);
-    *sent = (nwritten >= 0)? (size_t)nwritten : 0;
+    *sent = nwritten;
     return result;
   }