Instead of immediately reading all data a peer requests into outgoing net

buffers, we put placeholder buffers on the list and fill them as they are
needed. At most 4 blocks will be filled per peer we upload to. This number
should probably be made tunable or be based on SO_SNDBUF or something.
Anyway, this should lower btpd's memory usage if one has many uploads.

btpd/net.c

@@ -134,6 +134,8 @@ net_read32(const void *buf)
         | (uint16_t)*(p + 2) << 8 | *(p + 3);
 }
 
+#define BLOCK_MEM_COUNT 4
+
 static unsigned long
 net_write(struct peer *p, unsigned long wmax)
 {
@@ -143,13 +145,29 @@ net_write(struct peer *p, unsigned long wmax)
     int limited;
     ssize_t nwritten;
     unsigned long bcount;
+    int block_count = 0;
 
     limited = wmax > 0;
 
     niov = 0;
     assert((nl = BTPDQ_FIRST(&p->outq)) != NULL);
+    if (nl->nb->type == NB_TORRENTDATA)
+        block_count = 1;
     while ((niov < IOV_MAX && nl != NULL
                && (!limited || (limited && wmax > 0)))) {
+        if (nl->nb->type == NB_PIECE) {
+            if (block_count >= BLOCK_MEM_COUNT)
+                break;
+            struct net_buf *tdata = BTPDQ_NEXT(nl, entry)->nb;
+            if (tdata->buf == NULL) {
+                if (nb_torrentdata_fill(tdata, p->n->tp, nb_get_index(nl->nb),
+                        nb_get_begin(nl->nb), nb_get_length(nl->nb)) != 0) {
+                    peer_kill(p);
+                    return 0;
+                }
+            }
+            block_count++;
+        }
         if (niov > 0) {
             iov[niov].iov_base = nl->nb->buf;
             iov[niov].iov_len = nl->nb->len;

btpd/net_buf.c

@@ -90,13 +90,28 @@ nb_create_piece(uint32_t index, uint32_t begin, size_t blen)
 }
 
 struct net_buf *
-nb_create_torrentdata(char *block, size_t blen)
+nb_create_torrentdata(void)
 {
     struct net_buf *out;
-    out = nb_create_set(NB_TORRENTDATA, block, blen, kill_buf_free);
+    out = nb_create_set(NB_TORRENTDATA, NULL, 0, kill_buf_no);
     return out;
 }
 
+int
+nb_torrentdata_fill(struct net_buf *nb, struct torrent *tp, uint32_t index,
+    uint32_t begin, uint32_t length)
+{
+    int err;
+    uint8_t *content;
+    assert(nb->type == NB_TORRENTDATA && nb->buf == NULL);
+    if ((err = cm_get_bytes(tp, index, begin, length, &content)) != 0)
+        return err;
+    nb->buf = content;
+    nb->len = length;
+    nb->kill_buf = kill_buf_free;
+    return 0;
+}
+
 struct net_buf *
 nb_create_request(uint32_t index, uint32_t begin, uint32_t length)
 {

btpd/net_buf.h

@@ -36,7 +36,7 @@ struct peer;
 
 struct net_buf *nb_create_keepalive(void);
 struct net_buf *nb_create_piece(uint32_t index, uint32_t begin, size_t blen);
-struct net_buf *nb_create_torrentdata(char *block, size_t blen);
+struct net_buf *nb_create_torrentdata(void);
 struct net_buf *nb_create_request(uint32_t index,
     uint32_t begin, uint32_t length);
 struct net_buf *nb_create_cancel(uint32_t index,
@@ -51,6 +51,9 @@ struct net_buf *nb_create_bitfield(struct torrent *tp);
 struct net_buf *nb_create_bitdata(struct torrent *tp);
 struct net_buf *nb_create_shake(struct torrent *tp);
 
+int nb_torrentdata_fill(struct net_buf *nb, struct torrent *tp, uint32_t index,
+    uint32_t begin, uint32_t length);
+
 int nb_drop(struct net_buf *nb);
 void nb_hold(struct net_buf *nb);
 

btpd/peer.c

@@ -489,16 +489,13 @@ peer_on_request(struct peer *p, uint32_t index, uint32_t begin,
     btpd_log(BTPD_L_MSG, "received request(%u,%u,%u) from %p\n",
         index, begin, length, p);
     if ((p->flags & PF_NO_REQUESTS) == 0) {
-        uint8_t *content;
-        if (cm_get_bytes(p->n->tp, index, begin, length, &content) == 0) {
-            peer_send(p, nb_create_piece(index, begin, length));
-            peer_send(p, nb_create_torrentdata(content, length));
-            p->npiece_msgs++;
-            if (p->npiece_msgs >= MAXPIECEMSGS) {
-                peer_send(p, nb_create_choke());
-                peer_send(p, nb_create_unchoke());
-                p->flags |= PF_NO_REQUESTS;
-            }
+        peer_send(p, nb_create_piece(index, begin, length));
+        peer_send(p, nb_create_torrentdata());
+        p->npiece_msgs++;
+        if (p->npiece_msgs >= MAXPIECEMSGS) {
+            peer_send(p, nb_create_choke());
+            peer_send(p, nb_create_unchoke());
+            p->flags |= PF_NO_REQUESTS;
         }
     }
 }